| @@ -93,158 +93,152 @@ struct mlx5_pagefault { · 92 unmodified lines |
| 93 | 93 | |
| 94 | 94 | static u64 mlx5_imr_ksm_entries; |
| 95 | 95 | |
| 96 | | −static int check_parent(struct ib_umem_odp *odp, |
| 97 | | − struct mlx5_ib_mr *parent) |
| 96 | +void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, |
| 97 | + struct mlx5_ib_mr *imr, int flags) |
| 98 | 98 | { |
| 99 | | − struct mlx5_ib_mr *mr = odp->private; |
| 100 | | − |
| 101 | | − return mr && mr->parent == parent && !odp->dying; |
| 102 | | −} |
| 103 | | − |
| 104 | | −static struct ib_ucontext_per_mm *mr_to_per_mm(struct mlx5_ib_mr *mr) |
| 105 | | −{ |
| 106 | | − if (WARN_ON(!mr || !is_odp_mr(mr))) |
| 107 | | − return NULL; |
| 108 | | − |
| 109 | | − return to_ib_umem_odp(mr->umem)->per_mm; |
| 110 | | −} |
| 111 | | − |
| 112 | | −static struct ib_umem_odp *odp_next(struct ib_umem_odp *odp) |
| 113 | | −{ |
| 114 | | − struct mlx5_ib_mr *mr = odp->private, *parent = mr->parent; |
| 115 | | − struct ib_ucontext_per_mm *per_mm = odp->per_mm; |
| 116 | | − struct rb_node *rb; |
| 117 | | − |
| 118 | | − down_read(&per_mm->umem_rwsem); |
| 119 | | − while (1) { |
| 120 | | − rb = rb_next(&odp->interval_tree.rb); |
| 121 | | − if (!rb) |
| 122 | | − goto not_found; |
| 123 | | − odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); |
| 124 | | − if (check_parent(odp, parent)) |
| 125 | | − goto end; |
| 126 | | − } |
| 127 | | −not_found: |
| 128 | | − odp = NULL; |
| 129 | | −end: |
| 130 | | − up_read(&per_mm->umem_rwsem); |
| 131 | | − return odp; |
| 132 | | −} |
| 133 | | − |
| 134 | | −static struct ib_umem_odp *odp_lookup(u64 start, u64 length, |
| 135 | | − struct mlx5_ib_mr *parent) |
| 136 | | −{ |
| 137 | | − struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(parent); |
| 138 | | − struct ib_umem_odp *odp; |
| 139 | | − struct rb_node *rb; |
| 140 | | − |
| 141 | | − down_read(&per_mm->umem_rwsem); |
| 142 | | − odp = rbt_ib_umem_lookup(&per_mm->umem_tree, start, length); |
| 143 | | − if (!odp) |
| 144 | | − goto end; |
| 145 | | − |
| 146 | | − while (1) { |
| 147 | | − if (check_parent(odp, parent)) |
| 148 | | − goto end; |
| 149 | | − rb = rb_next(&odp->interval_tree.rb); |
| 150 | | − if (!rb) |
| 151 | | − goto not_found; |
| 152 | | − odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); |
| 153 | | − if (ib_umem_start(odp) > start + length) |
| 154 | | − goto not_found; |
| 155 | | − } |
| 156 | | −not_found: |
| 157 | | − odp = NULL; |
| 158 | | −end: |
| 159 | | − up_read(&per_mm->umem_rwsem); |
| 160 | | − return odp; |
| 161 | | −} |
| 162 | | − |
| 163 | | −void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, |
| 164 | | − size_t nentries, struct mlx5_ib_mr *mr, int flags) |
| 165 | | −{ |
| 166 | | − struct ib_pd *pd = mr->ibmr.pd; |
| 167 | | − struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 168 | | − struct ib_umem_odp *odp; |
| 169 | | − unsigned long va; |
| 170 | | − int i; |
| 99 | + struct mlx5_klm *end = pklm + nentries; |
| 171 | 100 | |
| 172 | 101 | if (flags & MLX5_IB_UPD_XLT_ZAP) { |
| 173 | | − for (i = 0; i < nentries; i++, pklm++) { |
| 102 | + for (; pklm != end; pklm++, idx++) { |
| 174 | 103 | pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); |
| 175 | | − pklm->key = cpu_to_be32(dev->null_mkey); |
| 104 | + pklm->key = cpu_to_be32(imr->dev->null_mkey); |
| 176 | 105 | pklm->va = 0; |
| 177 | 106 | } |
| 178 | 107 | return; |
| 179 | 108 | } |
| 180 | 109 | |
| 181 | 110 | /* |
| 182 | | − * The locking here is pretty subtle. Ideally the implicit children |
| 183 | | − * list would be protected by the umem_mutex, however that is not |
| 111 | + * The locking here is pretty subtle. Ideally the implicit_children |
| 112 | + * xarray would be protected by the umem_mutex, however that is not |
| 184 | 113 | * possible. Instead this uses a weaker update-then-lock pattern: |
| 185 | 114 | * |
| 186 | 115 | * srcu_read_lock() |
| 187 | | − * <change children list> |
| 116 | + * xa_store() |
| 188 | 117 | * mutex_lock(umem_mutex) |
| 189 | 118 | * mlx5_ib_update_xlt() |
| 190 | 119 | * mutex_unlock(umem_mutex) |
| 191 | 120 | * destroy lkey |
| 192 | 121 | * |
| 193 | | − * ie any change the children list must be followed by the locked |
| 194 | | − * update_xlt before destroying. |
| 122 | + * ie any change the xarray must be followed by the locked update_xlt |
| 123 | + * before destroying. |
| 195 | 124 | * |
| 196 | 125 | * The umem_mutex provides the acquire/release semantic needed to make |
| 197 | | − * the children list visible to a racing thread. While SRCU is not |
| 126 | + * the xa_store() visible to a racing thread. While SRCU is not |
| 198 | 127 | * technically required, using it gives consistent use of the SRCU |
| 199 | | − * locking around the children list. |
| 128 | + * locking around the xarray. |
| 200 | 129 | */ |
| 201 | | − lockdep_assert_held(&to_ib_umem_odp(mr->umem)->umem_mutex); |
| 202 | | − lockdep_assert_held(&mr->dev->mr_srcu); |
| 130 | + lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex); |
| 131 | + lockdep_assert_held(&imr->dev->odp_srcu); |
| 203 | 132 | |
| 204 | | − odp = odp_lookup(offset * MLX5_IMR_MTT_SIZE, |
| 205 | | − nentries * MLX5_IMR_MTT_SIZE, mr); |
| 133 | + for (; pklm != end; pklm++, idx++) { |
| 134 | + struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx); |
| 206 | 135 | |
| 207 | | − for (i = 0; i < nentries; i++, pklm++) { |
| 208 | 136 | pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); |
| 209 | | − va = (offset + i) * MLX5_IMR_MTT_SIZE; |
| 210 | | − if (odp && ib_umem_start(odp) == va) { |
| 211 | | − struct mlx5_ib_mr *mtt = odp->private; |
| 212 | | − |
| 137 | + if (mtt) { |
| 213 | 138 | pklm->key = cpu_to_be32(mtt->ibmr.lkey); |
| 214 | | − odp = odp_next(odp); |
| 139 | + pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE); |
| 215 | 140 | } else { |
| 216 | | − pklm->key = cpu_to_be32(dev->null_mkey); |
| 141 | + pklm->key = cpu_to_be32(imr->dev->null_mkey); |
| 142 | + pklm->va = 0; |
| 217 | 143 | } |
| 218 | | − mlx5_ib_dbg(dev, "[%d] va %lx key %x\n", |
| 219 | | − i, va, be32_to_cpu(pklm->key)); |
| 220 | 144 | } |
| 221 | 145 | } |
| 222 | 146 | |
| 223 | | −static void mr_leaf_free_action(struct work_struct *work) |
| 147 | +static void dma_fence_odp_mr(struct mlx5_ib_mr *mr) |
| 148 | +{ |
| 149 | + struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); |
| 150 | + |
| 151 | + /* Ensure mlx5_ib_invalidate_range() will not touch the MR any more */ |
| 152 | + mutex_lock(&odp->umem_mutex); |
| 153 | + if (odp->npages) { |
| 154 | + mlx5_mr_cache_invalidate(mr); |
| 155 | + ib_umem_odp_unmap_dma_pages(odp, ib_umem_start(odp), |
| 156 | + ib_umem_end(odp)); |
| 157 | + WARN_ON(odp->npages); |
| 158 | + } |
| 159 | + odp->private = NULL; |
| 160 | + mutex_unlock(&odp->umem_mutex); |
| 161 | + |
| 162 | + if (!mr->allocated_from_cache) { |
| 163 | + mlx5_core_destroy_mkey(mr->dev->mdev, &mr->mmkey); |
| 164 | + WARN_ON(mr->descs); |
| 165 | + } |
| 166 | +} |
| 167 | + |
| 168 | +/* |
| 169 | + * This must be called after the mr has been removed from implicit_children |
| 170 | + * and the SRCU synchronized. NOTE: The MR does not necessarily have to be |
| 171 | + * empty here, parallel page faults could have raced with the free process and |
| 172 | + * added pages to it. |
| 173 | + */ |
| 174 | +static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt) |
| 224 | 175 | { |
| 225 | | − struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work); |
| 226 | | − int idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; |
| 227 | | − struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent; |
| 176 | + struct mlx5_ib_mr *imr = mr->parent; |
| 228 | 177 | struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); |
| 178 | + struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); |
| 179 | + unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; |
| 229 | 180 | int srcu_key; |
| 230 | 181 | |
| 231 | | − mr->parent = NULL; |
| 232 | | − synchronize_srcu(&mr->dev->mr_srcu); |
| 182 | + /* implicit_child_mr's are not allowed to have deferred work */ |
| 183 | + WARN_ON(atomic_read(&mr->num_deferred_work)); |
| 233 | 184 | |
| 234 | | − if (smp_load_acquire(&imr->live)) { |
| 235 | | − srcu_key = srcu_read_lock(&mr->dev->mr_srcu); |
| 185 | + if (need_imr_xlt) { |
| 186 | + srcu_key = srcu_read_lock(&mr->dev->odp_srcu); |
| 236 | 187 | mutex_lock(&odp_imr->umem_mutex); |
| 237 | | − mlx5_ib_update_xlt(imr, idx, 1, 0, |
| 188 | + mlx5_ib_update_xlt(mr->parent, idx, 1, 0, |
| 238 | 189 | MLX5_IB_UPD_XLT_INDIRECT | |
| 239 | 190 | MLX5_IB_UPD_XLT_ATOMIC); |
| 240 | 191 | mutex_unlock(&odp_imr->umem_mutex); |
| 241 | | − srcu_read_unlock(&mr->dev->mr_srcu, srcu_key); |
| 192 | + srcu_read_unlock(&mr->dev->odp_srcu, srcu_key); |
| 242 | 193 | } |
| 243 | | − ib_umem_odp_release(odp); |
| 194 | + |
| 195 | + dma_fence_odp_mr(mr); |
| 196 | + |
| 197 | + mr->parent = NULL; |
| 244 | 198 | mlx5_mr_cache_free(mr->dev, mr); |
| 199 | + ib_umem_odp_release(odp); |
| 200 | + atomic_dec(&imr->num_deferred_work); |
| 201 | +} |
| 202 | + |
| 203 | +static void free_implicit_child_mr_work(struct work_struct *work) |
| 204 | +{ |
| 205 | + struct mlx5_ib_mr *mr = |
| 206 | + container_of(work, struct mlx5_ib_mr, odp_destroy.work); |
| 207 | + |
| 208 | + free_implicit_child_mr(mr, true); |
| 209 | +} |
| 210 | + |
| 211 | +static void free_implicit_child_mr_rcu(struct rcu_head *head) |
| 212 | +{ |
| 213 | + struct mlx5_ib_mr *mr = |
| 214 | + container_of(head, struct mlx5_ib_mr, odp_destroy.rcu); |
| 245 | 215 | |
| 246 | | − if (atomic_dec_and_test(&imr->num_leaf_free)) |
| 247 | | − wake_up(&imr->q_leaf_free); |
| 216 | + /* Freeing a MR is a sleeping operation, so bounce to a work queue */ |
| 217 | + INIT_WORK(&mr->odp_destroy.work, free_implicit_child_mr_work); |
| 218 | + queue_work(system_unbound_wq, &mr->odp_destroy.work); |
| 219 | +} |
| 220 | + |
| 221 | +static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr) |
| 222 | +{ |
| 223 | + struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); |
| 224 | + unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; |
| 225 | + struct mlx5_ib_mr *imr = mr->parent; |
| 226 | + |
| 227 | + xa_lock(&imr->implicit_children); |
| 228 | + /* |
| 229 | + * This can race with mlx5_ib_free_implicit_mr(), the first one to |
| 230 | + * reach the xa lock wins the race and destroys the MR. |
| 231 | + */ |
| 232 | + if (__xa_cmpxchg(&imr->implicit_children, idx, mr, NULL, GFP_ATOMIC) != |
| 233 | + mr) |
| 234 | + goto out_unlock; |
| 235 | + |
| 236 | + atomic_inc(&imr->num_deferred_work); |
| 237 | + call_srcu(&mr->dev->odp_srcu, &mr->odp_destroy.rcu, |
| 238 | + free_implicit_child_mr_rcu); |
| 239 | + |
| 240 | +out_unlock: |
| 241 | + xa_unlock(&imr->implicit_children); |
| 248 | 242 | } |
| 249 | 243 | |
| 250 | 244 | void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, |
| @@ -258,16 +252,15 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, · 7 unmodified lines |
| 258 | 252 | int in_block = 0; |
| 259 | 253 | u64 addr; |
| 260 | 254 | |
| 261 | | − if (!umem_odp) { |
| 262 | | − pr_err("invalidation called on NULL umem or non-ODP umem\n"); |
| 263 | | − return; |
| 264 | | − } |
| 265 | | − |
| 255 | + mutex_lock(&umem_odp->umem_mutex); |
| 256 | + /* |
| 257 | + * If npages is zero then umem_odp->private may not be setup yet. This |
| 258 | + * does not complete until after the first page is mapped for DMA. |
| 259 | + */ |
| 260 | + if (!umem_odp->npages) |
| 261 | + goto out; |
| 266 | 262 | mr = umem_odp->private; |
| 267 | 263 | |
| 268 | | − if (!mr || !mr->ibmr.pd) |
| 269 | | − return; |
| 270 | | − |
| 271 | 264 | start = max_t(u64, ib_umem_start(umem_odp), start); |
| 272 | 265 | end = min_t(u64, ib_umem_end(umem_odp), end); |
| 273 | 266 | |
| @@ -277,7 +270,6 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, · 3 unmodified lines |
| 277 | 270 | * overwrite the same MTTs. Concurent invalidations might race us, |
| 278 | 271 | * but they will write 0s as well, so no difference in the end result. |
| 279 | 272 | */ |
| 280 | | − mutex_lock(&umem_odp->umem_mutex); |
| 281 | 273 | for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { |
| 282 | 274 | idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; |
| 283 | 275 | /* |
| @@ -323,14 +315,9 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, · 39 unmodified lines |
| 323 | 315 | |
| 324 | 316 | ib_umem_odp_unmap_dma_pages(umem_odp, start, end); |
| 325 | 317 | |
| 326 | | − |
| 327 | | − if (unlikely(!umem_odp->npages && mr->parent && |
| 328 | | − !umem_odp->dying)) { |
| 329 | | − WRITE_ONCE(mr->live, 0); |
| 330 | | − umem_odp->dying = 1; |
| 331 | | − atomic_inc(&mr->parent->num_leaf_free); |
| 332 | | − schedule_work(&umem_odp->work); |
| 333 | | − } |
| 318 | + if (unlikely(!umem_odp->npages && mr->parent)) |
| 319 | + destroy_unused_implicit_child_mr(mr); |
| 320 | +out: |
| 334 | 321 | mutex_unlock(&umem_odp->umem_mutex); |
| 335 | 322 | } |
| 336 | 323 | |
| @@ -422,239 +409,213 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, · 85 unmodified lines |
| 422 | 409 | wq_num, err); |
| 423 | 410 | } |
| 424 | 411 | |
| 425 | | −static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd, |
| 426 | | − struct ib_umem_odp *umem_odp, |
| 427 | | − bool ksm, int access_flags) |
| 412 | +static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, |
| 413 | + unsigned long idx) |
| 428 | 414 | { |
| 429 | | − struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 415 | + struct ib_umem_odp *odp; |
| 430 | 416 | struct mlx5_ib_mr *mr; |
| 417 | + struct mlx5_ib_mr *ret; |
| 431 | 418 | int err; |
| 432 | 419 | |
| 433 | | − mr = mlx5_mr_cache_alloc(dev, ksm ? MLX5_IMR_KSM_CACHE_ENTRY : |
| 434 | | − MLX5_IMR_MTT_CACHE_ENTRY); |
| 420 | + odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem), |
| 421 | + idx * MLX5_IMR_MTT_SIZE, |
| 422 | + MLX5_IMR_MTT_SIZE); |
| 423 | + if (IS_ERR(odp)) |
| 424 | + return ERR_CAST(odp); |
| 435 | 425 | |
| 426 | + ret = mr = mlx5_mr_cache_alloc(imr->dev, MLX5_IMR_MTT_CACHE_ENTRY); |
| 436 | 427 | if (IS_ERR(mr)) |
| 437 | | − return mr; |
| 438 | | − |
| 439 | | − mr->ibmr.pd = pd; |
| 440 | | − |
| 441 | | − mr->dev = dev; |
| 442 | | − mr->access_flags = access_flags; |
| 443 | | − mr->mmkey.iova = 0; |
| 444 | | − mr->umem = &umem_odp->umem; |
| 445 | | − |
| 446 | | − if (ksm) { |
| 447 | | − err = mlx5_ib_update_xlt(mr, 0, |
| 448 | | − mlx5_imr_ksm_entries, |
| 449 | | − MLX5_KSM_PAGE_SHIFT, |
| 450 | | − MLX5_IB_UPD_XLT_INDIRECT | |
| 451 | | − MLX5_IB_UPD_XLT_ZAP | |
| 452 | | − MLX5_IB_UPD_XLT_ENABLE); |
| 453 | | − |
| 454 | | − } else { |
| 455 | | − err = mlx5_ib_update_xlt(mr, 0, |
| 456 | | − MLX5_IMR_MTT_ENTRIES, |
| 457 | | − PAGE_SHIFT, |
| 458 | | − MLX5_IB_UPD_XLT_ZAP | |
| 459 | | − MLX5_IB_UPD_XLT_ENABLE | |
| 460 | | − MLX5_IB_UPD_XLT_ATOMIC); |
| 461 | | − } |
| 462 | | − |
| 463 | | − if (err) |
| 464 | | − goto fail; |
| 428 | + goto out_umem; |
| 465 | 429 | |
| 430 | + mr->ibmr.pd = imr->ibmr.pd; |
| 431 | + mr->access_flags = imr->access_flags; |
| 432 | + mr->umem = &odp->umem; |
| 466 | 433 | mr->ibmr.lkey = mr->mmkey.key; |
| 467 | 434 | mr->ibmr.rkey = mr->mmkey.key; |
| 468 | | − |
| 469 | | − mlx5_ib_dbg(dev, "key %x dev %p mr %p\n", |
| 470 | | − mr->mmkey.key, dev->mdev, mr); |
| 471 | | − |
| 472 | | − return mr; |
| 473 | | − |
| 474 | | −fail: |
| 475 | | − mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); |
| 476 | | − mlx5_mr_cache_free(dev, mr); |
| 477 | | − |
| 478 | | − return ERR_PTR(err); |
| 479 | | −} |
| 480 | | − |
| 481 | | −static struct ib_umem_odp *implicit_mr_get_data(struct mlx5_ib_mr *mr, |
| 482 | | − u64 io_virt, size_t bcnt) |
| 483 | | −{ |
| 484 | | − struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.pd->device); |
| 485 | | − struct ib_umem_odp *odp, *result = NULL; |
| 486 | | − struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); |
| 487 | | − u64 addr = io_virt & MLX5_IMR_MTT_MASK; |
| 488 | | − int nentries = 0, start_idx = 0, ret; |
| 489 | | − struct mlx5_ib_mr *mtt; |
| 490 | | − |
| 491 | | − mutex_lock(&odp_mr->umem_mutex); |
| 492 | | − odp = odp_lookup(addr, 1, mr); |
| 493 | | − |
| 494 | | − mlx5_ib_dbg(dev, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n", |
| 495 | | − io_virt, bcnt, addr, odp); |
| 496 | | − |
| 497 | | −next_mr: |
| 498 | | − if (likely(odp)) { |
| 499 | | − if (nentries) |
| 500 | | − nentries++; |
| 501 | | − } else { |
| 502 | | − odp = ib_umem_odp_alloc_child(odp_mr, addr, MLX5_IMR_MTT_SIZE); |
| 503 | | − if (IS_ERR(odp)) { |
| 504 | | − mutex_unlock(&odp_mr->umem_mutex); |
| 505 | | − return ERR_CAST(odp); |
| 506 | | − } |
| 507 | | − |
| 508 | | − mtt = implicit_mr_alloc(mr->ibmr.pd, odp, 0, |
| 509 | | − mr->access_flags); |
| 510 | | − if (IS_ERR(mtt)) { |
| 511 | | − mutex_unlock(&odp_mr->umem_mutex); |
| 512 | | − ib_umem_odp_release(odp); |
| 513 | | − return ERR_CAST(mtt); |
| 514 | | − } |
| 515 | | − |
| 516 | | − odp->private = mtt; |
| 517 | | − mtt->umem = &odp->umem; |
| 518 | | − mtt->mmkey.iova = addr; |
| 519 | | − mtt->parent = mr; |
| 520 | | − INIT_WORK(&odp->work, mr_leaf_free_action); |
| 521 | | − |
| 522 | | − smp_store_release(&mtt->live, 1); |
| 523 | | − |
| 524 | | − if (!nentries) |
| 525 | | − start_idx = addr >> MLX5_IMR_MTT_SHIFT; |
| 526 | | − nentries++; |
| 527 | | − } |
| 528 | | − |
| 529 | | − /* Return first odp if region not covered by single one */ |
| 530 | | − if (likely(!result)) |
| 531 | | − result = odp; |
| 532 | | − |
| 533 | | − addr += MLX5_IMR_MTT_SIZE; |
| 534 | | − if (unlikely(addr < io_virt + bcnt)) { |
| 535 | | − odp = odp_next(odp); |
| 536 | | − if (odp && ib_umem_start(odp) != addr) |
| 537 | | − odp = NULL; |
| 538 | | − goto next_mr; |
| 435 | + mr->mmkey.iova = idx * MLX5_IMR_MTT_SIZE; |
| 436 | + mr->parent = imr; |
| 437 | + odp->private = mr; |
| 438 | + |
| 439 | + err = mlx5_ib_update_xlt(mr, 0, |
| 440 | + MLX5_IMR_MTT_ENTRIES, |
| 441 | + PAGE_SHIFT, |
| 442 | + MLX5_IB_UPD_XLT_ZAP | |
| 443 | + MLX5_IB_UPD_XLT_ENABLE); |
| 444 | + if (err) { |
| 445 | + ret = ERR_PTR(err); |
| 446 | + goto out_mr; |
| 539 | 447 | } |
| 540 | 448 | |
| 541 | | − if (unlikely(nentries)) { |
| 542 | | − ret = mlx5_ib_update_xlt(mr, start_idx, nentries, 0, |
| 543 | | − MLX5_IB_UPD_XLT_INDIRECT | |
| 544 | | − MLX5_IB_UPD_XLT_ATOMIC); |
| 545 | | − if (ret) { |
| 546 | | − mlx5_ib_err(dev, "Failed to update PAS\n"); |
| 547 | | − result = ERR_PTR(ret); |
| 449 | + /* |
| 450 | + * Once the store to either xarray completes any error unwind has to |
| 451 | + * use synchronize_srcu(). Avoid this with xa_reserve() |
| 452 | + */ |
| 453 | + ret = xa_cmpxchg(&imr->implicit_children, idx, NULL, mr, |
| 454 | + GFP_KERNEL); |
| 455 | + if (unlikely(ret)) { |
| 456 | + if (xa_is_err(ret)) { |
| 457 | + ret = ERR_PTR(xa_err(ret)); |
| 458 | + goto out_mr; |
| 548 | 459 | } |
| 460 | + /* |
| 461 | + * Another thread beat us to creating the child mr, use |
| 462 | + * theirs. |
| 463 | + */ |
| 464 | + goto out_mr; |
| 549 | 465 | } |
| 550 | 466 | |
| 551 | | − mutex_unlock(&odp_mr->umem_mutex); |
| 552 | | − return result; |
| 467 | + mlx5_ib_dbg(imr->dev, "key %x mr %p\n", mr->mmkey.key, mr); |
| 468 | + return mr; |
| 469 | + |
| 470 | +out_mr: |
| 471 | + mlx5_mr_cache_free(imr->dev, mr); |
| 472 | +out_umem: |
| 473 | + ib_umem_odp_release(odp); |
| 474 | + return ret; |
| 553 | 475 | } |
| 554 | 476 | |
| 555 | 477 | struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, |
| 556 | 478 | struct ib_udata *udata, |
| 557 | 479 | int access_flags) |
| 558 | 480 | { |
| 559 | | − struct mlx5_ib_mr *imr; |
| 481 | + struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device); |
| 560 | 482 | struct ib_umem_odp *umem_odp; |
| 483 | + struct mlx5_ib_mr *imr; |
| 484 | + int err; |
| 561 | 485 | |
| 562 | 486 | umem_odp = ib_umem_odp_alloc_implicit(udata, access_flags); |
| 563 | 487 | if (IS_ERR(umem_odp)) |
| 564 | 488 | return ERR_CAST(umem_odp); |
| 565 | 489 | |
| 566 | | − imr = implicit_mr_alloc(&pd->ibpd, umem_odp, 1, access_flags); |
| 490 | + imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY); |
| 567 | 491 | if (IS_ERR(imr)) { |
| 568 | | − ib_umem_odp_release(umem_odp); |
| 569 | | − return ERR_CAST(imr); |
| 492 | + err = PTR_ERR(imr); |
| 493 | + goto out_umem; |
| 570 | 494 | } |
| 571 | 495 | |
| 496 | + imr->ibmr.pd = &pd->ibpd; |
| 497 | + imr->access_flags = access_flags; |
| 498 | + imr->mmkey.iova = 0; |
| 499 | + imr->umem = &umem_odp->umem; |
| 500 | + imr->ibmr.lkey = imr->mmkey.key; |
| 501 | + imr->ibmr.rkey = imr->mmkey.key; |
| 572 | 502 | imr->umem = &umem_odp->umem; |
| 573 | | − init_waitqueue_head(&imr->q_leaf_free); |
| 574 | | − atomic_set(&imr->num_leaf_free, 0); |
| 575 | | − atomic_set(&imr->num_pending_prefetch, 0); |
| 576 | | − smp_store_release(&imr->live, 1); |
| 577 | | − |
| 578 | 503 | imr->is_odp_implicit = true; |
| 504 | + atomic_set(&imr->num_deferred_work, 0); |
| 505 | + xa_init(&imr->implicit_children); |
| 506 | + |
| 507 | + err = mlx5_ib_update_xlt(imr, 0, |
| 508 | + mlx5_imr_ksm_entries, |
| 509 | + MLX5_KSM_PAGE_SHIFT, |
| 510 | + MLX5_IB_UPD_XLT_INDIRECT | |
| 511 | + MLX5_IB_UPD_XLT_ZAP | |
| 512 | + MLX5_IB_UPD_XLT_ENABLE); |
| 513 | + if (err) |
| 514 | + goto out_mr; |
| 515 | + |
| 516 | + err = xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key), |
| 517 | + &imr->mmkey, GFP_KERNEL)); |
| 518 | + if (err) |
| 519 | + goto out_mr; |
| 579 | 520 | |
| 521 | + mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr); |
| 580 | 522 | return imr; |
| 523 | +out_mr: |
| 524 | + mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); |
| 525 | + mlx5_mr_cache_free(dev, imr); |
| 526 | +out_umem: |
| 527 | + ib_umem_odp_release(umem_odp); |
| 528 | + return ERR_PTR(err); |
| 581 | 529 | } |
| 582 | 530 | |
| 583 | 531 | void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) |
| 584 | 532 | { |
| 585 | | − struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr); |
| 586 | | − struct rb_node *node; |
| 533 | + struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); |
| 534 | + struct mlx5_ib_dev *dev = imr->dev; |
| 535 | + struct list_head destroy_list; |
| 536 | + struct mlx5_ib_mr *mtt; |
| 537 | + struct mlx5_ib_mr *tmp; |
| 538 | + unsigned long idx; |
| 587 | 539 | |
| 588 | | − down_read(&per_mm->umem_rwsem); |
| 589 | | − for (node = rb_first_cached(&per_mm->umem_tree); node; |
| 590 | | − node = rb_next(node)) { |
| 591 | | − struct ib_umem_odp *umem_odp = |
| 592 | | − rb_entry(node, struct ib_umem_odp, interval_tree.rb); |
| 593 | | − struct mlx5_ib_mr *mr = umem_odp->private; |
| 540 | + INIT_LIST_HEAD(&destroy_list); |
| 594 | 541 | |
| 595 | | − if (mr->parent != imr) |
| 596 | | − continue; |
| 542 | + xa_erase(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key)); |
| 543 | + /* |
| 544 | + * This stops the SRCU protected page fault path from touching either |
| 545 | + * the imr or any children. The page fault path can only reach the |
| 546 | + * children xarray via the imr. |
| 547 | + */ |
| 548 | + synchronize_srcu(&dev->odp_srcu); |
| 597 | 549 | |
| 598 | | − mutex_lock(&umem_odp->umem_mutex); |
| 599 | | − ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp), |
| 600 | | − ib_umem_end(umem_odp)); |
| 550 | + xa_lock(&imr->implicit_children); |
| 551 | + xa_for_each (&imr->implicit_children, idx, mtt) { |
| 552 | + __xa_erase(&imr->implicit_children, idx); |
| 553 | + list_add(&mtt->odp_destroy.elm, &destroy_list); |
| 554 | + } |
| 555 | + xa_unlock(&imr->implicit_children); |
| 601 | 556 | |
| 602 | | − if (umem_odp->dying) { |
| 603 | | − mutex_unlock(&umem_odp->umem_mutex); |
| 604 | | − continue; |
| 605 | | − } |
| 557 | + /* |
| 558 | + * num_deferred_work can only be incremented inside the odp_srcu, or |
| 559 | + * under xa_lock while the child is in the xarray. Thus at this point |
| 560 | + * it is only decreasing, and all work holding it is now on the wq. |
| 561 | + */ |
| 562 | + if (atomic_read(&imr->num_deferred_work)) { |
| 563 | + flush_workqueue(system_unbound_wq); |
| 564 | + WARN_ON(atomic_read(&imr->num_deferred_work)); |
| 565 | + } |
| 566 | + |
| 567 | + /* |
| 568 | + * Fence the imr before we destroy the children. This allows us to |
| 569 | + * skip updating the XLT of the imr during destroy of the child mkey |
| 570 | + * the imr points to. |
| 571 | + */ |
| 572 | + mlx5_mr_cache_invalidate(imr); |
| 573 | + |
| 574 | + list_for_each_entry_safe (mtt, tmp, &destroy_list, odp_destroy.elm) |
| 575 | + free_implicit_child_mr(mtt, false); |
| 576 | + |
| 577 | + mlx5_mr_cache_free(dev, imr); |
| 578 | + ib_umem_odp_release(odp_imr); |
| 579 | +} |
| 580 | + |
| 581 | +/** |
| 582 | + * mlx5_ib_fence_odp_mr - Stop all access to the ODP MR |
| 583 | + * @mr: to fence |
| 584 | + * |
| 585 | + * On return no parallel threads will be touching this MR and no DMA will be |
| 586 | + * active. |
| 587 | + */ |
| 588 | +void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr) |
| 589 | +{ |
| 590 | + /* Prevent new page faults and prefetch requests from succeeding */ |
| 591 | + xa_erase(&mr->dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)); |
| 592 | + |
| 593 | + /* Wait for all running page-fault handlers to finish. */ |
| 594 | + synchronize_srcu(&mr->dev->odp_srcu); |
| 606 | 595 | |
| 607 | | − umem_odp->dying = 1; |
| 608 | | − atomic_inc(&imr->num_leaf_free); |
| 609 | | − schedule_work(&umem_odp->work); |
| 610 | | − mutex_unlock(&umem_odp->umem_mutex); |
| 596 | + if (atomic_read(&mr->num_deferred_work)) { |
| 597 | + flush_workqueue(system_unbound_wq); |
| 598 | + WARN_ON(atomic_read(&mr->num_deferred_work)); |
| 611 | 599 | } |
| 612 | | − up_read(&per_mm->umem_rwsem); |
| 613 | 600 | |
| 614 | | − wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free)); |
| 601 | + dma_fence_odp_mr(mr); |
| 615 | 602 | } |
| 616 | 603 | |
| 617 | | −#define MLX5_PF_FLAGS_PREFETCH BIT(0) |
| 618 | 604 | #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) |
| 619 | | −static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, |
| 620 | | − u64 io_virt, size_t bcnt, u32 *bytes_mapped, |
| 621 | | − u32 flags) |
| 605 | +static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, |
| 606 | + u64 user_va, size_t bcnt, u32 *bytes_mapped, |
| 607 | + u32 flags) |
| 622 | 608 | { |
| 623 | | − int npages = 0, current_seq, page_shift, ret, np; |
| 624 | | − struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); |
| 609 | + int current_seq, page_shift, ret, np; |
| 625 | 610 | bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; |
| 626 | | − bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; |
| 627 | 611 | u64 access_mask; |
| 628 | 612 | u64 start_idx, page_mask; |
| 629 | | − struct ib_umem_odp *odp; |
| 630 | | − size_t size; |
| 631 | | − |
| 632 | | − if (odp_mr->is_implicit_odp) { |
| 633 | | − odp = implicit_mr_get_data(mr, io_virt, bcnt); |
| 634 | | − |
| 635 | | − if (IS_ERR(odp)) |
| 636 | | − return PTR_ERR(odp); |
| 637 | | − mr = odp->private; |
| 638 | | − } else { |
| 639 | | − odp = odp_mr; |
| 640 | | − } |
| 641 | | − |
| 642 | | −next_mr: |
| 643 | | − size = min_t(size_t, bcnt, ib_umem_end(odp) - io_virt); |
| 644 | 613 | |
| 645 | 614 | page_shift = odp->page_shift; |
| 646 | 615 | page_mask = ~(BIT(page_shift) - 1); |
| 647 | | − start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; |
| 616 | + start_idx = (user_va - (mr->mmkey.iova & page_mask)) >> page_shift; |
| 648 | 617 | access_mask = ODP_READ_ALLOWED_BIT; |
| 649 | 618 | |
| 650 | | − if (prefetch && !downgrade && !odp->umem.writable) { |
| 651 | | − /* prefetch with write-access must |
| 652 | | − * be supported by the MR |
| 653 | | − */ |
| 654 | | − ret = -EINVAL; |
| 655 | | − goto out; |
| 656 | | − } |
| 657 | | − |
| 658 | 619 | if (odp->umem.writable && !downgrade) |
| 659 | 620 | access_mask |= ODP_WRITE_ALLOWED_BIT; |
| 660 | 621 | |
| @@ -665,13 +626,10 @@ next_mr: · 4 unmodified lines |
| 665 | 626 | */ |
| 666 | 627 | smp_rmb(); |
| 667 | 628 | |
| 668 | | − ret = ib_umem_odp_map_dma_pages(odp, io_virt, size, access_mask, |
| 669 | | − current_seq); |
| 670 | | − |
| 671 | | − if (ret < 0) |
| 672 | | − goto out; |
| 673 | | − |
| 674 | | − np = ret; |
| 629 | + np = ib_umem_odp_map_dma_pages(odp, user_va, bcnt, access_mask, |
| 630 | + current_seq); |
| 631 | + if (np < 0) |
| 632 | + return np; |
| 675 | 633 | |
| 676 | 634 | mutex_lock(&odp->umem_mutex); |
| 677 | 635 | if (!ib_umem_mmu_notifier_retry(odp, current_seq)) { |
| @@ -689,35 +647,19 @@ next_mr: · 11 unmodified lines |
| 689 | 647 | |
| 690 | 648 | if (ret < 0) { |
| 691 | 649 | if (ret != -EAGAIN) |
| 692 | | − mlx5_ib_err(dev, "Failed to update mkey page tables\n"); |
| 650 | + mlx5_ib_err(mr->dev, |
| 651 | + "Failed to update mkey page tables\n"); |
| 693 | 652 | goto out; |
| 694 | 653 | } |
| 695 | 654 | |
| 696 | 655 | if (bytes_mapped) { |
| 697 | 656 | u32 new_mappings = (np << page_shift) - |
| 698 | | − (io_virt - round_down(io_virt, 1 << page_shift)); |
| 699 | | − *bytes_mapped += min_t(u32, new_mappings, size); |
| 700 | | − } |
| 701 | | − |
| 702 | | − npages += np << (page_shift - PAGE_SHIFT); |
| 703 | | − bcnt -= size; |
| 657 | + (user_va - round_down(user_va, 1 << page_shift)); |
| 704 | 658 | |
| 705 | | − if (unlikely(bcnt)) { |
| 706 | | − struct ib_umem_odp *next; |
| 707 | | − |
| 708 | | − io_virt += size; |
| 709 | | − next = odp_next(odp); |
| 710 | | − if (unlikely(!next || ib_umem_start(next) != io_virt)) { |
| 711 | | − mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n", |
| 712 | | − io_virt, next); |
| 713 | | − return -EAGAIN; |
| 714 | | − } |
| 715 | | − odp = next; |
| 716 | | − mr = odp->private; |
| 717 | | − goto next_mr; |
| 659 | + *bytes_mapped += min_t(u32, new_mappings, bcnt); |
| 718 | 660 | } |
| 719 | 661 | |
| 720 | | − return npages; |
| 662 | + return np << (page_shift - PAGE_SHIFT); |
| 721 | 663 | |
| 722 | 664 | out: |
| 723 | 665 | if (ret == -EAGAIN) { |
| @@ -726,7 +668,7 @@ out: · 2 unmodified lines |
| 726 | 668 | if (!wait_for_completion_timeout(&odp->notifier_completion, |
| 727 | 669 | timeout)) { |
| 728 | 670 | mlx5_ib_warn( |
| 729 | | − dev, |
| 671 | + mr->dev, |
| 730 | 672 | "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n", |
| 731 | 673 | current_seq, odp->notifiers_seq, |
| 732 | 674 | odp->notifiers_count); |
| @@ -736,6 +678,109 @@ out: · 3 unmodified lines |
| 736 | 678 | return ret; |
| 737 | 679 | } |
| 738 | 680 | |
| 681 | +static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, |
| 682 | + struct ib_umem_odp *odp_imr, u64 user_va, |
| 683 | + size_t bcnt, u32 *bytes_mapped, u32 flags) |
| 684 | +{ |
| 685 | + unsigned long end_idx = (user_va + bcnt - 1) >> MLX5_IMR_MTT_SHIFT; |
| 686 | + unsigned long upd_start_idx = end_idx + 1; |
| 687 | + unsigned long upd_len = 0; |
| 688 | + unsigned long npages = 0; |
| 689 | + int err; |
| 690 | + int ret; |
| 691 | + |
| 692 | + if (unlikely(user_va >= mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE || |
| 693 | + mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE - user_va < bcnt)) |
| 694 | + return -EFAULT; |
| 695 | + |
| 696 | + /* Fault each child mr that intersects with our interval. */ |
| 697 | + while (bcnt) { |
| 698 | + unsigned long idx = user_va >> MLX5_IMR_MTT_SHIFT; |
| 699 | + struct ib_umem_odp *umem_odp; |
| 700 | + struct mlx5_ib_mr *mtt; |
| 701 | + u64 len; |
| 702 | + |
| 703 | + mtt = xa_load(&imr->implicit_children, idx); |
| 704 | + if (unlikely(!mtt)) { |
| 705 | + mtt = implicit_get_child_mr(imr, idx); |
| 706 | + if (IS_ERR(mtt)) { |
| 707 | + ret = PTR_ERR(mtt); |
| 708 | + goto out; |
| 709 | + } |
| 710 | + upd_start_idx = min(upd_start_idx, idx); |
| 711 | + upd_len = idx - upd_start_idx + 1; |
| 712 | + } |
| 713 | + |
| 714 | + umem_odp = to_ib_umem_odp(mtt->umem); |
| 715 | + len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) - |
| 716 | + user_va; |
| 717 | + |
| 718 | + ret = pagefault_real_mr(mtt, umem_odp, user_va, len, |
| 719 | + bytes_mapped, flags); |
| 720 | + if (ret < 0) |
| 721 | + goto out; |
| 722 | + user_va += len; |
| 723 | + bcnt -= len; |
| 724 | + npages += ret; |
| 725 | + } |
| 726 | + |
| 727 | + ret = npages; |
| 728 | + |
| 729 | + /* |
| 730 | + * Any time the implicit_children are changed we must perform an |
| 731 | + * update of the xlt before exiting to ensure the HW and the |
| 732 | + * implicit_children remains synchronized. |
| 733 | + */ |
| 734 | +out: |
| 735 | + if (likely(!upd_len)) |
| 736 | + return ret; |
| 737 | + |
| 738 | + /* |
| 739 | + * Notice this is not strictly ordered right, the KSM is updated after |
| 740 | + * the implicit_children is updated, so a parallel page fault could |
| 741 | + * see a MR that is not yet visible in the KSM. This is similar to a |
| 742 | + * parallel page fault seeing a MR that is being concurrently removed |
| 743 | + * from the KSM. Both of these improbable situations are resolved |
| 744 | + * safely by resuming the HW and then taking another page fault. The |
| 745 | + * next pagefault handler will see the new information. |
| 746 | + */ |
| 747 | + mutex_lock(&odp_imr->umem_mutex); |
| 748 | + err = mlx5_ib_update_xlt(imr, upd_start_idx, upd_len, 0, |
| 749 | + MLX5_IB_UPD_XLT_INDIRECT | |
| 750 | + MLX5_IB_UPD_XLT_ATOMIC); |
| 751 | + mutex_unlock(&odp_imr->umem_mutex); |
| 752 | + if (err) { |
| 753 | + mlx5_ib_err(imr->dev, "Failed to update PAS\n"); |
| 754 | + return err; |
| 755 | + } |
| 756 | + return ret; |
| 757 | +} |
| 758 | + |
| 759 | +/* |
| 760 | + * Returns: |
| 761 | + * -EFAULT: The io_virt->bcnt is not within the MR, it covers pages that are |
| 762 | + * not accessible, or the MR is no longer valid. |
| 763 | + * -EAGAIN/-ENOMEM: The operation should be retried |
| 764 | + * |
| 765 | + * -EINVAL/others: General internal malfunction |
| 766 | + * >0: Number of pages mapped |
| 767 | + */ |
| 768 | +static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, |
| 769 | + u32 *bytes_mapped, u32 flags) |
| 770 | +{ |
| 771 | + struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); |
| 772 | + |
| 773 | + if (!odp->is_implicit_odp) { |
| 774 | + if (unlikely(io_virt < ib_umem_start(odp) || |
| 775 | + ib_umem_end(odp) - io_virt < bcnt)) |
| 776 | + return -EFAULT; |
| 777 | + return pagefault_real_mr(mr, odp, io_virt, bcnt, bytes_mapped, |
| 778 | + flags); |
| 779 | + } |
| 780 | + return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped, |
| 781 | + flags); |
| 782 | +} |
| 783 | + |
| 739 | 784 | struct pf_frame { |
| 740 | 785 | struct pf_frame *next; |
| 741 | 786 | u32 key; |
| @@ -783,10 +828,9 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, · 41 unmodified lines |
| 783 | 828 | struct ib_pd *pd, u32 key, |
| 784 | 829 | u64 io_virt, size_t bcnt, |
| 785 | 830 | u32 *bytes_committed, |
| 786 | | − u32 *bytes_mapped, u32 flags) |
| 831 | + u32 *bytes_mapped) |
| 787 | 832 | { |
| 788 | 833 | int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; |
| 789 | | − bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; |
| 790 | 834 | struct pf_frame *head = NULL, *frame; |
| 791 | 835 | struct mlx5_core_mkey *mmkey; |
| 792 | 836 | struct mlx5_ib_mr *mr; |
| @@ -795,55 +839,39 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, · 2 unmodified lines |
| 795 | 839 | size_t offset; |
| 796 | 840 | int ndescs; |
| 797 | 841 | |
| 798 | | − srcu_key = srcu_read_lock(&dev->mr_srcu); |
| 842 | + srcu_key = srcu_read_lock(&dev->odp_srcu); |
| 799 | 843 | |
| 800 | 844 | io_virt += *bytes_committed; |
| 801 | 845 | bcnt -= *bytes_committed; |
| 802 | 846 | |
| 803 | 847 | next_mr: |
| 804 | | − mmkey = xa_load(&dev->mdev->priv.mkey_table, mlx5_base_mkey(key)); |
| 848 | + mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(key)); |
| 849 | + if (!mmkey) { |
| 850 | + mlx5_ib_dbg( |
| 851 | + dev, |
| 852 | + "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", |
| 853 | + key); |
| 854 | + if (bytes_mapped) |
| 855 | + *bytes_mapped += bcnt; |
| 856 | + /* |
| 857 | + * The user could specify a SGL with multiple lkeys and only |
| 858 | + * some of them are ODP. Treat the non-ODP ones as fully |
| 859 | + * faulted. |
| 860 | + */ |
| 861 | + ret = 0; |
| 862 | + goto srcu_unlock; |
| 863 | + } |
| 805 | 864 | if (!mkey_is_eq(mmkey, key)) { |
| 806 | 865 | mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); |
| 807 | 866 | ret = -EFAULT; |
| 808 | 867 | goto srcu_unlock; |
| 809 | 868 | } |
| 810 | 869 | |
| 811 | | − if (prefetch && mmkey->type != MLX5_MKEY_MR) { |
| 812 | | − mlx5_ib_dbg(dev, "prefetch is allowed only for MR\n"); |
| 813 | | − ret = -EINVAL; |
| 814 | | − goto srcu_unlock; |
| 815 | | − } |
| 816 | | − |
| 817 | 870 | switch (mmkey->type) { |
| 818 | 871 | case MLX5_MKEY_MR: |
| 819 | 872 | mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); |
| 820 | | − if (!smp_load_acquire(&mr->live) || !mr->ibmr.pd) { |
| 821 | | − mlx5_ib_dbg(dev, "got dead MR\n"); |
| 822 | | − ret = -EFAULT; |
| 823 | | − goto srcu_unlock; |
| 824 | | − } |
| 825 | | − |
| 826 | | − if (prefetch) { |
| 827 | | − if (!is_odp_mr(mr) || |
| 828 | | − mr->ibmr.pd != pd) { |
| 829 | | − mlx5_ib_dbg(dev, "Invalid prefetch request: %s\n", |
| 830 | | − is_odp_mr(mr) ? "MR is not ODP" : |
| 831 | | − "PD is not of the MR"); |
| 832 | | − ret = -EINVAL; |
| 833 | | − goto srcu_unlock; |
| 834 | | − } |
| 835 | | − } |
| 836 | 873 | |
| 837 | | − if (!is_odp_mr(mr)) { |
| 838 | | − mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", |
| 839 | | − key); |
| 840 | | − if (bytes_mapped) |
| 841 | | − *bytes_mapped += bcnt; |
| 842 | | − ret = 0; |
| 843 | | − goto srcu_unlock; |
| 844 | | − } |
| 845 | | − |
| 846 | | − ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped, flags); |
| 874 | + ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0); |
| 847 | 875 | if (ret < 0) |
| 848 | 876 | goto srcu_unlock; |
| 849 | 877 | |
| @@ -943,7 +971,7 @@ srcu_unlock: · 93 unmodified lines |
| 943 | 971 | } |
| 944 | 972 | kfree(out); |
| 945 | 973 | |
| 946 | | − srcu_read_unlock(&dev->mr_srcu, srcu_key); |
| 974 | + srcu_read_unlock(&dev->odp_srcu, srcu_key); |
| 947 | 975 | *bytes_committed = 0; |
| 948 | 976 | return ret ? ret : npages; |
| 949 | 977 | } |
| @@ -1024,7 +1052,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 74 unmodified lines |
| 1024 | 1052 | ret = pagefault_single_data_segment(dev, NULL, key, |
| 1025 | 1053 | io_virt, bcnt, |
| 1026 | 1054 | &pfault->bytes_committed, |
| 1027 | | − bytes_mapped, 0); |
| 1055 | + bytes_mapped); |
| 1028 | 1056 | if (ret < 0) |
| 1029 | 1057 | break; |
| 1030 | 1058 | npages += ret; |
| @@ -1307,8 +1335,7 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, · 276 unmodified lines |
| 1307 | 1335 | } |
| 1308 | 1336 | |
| 1309 | 1337 | ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, |
| 1310 | | − &pfault->bytes_committed, NULL, |
| 1311 | | − 0); |
| 1338 | + &pfault->bytes_committed, NULL); |
| 1312 | 1339 | if (ret == -EAGAIN) { |
| 1313 | 1340 | /* We're racing with an invalidation, don't prefetch */ |
| 1314 | 1341 | prefetch_activated = 0; |
| @@ -1335,8 +1362,7 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, · 20 unmodified lines |
| 1335 | 1362 | |
| 1336 | 1363 | ret = pagefault_single_data_segment(dev, NULL, rkey, address, |
| 1337 | 1364 | prefetch_len, |
| 1338 | | − &bytes_committed, NULL, |
| 1339 | | − 0); |
| 1365 | + &bytes_committed, NULL); |
| 1340 | 1366 | if (ret < 0 && ret != -EAGAIN) { |
| 1341 | 1367 | mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", |
| 1342 | 1368 | ret, pfault->token, address, prefetch_len); |
| @@ -1639,114 +1665,128 @@ int mlx5_ib_odp_init(void) · 296 unmodified lines |
| 1639 | 1665 | |
| 1640 | 1666 | struct prefetch_mr_work { |
| 1641 | 1667 | struct work_struct work; |
| 1642 | | − struct ib_pd *pd; |
| 1643 | 1668 | u32 pf_flags; |
| 1644 | 1669 | u32 num_sge; |
| 1645 | | − struct ib_sge sg_list[0]; |
| 1670 | + struct { |
| 1671 | + u64 io_virt; |
| 1672 | + struct mlx5_ib_mr *mr; |
| 1673 | + size_t length; |
| 1674 | + } frags[]; |
| 1646 | 1675 | }; |
| 1647 | 1676 | |
| 1648 | | −static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev, |
| 1649 | | − struct ib_sge *sg_list, u32 num_sge, |
| 1650 | | − u32 from) |
| 1677 | +static void destroy_prefetch_work(struct prefetch_mr_work *work) |
| 1651 | 1678 | { |
| 1652 | 1679 | u32 i; |
| 1653 | | − int srcu_key; |
| 1654 | | − |
| 1655 | | − srcu_key = srcu_read_lock(&dev->mr_srcu); |
| 1656 | | − |
| 1657 | | − for (i = from; i < num_sge; ++i) { |
| 1658 | | − struct mlx5_core_mkey *mmkey; |
| 1659 | | − struct mlx5_ib_mr *mr; |
| 1660 | | − |
| 1661 | | − mmkey = xa_load(&dev->mdev->priv.mkey_table, |
| 1662 | | − mlx5_base_mkey(sg_list[i].lkey)); |
| 1663 | | − mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); |
| 1664 | | − atomic_dec(&mr->num_pending_prefetch); |
| 1665 | | − } |
| 1666 | 1680 | |
| 1667 | | − srcu_read_unlock(&dev->mr_srcu, srcu_key); |
| 1681 | + for (i = 0; i < work->num_sge; ++i) |
| 1682 | + atomic_dec(&work->frags[i].mr->num_deferred_work); |
| 1683 | + kvfree(work); |
| 1668 | 1684 | } |
| 1669 | 1685 | |
| 1670 | | −static bool num_pending_prefetch_inc(struct ib_pd *pd, |
| 1671 | | − struct ib_sge *sg_list, u32 num_sge) |
| 1686 | +static struct mlx5_ib_mr * |
| 1687 | +get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, |
| 1688 | + u32 lkey) |
| 1672 | 1689 | { |
| 1673 | 1690 | struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 1674 | | − bool ret = true; |
| 1675 | | − u32 i; |
| 1691 | + struct mlx5_core_mkey *mmkey; |
| 1692 | + struct ib_umem_odp *odp; |
| 1693 | + struct mlx5_ib_mr *mr; |
| 1676 | 1694 | |
| 1677 | | − for (i = 0; i < num_sge; ++i) { |
| 1678 | | − struct mlx5_core_mkey *mmkey; |
| 1679 | | − struct mlx5_ib_mr *mr; |
| 1695 | + lockdep_assert_held(&dev->odp_srcu); |
| 1680 | 1696 | |
| 1681 | | − mmkey = xa_load(&dev->mdev->priv.mkey_table, |
| 1682 | | − mlx5_base_mkey(sg_list[i].lkey)); |
| 1683 | | − if (!mmkey || mmkey->key != sg_list[i].lkey) { |
| 1684 | | − ret = false; |
| 1685 | | − break; |
| 1686 | | − } |
| 1697 | + mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey)); |
| 1698 | + if (!mmkey || mmkey->key != lkey || mmkey->type != MLX5_MKEY_MR) |
| 1699 | + return NULL; |
| 1687 | 1700 | |
| 1688 | | − if (mmkey->type != MLX5_MKEY_MR) { |
| 1689 | | − ret = false; |
| 1690 | | − break; |
| 1691 | | − } |
| 1701 | + mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); |
| 1692 | 1702 | |
| 1693 | | − mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); |
| 1703 | + if (mr->ibmr.pd != pd) |
| 1704 | + return NULL; |
| 1694 | 1705 | |
| 1695 | | − if (!smp_load_acquire(&mr->live)) { |
| 1696 | | − ret = false; |
| 1697 | | − break; |
| 1698 | | − } |
| 1706 | + odp = to_ib_umem_odp(mr->umem); |
| 1699 | 1707 | |
| 1700 | | − if (mr->ibmr.pd != pd) { |
| 1701 | | − ret = false; |
| 1702 | | − break; |
| 1703 | | − } |
| 1708 | + /* prefetch with write-access must be supported by the MR */ |
| 1709 | + if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && |
| 1710 | + !odp->umem.writable) |
| 1711 | + return NULL; |
| 1704 | 1712 | |
| 1705 | | − atomic_inc(&mr->num_pending_prefetch); |
| 1706 | | − } |
| 1713 | + return mr; |
| 1714 | +} |
| 1707 | 1715 | |
| 1708 | | − if (!ret) |
| 1709 | | − num_pending_prefetch_dec(dev, sg_list, i, 0); |
| 1716 | +static void mlx5_ib_prefetch_mr_work(struct work_struct *w) |
| 1717 | +{ |
| 1718 | + struct prefetch_mr_work *work = |
| 1719 | + container_of(w, struct prefetch_mr_work, work); |
| 1720 | + u32 bytes_mapped = 0; |
| 1721 | + u32 i; |
| 1710 | 1722 | |
| 1711 | | − return ret; |
| 1723 | + for (i = 0; i < work->num_sge; ++i) |
| 1724 | + pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, |
| 1725 | + work->frags[i].length, &bytes_mapped, |
| 1726 | + work->pf_flags); |
| 1727 | + |
| 1728 | + destroy_prefetch_work(work); |
| 1712 | 1729 | } |
| 1713 | 1730 | |
| 1714 | | −static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, u32 pf_flags, |
| 1715 | | − struct ib_sge *sg_list, u32 num_sge) |
| 1731 | +static bool init_prefetch_work(struct ib_pd *pd, |
| 1732 | + enum ib_uverbs_advise_mr_advice advice, |
| 1733 | + u32 pf_flags, struct prefetch_mr_work *work, |
| 1734 | + struct ib_sge *sg_list, u32 num_sge) |
| 1716 | 1735 | { |
| 1717 | 1736 | u32 i; |
| 1718 | | − int ret = 0; |
| 1719 | | − struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 1737 | + |
| 1738 | + INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); |
| 1739 | + work->pf_flags = pf_flags; |
| 1720 | 1740 | |
| 1721 | 1741 | for (i = 0; i < num_sge; ++i) { |
| 1722 | | − struct ib_sge *sg = &sg_list[i]; |
| 1723 | | − int bytes_committed = 0; |
| 1742 | + work->frags[i].io_virt = sg_list[i].addr; |
| 1743 | + work->frags[i].length = sg_list[i].length; |
| 1744 | + work->frags[i].mr = |
| 1745 | + get_prefetchable_mr(pd, advice, sg_list[i].lkey); |
| 1746 | + if (!work->frags[i].mr) { |
| 1747 | + work->num_sge = i - 1; |
| 1748 | + if (i) |
| 1749 | + destroy_prefetch_work(work); |
| 1750 | + return false; |
| 1751 | + } |
| 1724 | 1752 | |
| 1725 | | − ret = pagefault_single_data_segment(dev, pd, sg->lkey, sg->addr, |
| 1726 | | − sg->length, |
| 1727 | | − &bytes_committed, NULL, |
| 1728 | | − pf_flags); |
| 1729 | | − if (ret < 0) |
| 1730 | | − break; |
| 1753 | + /* Keep the MR pointer will valid outside the SRCU */ |
| 1754 | + atomic_inc(&work->frags[i].mr->num_deferred_work); |
| 1731 | 1755 | } |
| 1732 | | − |
| 1733 | | − return ret < 0 ? ret : 0; |
| 1756 | + work->num_sge = num_sge; |
| 1757 | + return true; |
| 1734 | 1758 | } |
| 1735 | 1759 | |
| 1736 | | −static void mlx5_ib_prefetch_mr_work(struct work_struct *work) |
| 1760 | +static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, |
| 1761 | + enum ib_uverbs_advise_mr_advice advice, |
| 1762 | + u32 pf_flags, struct ib_sge *sg_list, |
| 1763 | + u32 num_sge) |
| 1737 | 1764 | { |
| 1738 | | − struct prefetch_mr_work *w = |
| 1739 | | − container_of(work, struct prefetch_mr_work, work); |
| 1765 | + struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 1766 | + u32 bytes_mapped = 0; |
| 1767 | + int srcu_key; |
| 1768 | + int ret = 0; |
| 1769 | + u32 i; |
| 1770 | + |
| 1771 | + srcu_key = srcu_read_lock(&dev->odp_srcu); |
| 1772 | + for (i = 0; i < num_sge; ++i) { |
| 1773 | + struct mlx5_ib_mr *mr; |
| 1740 | 1774 | |
| 1741 | | − if (ib_device_try_get(w->pd->device)) { |
| 1742 | | − mlx5_ib_prefetch_sg_list(w->pd, w->pf_flags, w->sg_list, |
| 1743 | | − w->num_sge); |
| 1744 | | − ib_device_put(w->pd->device); |
| 1775 | + mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); |
| 1776 | + if (!mr) { |
| 1777 | + ret = -ENOENT; |
| 1778 | + goto out; |
| 1779 | + } |
| 1780 | + ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length, |
| 1781 | + &bytes_mapped, pf_flags); |
| 1782 | + if (ret < 0) |
| 1783 | + goto out; |
| 1745 | 1784 | } |
| 1785 | + ret = 0; |
| 1746 | 1786 | |
| 1747 | | − num_pending_prefetch_dec(to_mdev(w->pd->device), w->sg_list, |
| 1748 | | − w->num_sge, 0); |
| 1749 | | − kvfree(w); |
| 1787 | +out: |
| 1788 | + srcu_read_unlock(&dev->odp_srcu, srcu_key); |
| 1789 | + return ret; |
| 1750 | 1790 | } |
| 1751 | 1791 | |
| 1752 | 1792 | int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, |
| @@ -1754,43 +1794,27 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, · 1 unmodified line |
| 1754 | 1794 | u32 flags, struct ib_sge *sg_list, u32 num_sge) |
| 1755 | 1795 | { |
| 1756 | 1796 | struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 1757 | | − u32 pf_flags = MLX5_PF_FLAGS_PREFETCH; |
| 1797 | + u32 pf_flags = 0; |
| 1758 | 1798 | struct prefetch_mr_work *work; |
| 1759 | | − bool valid_req; |
| 1760 | 1799 | int srcu_key; |
| 1761 | 1800 | |
| 1762 | 1801 | if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) |
| 1763 | 1802 | pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; |
| 1764 | 1803 | |
| 1765 | 1804 | if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) |
| 1766 | | − return mlx5_ib_prefetch_sg_list(pd, pf_flags, sg_list, |
| 1805 | + return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list, |
| 1767 | 1806 | num_sge); |
| 1768 | 1807 | |
| 1769 | | − work = kvzalloc(struct_size(work, sg_list, num_sge), GFP_KERNEL); |
| 1808 | + work = kvzalloc(struct_size(work, frags, num_sge), GFP_KERNEL); |
| 1770 | 1809 | if (!work) |
| 1771 | 1810 | return -ENOMEM; |
| 1772 | 1811 | |
| 1773 | | − memcpy(work->sg_list, sg_list, num_sge * sizeof(struct ib_sge)); |
| 1774 | | − |
| 1775 | | − /* It is guaranteed that the pd when work is executed is the pd when |
| 1776 | | − * work was queued since pd can't be destroyed while it holds MRs and |
| 1777 | | − * destroying a MR leads to flushing the workquque |
| 1778 | | − */ |
| 1779 | | − work->pd = pd; |
| 1780 | | − work->pf_flags = pf_flags; |
| 1781 | | − work->num_sge = num_sge; |
| 1782 | | − |
| 1783 | | − INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); |
| 1784 | | − |
| 1785 | | − srcu_key = srcu_read_lock(&dev->mr_srcu); |
| 1786 | | − |
| 1787 | | − valid_req = num_pending_prefetch_inc(pd, sg_list, num_sge); |
| 1788 | | − if (valid_req) |
| 1789 | | − queue_work(system_unbound_wq, &work->work); |
| 1790 | | − else |
| 1791 | | − kvfree(work); |
| 1792 | | − |
| 1793 | | − srcu_read_unlock(&dev->mr_srcu, srcu_key); |
| 1794 | | − |
| 1795 | | − return valid_req ? 0 : -EINVAL; |
| 1812 | + srcu_key = srcu_read_lock(&dev->odp_srcu); |
| 1813 | + if (!init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge)) { |
| 1814 | + srcu_read_unlock(&dev->odp_srcu, srcu_key); |
| 1815 | + return -EINVAL; |
| 1816 | + } |
| 1817 | + queue_work(system_unbound_wq, &work->work); |
| 1818 | + srcu_read_unlock(&dev->odp_srcu, srcu_key); |
| 1819 | + return 0; |
| 1796 | 1820 | } |