| @@ -24,6 +24,7 @@ · 23 unmodified lines |
| 24 | #include <linux/compat.h> | 24 | #include <linux/compat.h> |
| 25 | #include <linux/device.h> | 25 | #include <linux/device.h> |
| 26 | #include <linux/fs.h> | 26 | #include <linux/fs.h> |
| | 27 | #include <linux/highmem.h> |
| 27 | #include <linux/iommu.h> | 28 | #include <linux/iommu.h> |
| 28 | #include <linux/module.h> | 29 | #include <linux/module.h> |
| 29 | #include <linux/mm.h> | 30 | #include <linux/mm.h> |
| @@ -69,11 +70,15 @@ struct vfio_iommu { · 39 unmodified lines |
| 69 | struct rb_root dma_list; | 70 | struct rb_root dma_list; |
| 70 | struct blocking_notifier_head notifier; | 71 | struct blocking_notifier_head notifier; |
| 71 | unsigned int dma_avail; | 72 | unsigned int dma_avail; |
| | 73 | unsigned int vaddr_invalid_count; |
| 72 | uint64_t pgsize_bitmap; | 74 | uint64_t pgsize_bitmap; |
| | 75 | uint64_t num_non_pinned_groups; |
| | 76 | wait_queue_head_t vaddr_wait; |
| 73 | bool v2; | 77 | bool v2; |
| 74 | bool nesting; | 78 | bool nesting; |
| 75 | bool dirty_page_tracking; | 79 | bool dirty_page_tracking; |
| 76 | bool pinned_page_dirty_scope; | 80 | bool pinned_page_dirty_scope; |
| | 81 | bool container_open; |
| 77 | }; | 82 | }; |
| 78 | | 83 | |
| 79 | struct vfio_domain { | 84 | struct vfio_domain { |
| @@ -92,11 +97,20 @@ struct vfio_dma { · 12 unmodified lines |
| 92 | int prot; /* IOMMU_READ/WRITE */ | 97 | int prot; /* IOMMU_READ/WRITE */ |
| 93 | bool iommu_mapped; | 98 | bool iommu_mapped; |
| 94 | bool lock_cap; /* capable(CAP_IPC_LOCK) */ | 99 | bool lock_cap; /* capable(CAP_IPC_LOCK) */ |
| | 100 | bool vaddr_invalid; |
| 95 | struct task_struct *task; | 101 | struct task_struct *task; |
| 96 | struct rb_root pfn_list; /* Ex-user pinned pfn list */ | 102 | struct rb_root pfn_list; /* Ex-user pinned pfn list */ |
| 97 | unsigned long *bitmap; | 103 | unsigned long *bitmap; |
| 98 | }; | 104 | }; |
| 99 | | 105 | |
| | 106 | struct vfio_batch { |
| | 107 | struct page **pages; /* for pin_user_pages_remote */ |
| | 108 | struct page *fallback_page; /* if pages alloc fails */ |
| | 109 | int capacity; /* length of pages array */ |
| | 110 | int size; /* of batch currently */ |
| | 111 | int offset; /* of next entry in pages */ |
| | 112 | }; |
| | 113 | |
| 100 | struct vfio_group { | 114 | struct vfio_group { |
| 101 | struct iommu_group *iommu_group; | 115 | struct iommu_group *iommu_group; |
| 102 | struct list_head next; | 116 | struct list_head next; |
| @@ -143,12 +157,13 @@ struct vfio_regions { · 40 unmodified lines |
| 143 | #define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX) | 157 | #define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX) |
| 144 | #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) | 158 | #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) |
| 145 | | 159 | |
| | 160 | #define WAITED 1 |
| | 161 | |
| 146 | static int put_pfn(unsigned long pfn, int prot); | 162 | static int put_pfn(unsigned long pfn, int prot); |
| 147 | | 163 | |
| 148 | static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, | 164 | static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, |
| 149 | struct iommu_group *iommu_group); | 165 | struct iommu_group *iommu_group); |
| 150 | | 166 | |
| 151 | static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu); | | |
| 152 | /* | 167 | /* |
| 153 | * This code handles mapping and unmapping of user data buffers | 168 | * This code handles mapping and unmapping of user data buffers |
| 154 | * into DMA'ble space using the IOMMU | 169 | * into DMA'ble space using the IOMMU |
| @@ -173,6 +188,31 @@ static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, · 18 unmodified lines |
| 173 | return NULL; | 188 | return NULL; |
| 174 | } | 189 | } |
| 175 | | 190 | |
| | 191 | static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu, |
| | 192 | dma_addr_t start, size_t size) |
| | 193 | { |
| | 194 | struct rb_node *res = NULL; |
| | 195 | struct rb_node *node = iommu->dma_list.rb_node; |
| | 196 | struct vfio_dma *dma_res = NULL; |
| | 197 | |
| | 198 | while (node) { |
| | 199 | struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); |
| | 200 | |
| | 201 | if (start < dma->iova + dma->size) { |
| | 202 | res = node; |
| | 203 | dma_res = dma; |
| | 204 | if (start >= dma->iova) |
| | 205 | break; |
| | 206 | node = node->rb_left; |
| | 207 | } else { |
| | 208 | node = node->rb_right; |
| | 209 | } |
| | 210 | } |
| | 211 | if (res && size && dma_res->iova >= start + size) |
| | 212 | res = NULL; |
| | 213 | return res; |
| | 214 | } |
| | 215 | |
| 176 | static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new) | 216 | static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new) |
| 177 | { | 217 | { |
| 178 | struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL; | 218 | struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL; |
| @@ -236,6 +276,18 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) · 57 unmodified lines |
| 236 | } | 276 | } |
| 237 | } | 277 | } |
| 238 | | 278 | |
| | 279 | static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) |
| | 280 | { |
| | 281 | struct rb_node *n; |
| | 282 | unsigned long pgshift = __ffs(iommu->pgsize_bitmap); |
| | 283 | |
| | 284 | for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { |
| | 285 | struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); |
| | 286 | |
| | 287 | bitmap_set(dma->bitmap, 0, dma->size >> pgshift); |
| | 288 | } |
| | 289 | } |
| | 290 | |
| 239 | static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize) | 291 | static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize) |
| 240 | { | 292 | { |
| 241 | struct rb_node *n; | 293 | struct rb_node *n; |
| @@ -415,13 +467,54 @@ static int put_pfn(unsigned long pfn, int prot) · 173 unmodified lines |
| 415 | return 0; | 467 | return 0; |
| 416 | } | 468 | } |
| 417 | | 469 | |
| | 470 | #define VFIO_BATCH_MAX_CAPACITY (PAGE_SIZE / sizeof(struct page *)) |
| | 471 | |
| | 472 | static void vfio_batch_init(struct vfio_batch *batch) |
| | 473 | { |
| | 474 | batch->size = 0; |
| | 475 | batch->offset = 0; |
| | 476 | |
| | 477 | if (unlikely(disable_hugepages)) |
| | 478 | goto fallback; |
| | 479 | |
| | 480 | batch->pages = (struct page **) __get_free_page(GFP_KERNEL); |
| | 481 | if (!batch->pages) |
| | 482 | goto fallback; |
| | 483 | |
| | 484 | batch->capacity = VFIO_BATCH_MAX_CAPACITY; |
| | 485 | return; |
| | 486 | |
| | 487 | fallback: |
| | 488 | batch->pages = &batch->fallback_page; |
| | 489 | batch->capacity = 1; |
| | 490 | } |
| | 491 | |
| | 492 | static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma) |
| | 493 | { |
| | 494 | while (batch->size) { |
| | 495 | unsigned long pfn = page_to_pfn(batch->pages[batch->offset]); |
| | 496 | |
| | 497 | put_pfn(pfn, dma->prot); |
| | 498 | batch->offset++; |
| | 499 | batch->size--; |
| | 500 | } |
| | 501 | } |
| | 502 | |
| | 503 | static void vfio_batch_fini(struct vfio_batch *batch) |
| | 504 | { |
| | 505 | if (batch->capacity == VFIO_BATCH_MAX_CAPACITY) |
| | 506 | free_page((unsigned long)batch->pages); |
| | 507 | } |
| | 508 | |
| 418 | static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, | 509 | static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, |
| 419 | unsigned long vaddr, unsigned long *pfn, | 510 | unsigned long vaddr, unsigned long *pfn, |
| 420 | bool write_fault) | 511 | bool write_fault) |
| 421 | { | 512 | { |
| | 513 | pte_t *ptep; |
| | 514 | spinlock_t *ptl; |
| 422 | int ret; | 515 | int ret; |
| 423 | | 516 | |
| 424 | ret = follow_pfn(vma, vaddr, pfn); | 517 | ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); |
| 425 | if (ret) { | 518 | if (ret) { |
| 426 | bool unlocked = false; | 519 | bool unlocked = false; |
| 427 | | 520 | |
| @@ -435,16 +528,28 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, · 7 unmodified lines |
| 435 | if (ret) | 528 | if (ret) |
| 436 | return ret; | 529 | return ret; |
| 437 | | 530 | |
| 438 | ret = follow_pfn(vma, vaddr, pfn); | 531 | ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); |
| | 532 | if (ret) |
| | 533 | return ret; |
| 439 | } | 534 | } |
| 440 | | 535 | |
| | 536 | if (write_fault && !pte_write(*ptep)) |
| | 537 | ret = -EFAULT; |
| | 538 | else |
| | 539 | *pfn = pte_pfn(*ptep); |
| | 540 | |
| | 541 | pte_unmap_unlock(ptep, ptl); |
| 441 | return ret; | 542 | return ret; |
| 442 | } | 543 | } |
| 443 | | 544 | |
| 444 | static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, | 545 | /* |
| 445 | int prot, unsigned long *pfn) | 546 | * Returns the positive number of pfns successfully obtained or a negative |
| | 547 | * error code. |
| | 548 | */ |
| | 549 | static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr, |
| | 550 | long npages, int prot, unsigned long *pfn, |
| | 551 | struct page **pages) |
| 446 | { | 552 | { |
| 447 | struct page *page[1]; | | |
| 448 | struct vm_area_struct *vma; | 553 | struct vm_area_struct *vma; |
| 449 | unsigned int flags = 0; | 554 | unsigned int flags = 0; |
| 450 | int ret; | 555 | int ret; |
| @@ -453,11 +558,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, · 2 unmodified lines |
| 453 | flags |= FOLL_WRITE; | 558 | flags |= FOLL_WRITE; |
| 454 | | 559 | |
| 455 | mmap_read_lock(mm); | 560 | mmap_read_lock(mm); |
| 456 | ret = pin_user_pages_remote(mm, vaddr, 1, flags | FOLL_LONGTERM, | 561 | ret = pin_user_pages_remote(mm, vaddr, npages, flags | FOLL_LONGTERM, |
| 457 | page, NULL, NULL); | 562 | pages, NULL, NULL); |
| 458 | if (ret == 1) { | 563 | if (ret > 0) { |
| 459 | *pfn = page_to_pfn(page[0]); | 564 | *pfn = page_to_pfn(pages[0]); |
| 460 | ret = 0; | | |
| 461 | goto done; | 565 | goto done; |
| 462 | } | 566 | } |
| 463 | | 567 | |
| @@ -471,14 +575,73 @@ retry: · 7 unmodified lines |
| 471 | if (ret == -EAGAIN) | 575 | if (ret == -EAGAIN) |
| 472 | goto retry; | 576 | goto retry; |
| 473 | | 577 | |
| 474 | if (!ret && !is_invalid_reserved_pfn(*pfn)) | 578 | if (!ret) { |
| 475 | ret = -EFAULT; | 579 | if (is_invalid_reserved_pfn(*pfn)) |
| | 580 | ret = 1; |
| | 581 | else |
| | 582 | ret = -EFAULT; |
| | 583 | } |
| 476 | } | 584 | } |
| 477 | done: | 585 | done: |
| 478 | mmap_read_unlock(mm); | 586 | mmap_read_unlock(mm); |
| 479 | return ret; | 587 | return ret; |
| 480 | } | 588 | } |
| 481 | | 589 | |
| | 590 | static int vfio_wait(struct vfio_iommu *iommu) |
| | 591 | { |
| | 592 | DEFINE_WAIT(wait); |
| | 593 | |
| | 594 | prepare_to_wait(&iommu->vaddr_wait, &wait, TASK_KILLABLE); |
| | 595 | mutex_unlock(&iommu->lock); |
| | 596 | schedule(); |
| | 597 | mutex_lock(&iommu->lock); |
| | 598 | finish_wait(&iommu->vaddr_wait, &wait); |
| | 599 | if (kthread_should_stop() || !iommu->container_open || |
| | 600 | fatal_signal_pending(current)) { |
| | 601 | return -EFAULT; |
| | 602 | } |
| | 603 | return WAITED; |
| | 604 | } |
| | 605 | |
| | 606 | /* |
| | 607 | * Find dma struct and wait for its vaddr to be valid. iommu lock is dropped |
| | 608 | * if the task waits, but is re-locked on return. Return result in *dma_p. |
| | 609 | * Return 0 on success with no waiting, WAITED on success if waited, and -errno |
| | 610 | * on error. |
| | 611 | */ |
| | 612 | static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start, |
| | 613 | size_t size, struct vfio_dma **dma_p) |
| | 614 | { |
| | 615 | int ret; |
| | 616 | |
| | 617 | do { |
| | 618 | *dma_p = vfio_find_dma(iommu, start, size); |
| | 619 | if (!*dma_p) |
| | 620 | ret = -EINVAL; |
| | 621 | else if (!(*dma_p)->vaddr_invalid) |
| | 622 | ret = 0; |
| | 623 | else |
| | 624 | ret = vfio_wait(iommu); |
| | 625 | } while (ret > 0); |
| | 626 | |
| | 627 | return ret; |
| | 628 | } |
| | 629 | |
| | 630 | /* |
| | 631 | * Wait for all vaddr in the dma_list to become valid. iommu lock is dropped |
| | 632 | * if the task waits, but is re-locked on return. Return 0 on success with no |
| | 633 | * waiting, WAITED on success if waited, and -errno on error. |
| | 634 | */ |
| | 635 | static int vfio_wait_all_valid(struct vfio_iommu *iommu) |
| | 636 | { |
| | 637 | int ret = 0; |
| | 638 | |
| | 639 | while (iommu->vaddr_invalid_count && ret >= 0) |
| | 640 | ret = vfio_wait(iommu); |
| | 641 | |
| | 642 | return ret; |
| | 643 | } |
| | 644 | |
| 482 | /* | 645 | /* |
| 483 | * Attempt to pin pages. We really don't want to track all the pfns and | 646 | * Attempt to pin pages. We really don't want to track all the pfns and |
| 484 | * the iommu can only map chunks of consecutive pfns anyway, so get the | 647 | * the iommu can only map chunks of consecutive pfns anyway, so get the |
| @@ -486,76 +649,102 @@ done: · 1 unmodified line |
| 486 | */ | 649 | */ |
| 487 | static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, | 650 | static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, |
| 488 | long npage, unsigned long *pfn_base, | 651 | long npage, unsigned long *pfn_base, |
| 489 | unsigned long limit) | 652 | unsigned long limit, struct vfio_batch *batch) |
| 490 | { | 653 | { |
| 491 | unsigned long pfn = 0; | 654 | unsigned long pfn; |
| | 655 | struct mm_struct *mm = current->mm; |
| 492 | long ret, pinned = 0, lock_acct = 0; | 656 | long ret, pinned = 0, lock_acct = 0; |
| 493 | bool rsvd; | 657 | bool rsvd; |
| 494 | dma_addr_t iova = vaddr - dma->vaddr + dma->iova; | 658 | dma_addr_t iova = vaddr - dma->vaddr + dma->iova; |
| 495 | | 659 | |
| 496 | /* This code path is only user initiated */ | 660 | /* This code path is only user initiated */ |
| 497 | if (!current->mm) | 661 | if (!mm) |
| 498 | return -ENODEV; | 662 | return -ENODEV; |
| 499 | | 663 | |
| 500 | ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, pfn_base); | 664 | if (batch->size) { |
| 501 | if (ret) | 665 | /* Leftover pages in batch from an earlier call. */ |
| 502 | return ret; | 666 | *pfn_base = page_to_pfn(batch->pages[batch->offset]); |
| 503 | | 667 | pfn = *pfn_base; |
| 504 | pinned++; | 668 | rsvd = is_invalid_reserved_pfn(*pfn_base); |
| 505 | rsvd = is_invalid_reserved_pfn(*pfn_base); | 669 | } else { |
| 506 | | 670 | *pfn_base = 0; |
| 507 | /* | | |
| 508 | * Reserved pages aren't counted against the user, externally pinned | | |
| 509 | * pages are already counted against the user. | | |
| 510 | */ | | |
| 511 | if (!rsvd && !vfio_find_vpfn(dma, iova)) { | | |
| 512 | if (!dma->lock_cap && current->mm->locked_vm + 1 > limit) { | | |
| 513 | put_pfn(*pfn_base, dma->prot); | | |
| 514 | pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, | | |
| 515 | limit << PAGE_SHIFT); | | |
| 516 | return -ENOMEM; | | |
| 517 | } | | |
| 518 | lock_acct++; | | |
| 519 | } | 671 | } |
| 520 | | 672 | |
| 521 | if (unlikely(disable_hugepages)) | 673 | while (npage) { |
| 522 | goto out; | 674 | if (!batch->size) { |
| | 675 | /* Empty batch, so refill it. */ |
| | 676 | long req_pages = min_t(long, npage, batch->capacity); |
| 523 | | 677 | |
| 524 | /* Lock all the consecutive pages from pfn_base */ | 678 | ret = vaddr_get_pfns(mm, vaddr, req_pages, dma->prot, |
| 525 | for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage; | 679 | &pfn, batch->pages); |
| 526 | pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) { | 680 | if (ret < 0) |
| 527 | ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn); | 681 | goto unpin_out; |
| 528 | if (ret) | | |
| 529 | break; | | |
| 530 | | 682 | |
| 531 | if (pfn != *pfn_base + pinned || | 683 | batch->size = ret; |
| 532 | rsvd != is_invalid_reserved_pfn(pfn)) { | 684 | batch->offset = 0; |
| 533 | put_pfn(pfn, dma->prot); | 685 | |
| 534 | break; | 686 | if (!*pfn_base) { |
| | 687 | *pfn_base = pfn; |
| | 688 | rsvd = is_invalid_reserved_pfn(*pfn_base); |
| | 689 | } |
| 535 | } | 690 | } |
| 536 | | 691 | |
| 537 | if (!rsvd && !vfio_find_vpfn(dma, iova)) { | 692 | /* |
| 538 | if (!dma->lock_cap && | 693 | * pfn is preset for the first iteration of this inner loop and |
| 539 | current->mm->locked_vm + lock_acct + 1 > limit) { | 694 | * updated at the end to handle a VM_PFNMAP pfn. In that case, |
| 540 | put_pfn(pfn, dma->prot); | 695 | * batch->pages isn't valid (there's no struct page), so allow |
| 541 | pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", | 696 | * batch->pages to be touched only when there's more than one |
| 542 | __func__, limit << PAGE_SHIFT); | 697 | * pfn to check, which guarantees the pfns are from a |
| 543 | ret = -ENOMEM; | 698 | * !VM_PFNMAP vma. |
| 544 | goto unpin_out; | 699 | */ |
| | 700 | while (true) { |
| | 701 | if (pfn != *pfn_base + pinned || |
| | 702 | rsvd != is_invalid_reserved_pfn(pfn)) |
| | 703 | goto out; |
| | 704 | |
| | 705 | /* |
| | 706 | * Reserved pages aren't counted against the user, |
| | 707 | * externally pinned pages are already counted against |
| | 708 | * the user. |
| | 709 | */ |
| | 710 | if (!rsvd && !vfio_find_vpfn(dma, iova)) { |
| | 711 | if (!dma->lock_cap && |
| | 712 | mm->locked_vm + lock_acct + 1 > limit) { |
| | 713 | pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", |
| | 714 | __func__, limit << PAGE_SHIFT); |
| | 715 | ret = -ENOMEM; |
| | 716 | goto unpin_out; |
| | 717 | } |
| | 718 | lock_acct++; |
| 545 | } | 719 | } |
| 546 | lock_acct++; | 720 | |
| | 721 | pinned++; |
| | 722 | npage--; |
| | 723 | vaddr += PAGE_SIZE; |
| | 724 | iova += PAGE_SIZE; |
| | 725 | batch->offset++; |
| | 726 | batch->size--; |
| | 727 | |
| | 728 | if (!batch->size) |
| | 729 | break; |
| | 730 | |
| | 731 | pfn = page_to_pfn(batch->pages[batch->offset]); |
| 547 | } | 732 | } |
| | 733 | |
| | 734 | if (unlikely(disable_hugepages)) |
| | 735 | break; |
| 548 | } | 736 | } |
| 549 | | 737 | |
| 550 | out: | 738 | out: |
| 551 | ret = vfio_lock_acct(dma, lock_acct, false); | 739 | ret = vfio_lock_acct(dma, lock_acct, false); |
| 552 | | 740 | |
| 553 | unpin_out: | 741 | unpin_out: |
| 554 | if (ret) { | 742 | if (ret < 0) { |
| 555 | if (!rsvd) { | 743 | if (pinned && !rsvd) { |
| 556 | for (pfn = *pfn_base ; pinned ; pfn++, pinned--) | 744 | for (pfn = *pfn_base ; pinned ; pfn++, pinned--) |
| 557 | put_pfn(pfn, dma->prot); | 745 | put_pfn(pfn, dma->prot); |
| 558 | } | 746 | } |
| | 747 | vfio_batch_unpin(batch, dma); |
| 559 | | 748 | |
| 560 | return ret; | 749 | return ret; |
| 561 | } | 750 | } |
| @@ -587,6 +776,7 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, · 25 unmodified lines |
| 587 | static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, | 776 | static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, |
| 588 | unsigned long *pfn_base, bool do_accounting) | 777 | unsigned long *pfn_base, bool do_accounting) |
| 589 | { | 778 | { |
| | 779 | struct page *pages[1]; |
| 590 | struct mm_struct *mm; | 780 | struct mm_struct *mm; |
| 591 | int ret; | 781 | int ret; |
| 592 | | 782 | |
| @@ -594,8 +784,8 @@ static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, · 1 unmodified line |
| 594 | if (!mm) | 784 | if (!mm) |
| 595 | return -ENODEV; | 785 | return -ENODEV; |
| 596 | | 786 | |
| 597 | ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base); | 787 | ret = vaddr_get_pfns(mm, vaddr, 1, dma->prot, pfn_base, pages); |
| 598 | if (!ret && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { | 788 | if (ret == 1 && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { |
| 599 | ret = vfio_lock_acct(dma, 1, true); | 789 | ret = vfio_lock_acct(dma, 1, true); |
| 600 | if (ret) { | 790 | if (ret) { |
| 601 | put_pfn(*pfn_base, dma->prot); | 791 | put_pfn(*pfn_base, dma->prot); |
| @@ -640,6 +830,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 38 unmodified lines |
| 640 | unsigned long remote_vaddr; | 830 | unsigned long remote_vaddr; |
| 641 | struct vfio_dma *dma; | 831 | struct vfio_dma *dma; |
| 642 | bool do_accounting; | 832 | bool do_accounting; |
| | 833 | dma_addr_t iova; |
| 643 | | 834 | |
| 644 | if (!iommu || !user_pfn || !phys_pfn) | 835 | if (!iommu || !user_pfn || !phys_pfn) |
| 645 | return -EINVAL; | 836 | return -EINVAL; |
| @@ -650,6 +841,22 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 4 unmodified lines |
| 650 | | 841 | |
| 651 | mutex_lock(&iommu->lock); | 842 | mutex_lock(&iommu->lock); |
| 652 | | 843 | |
| | 844 | /* |
| | 845 | * Wait for all necessary vaddr's to be valid so they can be used in |
| | 846 | * the main loop without dropping the lock, to avoid racing vs unmap. |
| | 847 | */ |
| | 848 | again: |
| | 849 | if (iommu->vaddr_invalid_count) { |
| | 850 | for (i = 0; i < npage; i++) { |
| | 851 | iova = user_pfn[i] << PAGE_SHIFT; |
| | 852 | ret = vfio_find_dma_valid(iommu, iova, PAGE_SIZE, &dma); |
| | 853 | if (ret < 0) |
| | 854 | goto pin_done; |
| | 855 | if (ret == WAITED) |
| | 856 | goto again; |
| | 857 | } |
| | 858 | } |
| | 859 | |
| 653 | /* Fail if notifier list is empty */ | 860 | /* Fail if notifier list is empty */ |
| 654 | if (!iommu->notifier.head) { | 861 | if (!iommu->notifier.head) { |
| 655 | ret = -EINVAL; | 862 | ret = -EINVAL; |
| @@ -664,7 +871,6 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 8 unmodified lines |
| 664 | do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu); | 871 | do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu); |
| 665 | | 872 | |
| 666 | for (i = 0; i < npage; i++) { | 873 | for (i = 0; i < npage; i++) { |
| 667 | dma_addr_t iova; | | |
| 668 | struct vfio_pfn *vpfn; | 874 | struct vfio_pfn *vpfn; |
| 669 | | 875 | |
| 670 | iova = user_pfn[i] << PAGE_SHIFT; | 876 | iova = user_pfn[i] << PAGE_SHIFT; |
| @@ -714,7 +920,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, · 43 unmodified lines |
| 714 | group = vfio_iommu_find_iommu_group(iommu, iommu_group); | 920 | group = vfio_iommu_find_iommu_group(iommu, iommu_group); |
| 715 | if (!group->pinned_page_dirty_scope) { | 921 | if (!group->pinned_page_dirty_scope) { |
| 716 | group->pinned_page_dirty_scope = true; | 922 | group->pinned_page_dirty_scope = true; |
| 717 | update_pinned_page_dirty_scope(iommu); | 923 | iommu->num_non_pinned_groups--; |
| 718 | } | 924 | } |
| 719 | | 925 | |
| 720 | goto pin_done; | 926 | goto pin_done; |
| @@ -945,10 +1151,15 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, · 224 unmodified lines |
| 945 | | 1151 | |
| 946 | static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) | 1152 | static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) |
| 947 | { | 1153 | { |
| | 1154 | WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); |
| 948 | vfio_unmap_unpin(iommu, dma, true); | 1155 | vfio_unmap_unpin(iommu, dma, true); |
| 949 | vfio_unlink_dma(iommu, dma); | 1156 | vfio_unlink_dma(iommu, dma); |
| 950 | put_task_struct(dma->task); | 1157 | put_task_struct(dma->task); |
| 951 | vfio_dma_bitmap_free(dma); | 1158 | vfio_dma_bitmap_free(dma); |
| | 1159 | if (dma->vaddr_invalid) { |
| | 1160 | iommu->vaddr_invalid_count--; |
| | 1161 | wake_up_all(&iommu->vaddr_wait); |
| | 1162 | } |
| 952 | kfree(dma); | 1163 | kfree(dma); |
| 953 | iommu->dma_avail++; | 1164 | iommu->dma_avail++; |
| 954 | } | 1165 | } |
| @@ -991,7 +1202,7 @@ static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, · 36 unmodified lines |
| 991 | * mark all pages dirty if any IOMMU capable device is not able | 1202 | * mark all pages dirty if any IOMMU capable device is not able |
| 992 | * to report dirty pages and all pages are pinned and mapped. | 1203 | * to report dirty pages and all pages are pinned and mapped. |
| 993 | */ | 1204 | */ |
| 994 | if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped) | 1205 | if (iommu->num_non_pinned_groups && dma->iommu_mapped) |
| 995 | bitmap_set(dma->bitmap, 0, nbits); | 1206 | bitmap_set(dma->bitmap, 0, nbits); |
| 996 | | 1207 | |
| 997 | if (shift) { | 1208 | if (shift) { |
| @@ -1074,34 +1285,36 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, · 76 unmodified lines |
| 1074 | { | 1285 | { |
| 1075 | struct vfio_dma *dma, *dma_last = NULL; | 1286 | struct vfio_dma *dma, *dma_last = NULL; |
| 1076 | size_t unmapped = 0, pgsize; | 1287 | size_t unmapped = 0, pgsize; |
| 1077 | int ret = 0, retries = 0; | 1288 | int ret = -EINVAL, retries = 0; |
| 1078 | unsigned long pgshift; | 1289 | unsigned long pgshift; |
| | 1290 | dma_addr_t iova = unmap->iova; |
| | 1291 | unsigned long size = unmap->size; |
| | 1292 | bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL; |
| | 1293 | bool invalidate_vaddr = unmap->flags & VFIO_DMA_UNMAP_FLAG_VADDR; |
| | 1294 | struct rb_node *n, *first_n; |
| 1079 | | 1295 | |
| 1080 | mutex_lock(&iommu->lock); | 1296 | mutex_lock(&iommu->lock); |
| 1081 | | 1297 | |
| 1082 | pgshift = __ffs(iommu->pgsize_bitmap); | 1298 | pgshift = __ffs(iommu->pgsize_bitmap); |
| 1083 | pgsize = (size_t)1 << pgshift; | 1299 | pgsize = (size_t)1 << pgshift; |
| 1084 | | 1300 | |
| 1085 | if (unmap->iova & (pgsize - 1)) { | 1301 | if (iova & (pgsize - 1)) |
| 1086 | ret = -EINVAL; | | |
| 1087 | goto unlock; | 1302 | goto unlock; |
| 1088 | } | | |
| 1089 | | 1303 | |
| 1090 | if (!unmap->size || unmap->size & (pgsize - 1)) { | 1304 | if (unmap_all) { |
| 1091 | ret = -EINVAL; | 1305 | if (iova || size) |
| | 1306 | goto unlock; |
| | 1307 | size = SIZE_MAX; |
| | 1308 | } else if (!size || size & (pgsize - 1)) { |
| 1092 | goto unlock; | 1309 | goto unlock; |
| 1093 | } | 1310 | } |
| 1094 | | 1311 | |
| 1095 | if (unmap->iova + unmap->size - 1 < unmap->iova || | 1312 | if (iova + size - 1 < iova || size > SIZE_MAX) |
| 1096 | unmap->size > SIZE_MAX) { | | |
| 1097 | ret = -EINVAL; | | |
| 1098 | goto unlock; | 1313 | goto unlock; |
| 1099 | } | | |
| 1100 | | 1314 | |
| 1101 | /* When dirty tracking is enabled, allow only min supported pgsize */ | 1315 | /* When dirty tracking is enabled, allow only min supported pgsize */ |
| 1102 | if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && | 1316 | if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && |
| 1103 | (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { | 1317 | (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { |
| 1104 | ret = -EINVAL; | | |
| 1105 | goto unlock; | 1318 | goto unlock; |
| 1106 | } | 1319 | } |
| 1107 | | 1320 | |
| @@ -1138,21 +1351,25 @@ again: · 30 unmodified lines |
| 1138 | * will only return success and a size of zero if there were no | 1351 | * will only return success and a size of zero if there were no |
| 1139 | * mappings within the range. | 1352 | * mappings within the range. |
| 1140 | */ | 1353 | */ |
| 1141 | if (iommu->v2) { | 1354 | if (iommu->v2 && !unmap_all) { |
| 1142 | dma = vfio_find_dma(iommu, unmap->iova, 1); | 1355 | dma = vfio_find_dma(iommu, iova, 1); |
| 1143 | if (dma && dma->iova != unmap->iova) { | 1356 | if (dma && dma->iova != iova) |
| 1144 | ret = -EINVAL; | | |
| 1145 | goto unlock; | 1357 | goto unlock; |
| 1146 | } | 1358 | |
| 1147 | dma = vfio_find_dma(iommu, unmap->iova + unmap->size - 1, 0); | 1359 | dma = vfio_find_dma(iommu, iova + size - 1, 0); |
| 1148 | if (dma && dma->iova + dma->size != unmap->iova + unmap->size) { | 1360 | if (dma && dma->iova + dma->size != iova + size) |
| 1149 | ret = -EINVAL; | | |
| 1150 | goto unlock; | 1361 | goto unlock; |
| 1151 | } | | |
| 1152 | } | 1362 | } |
| 1153 | | 1363 | |
| 1154 | while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) { | 1364 | ret = 0; |
| 1155 | if (!iommu->v2 && unmap->iova > dma->iova) | 1365 | n = first_n = vfio_find_dma_first_node(iommu, iova, size); |
| | 1366 | |
| | 1367 | while (n) { |
| | 1368 | dma = rb_entry(n, struct vfio_dma, node); |
| | 1369 | if (dma->iova >= iova + size) |
| | 1370 | break; |
| | 1371 | |
| | 1372 | if (!iommu->v2 && iova > dma->iova) |
| 1156 | break; | 1373 | break; |
| 1157 | /* | 1374 | /* |
| 1158 | * Task with same address space who mapped this iova range is | 1375 | * Task with same address space who mapped this iova range is |
| @@ -1161,6 +1378,27 @@ again: · 2 unmodified lines |
| 1161 | if (dma->task->mm != current->mm) | 1378 | if (dma->task->mm != current->mm) |
| 1162 | break; | 1379 | break; |
| 1163 | | 1380 | |
| | 1381 | if (invalidate_vaddr) { |
| | 1382 | if (dma->vaddr_invalid) { |
| | 1383 | struct rb_node *last_n = n; |
| | 1384 | |
| | 1385 | for (n = first_n; n != last_n; n = rb_next(n)) { |
| | 1386 | dma = rb_entry(n, |
| | 1387 | struct vfio_dma, node); |
| | 1388 | dma->vaddr_invalid = false; |
| | 1389 | iommu->vaddr_invalid_count--; |
| | 1390 | } |
| | 1391 | ret = -EINVAL; |
| | 1392 | unmapped = 0; |
| | 1393 | break; |
| | 1394 | } |
| | 1395 | dma->vaddr_invalid = true; |
| | 1396 | iommu->vaddr_invalid_count++; |
| | 1397 | unmapped += dma->size; |
| | 1398 | n = rb_next(n); |
| | 1399 | continue; |
| | 1400 | } |
| | 1401 | |
| 1164 | if (!RB_EMPTY_ROOT(&dma->pfn_list)) { | 1402 | if (!RB_EMPTY_ROOT(&dma->pfn_list)) { |
| 1165 | struct vfio_iommu_type1_dma_unmap nb_unmap; | 1403 | struct vfio_iommu_type1_dma_unmap nb_unmap; |
| 1166 | | 1404 | |
| @@ -1190,12 +1428,13 @@ again: · 23 unmodified lines |
| 1190 | | 1428 | |
| 1191 | if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { | 1429 | if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { |
| 1192 | ret = update_user_bitmap(bitmap->data, iommu, dma, | 1430 | ret = update_user_bitmap(bitmap->data, iommu, dma, |
| 1193 | unmap->iova, pgsize); | 1431 | iova, pgsize); |
| 1194 | if (ret) | 1432 | if (ret) |
| 1195 | break; | 1433 | break; |
| 1196 | } | 1434 | } |
| 1197 | | 1435 | |
| 1198 | unmapped += dma->size; | 1436 | unmapped += dma->size; |
| | 1437 | n = rb_next(n); |
| 1199 | vfio_remove_dma(iommu, dma); | 1438 | vfio_remove_dma(iommu, dma); |
| 1200 | } | 1439 | } |
| 1201 | | 1440 | |
| @@ -1239,15 +1478,19 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, · 37 unmodified lines |
| 1239 | { | 1478 | { |
| 1240 | dma_addr_t iova = dma->iova; | 1479 | dma_addr_t iova = dma->iova; |
| 1241 | unsigned long vaddr = dma->vaddr; | 1480 | unsigned long vaddr = dma->vaddr; |
| | 1481 | struct vfio_batch batch; |
| 1242 | size_t size = map_size; | 1482 | size_t size = map_size; |
| 1243 | long npage; | 1483 | long npage; |
| 1244 | unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; | 1484 | unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; |
| 1245 | int ret = 0; | 1485 | int ret = 0; |
| 1246 | | 1486 | |
| | 1487 | vfio_batch_init(&batch); |
| | 1488 | |
| 1247 | while (size) { | 1489 | while (size) { |
| 1248 | /* Pin a contiguous chunk of memory */ | 1490 | /* Pin a contiguous chunk of memory */ |
| 1249 | npage = vfio_pin_pages_remote(dma, vaddr + dma->size, | 1491 | npage = vfio_pin_pages_remote(dma, vaddr + dma->size, |
| 1250 | size >> PAGE_SHIFT, &pfn, limit); | 1492 | size >> PAGE_SHIFT, &pfn, limit, |
| | 1493 | &batch); |
| 1251 | if (npage <= 0) { | 1494 | if (npage <= 0) { |
| 1252 | WARN_ON(!npage); | 1495 | WARN_ON(!npage); |
| 1253 | ret = (int)npage; | 1496 | ret = (int)npage; |
| @@ -1260,6 +1503,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, · 6 unmodified lines |
| 1260 | if (ret) { | 1503 | if (ret) { |
| 1261 | vfio_unpin_pages_remote(dma, iova + dma->size, pfn, | 1504 | vfio_unpin_pages_remote(dma, iova + dma->size, pfn, |
| 1262 | npage, true); | 1505 | npage, true); |
| | 1506 | vfio_batch_unpin(&batch, dma); |
| 1263 | break; | 1507 | break; |
| 1264 | } | 1508 | } |
| 1265 | | 1509 | |
| @@ -1267,6 +1511,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, · 1 unmodified line |
| 1267 | dma->size += npage << PAGE_SHIFT; | 1511 | dma->size += npage << PAGE_SHIFT; |
| 1268 | } | 1512 | } |
| 1269 | | 1513 | |
| | 1514 | vfio_batch_fini(&batch); |
| 1270 | dma->iommu_mapped = true; | 1515 | dma->iommu_mapped = true; |
| 1271 | | 1516 | |
| 1272 | if (ret) | 1517 | if (ret) |
| @@ -1299,6 +1544,7 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, · 26 unmodified lines |
| 1299 | static int vfio_dma_do_map(struct vfio_iommu *iommu, | 1544 | static int vfio_dma_do_map(struct vfio_iommu *iommu, |
| 1300 | struct vfio_iommu_type1_dma_map *map) | 1545 | struct vfio_iommu_type1_dma_map *map) |
| 1301 | { | 1546 | { |
| | 1547 | bool set_vaddr = map->flags & VFIO_DMA_MAP_FLAG_VADDR; |
| 1302 | dma_addr_t iova = map->iova; | 1548 | dma_addr_t iova = map->iova; |
| 1303 | unsigned long vaddr = map->vaddr; | 1549 | unsigned long vaddr = map->vaddr; |
| 1304 | size_t size = map->size; | 1550 | size_t size = map->size; |
| @@ -1316,13 +1562,16 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, · 11 unmodified lines |
| 1316 | if (map->flags & VFIO_DMA_MAP_FLAG_READ) | 1562 | if (map->flags & VFIO_DMA_MAP_FLAG_READ) |
| 1317 | prot |= IOMMU_READ; | 1563 | prot |= IOMMU_READ; |
| 1318 | | 1564 | |
| | 1565 | if ((prot && set_vaddr) || (!prot && !set_vaddr)) |
| | 1566 | return -EINVAL; |
| | 1567 | |
| 1319 | mutex_lock(&iommu->lock); | 1568 | mutex_lock(&iommu->lock); |
| 1320 | | 1569 | |
| 1321 | pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); | 1570 | pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); |
| 1322 | | 1571 | |
| 1323 | WARN_ON((pgsize - 1) & PAGE_MASK); | 1572 | WARN_ON((pgsize - 1) & PAGE_MASK); |
| 1324 | | 1573 | |
| 1325 | if (!prot || !size || (size | iova | vaddr) & (pgsize - 1)) { | 1574 | if (!size || (size | iova | vaddr) & (pgsize - 1)) { |
| 1326 | ret = -EINVAL; | 1575 | ret = -EINVAL; |
| 1327 | goto out_unlock; | 1576 | goto out_unlock; |
| 1328 | } | 1577 | } |
| @@ -1333,7 +1582,21 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, · 4 unmodified lines |
| 1333 | goto out_unlock; | 1582 | goto out_unlock; |
| 1334 | } | 1583 | } |
| 1335 | | 1584 | |
| 1336 | if (vfio_find_dma(iommu, iova, size)) { | 1585 | dma = vfio_find_dma(iommu, iova, size); |
| | 1586 | if (set_vaddr) { |
| | 1587 | if (!dma) { |
| | 1588 | ret = -ENOENT; |
| | 1589 | } else if (!dma->vaddr_invalid || dma->iova != iova || |
| | 1590 | dma->size != size) { |
| | 1591 | ret = -EINVAL; |
| | 1592 | } else { |
| | 1593 | dma->vaddr = vaddr; |
| | 1594 | dma->vaddr_invalid = false; |
| | 1595 | iommu->vaddr_invalid_count--; |
| | 1596 | wake_up_all(&iommu->vaddr_wait); |
| | 1597 | } |
| | 1598 | goto out_unlock; |
| | 1599 | } else if (dma) { |
| 1337 | ret = -EEXIST; | 1600 | ret = -EEXIST; |
| 1338 | goto out_unlock; | 1601 | goto out_unlock; |
| 1339 | } | 1602 | } |
| @@ -1425,16 +1688,23 @@ static int vfio_bus_type(struct device *dev, void *data) · 85 unmodified lines |
| 1425 | static int vfio_iommu_replay(struct vfio_iommu *iommu, | 1688 | static int vfio_iommu_replay(struct vfio_iommu *iommu, |
| 1426 | struct vfio_domain *domain) | 1689 | struct vfio_domain *domain) |
| 1427 | { | 1690 | { |
| | 1691 | struct vfio_batch batch; |
| 1428 | struct vfio_domain *d = NULL; | 1692 | struct vfio_domain *d = NULL; |
| 1429 | struct rb_node *n; | 1693 | struct rb_node *n; |
| 1430 | unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; | 1694 | unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; |
| 1431 | int ret; | 1695 | int ret; |
| 1432 | | 1696 | |
| | 1697 | ret = vfio_wait_all_valid(iommu); |
| | 1698 | if (ret < 0) |
| | 1699 | return ret; |
| | 1700 | |
| 1433 | /* Arbitrarily pick the first domain in the list for lookups */ | 1701 | /* Arbitrarily pick the first domain in the list for lookups */ |
| 1434 | if (!list_empty(&iommu->domain_list)) | 1702 | if (!list_empty(&iommu->domain_list)) |
| 1435 | d = list_first_entry(&iommu->domain_list, | 1703 | d = list_first_entry(&iommu->domain_list, |
| 1436 | struct vfio_domain, next); | 1704 | struct vfio_domain, next); |
| 1437 | | 1705 | |
| | 1706 | vfio_batch_init(&batch); |
| | 1707 | |
| 1438 | n = rb_first(&iommu->dma_list); | 1708 | n = rb_first(&iommu->dma_list); |
| 1439 | | 1709 | |
| 1440 | for (; n; n = rb_next(n)) { | 1710 | for (; n; n = rb_next(n)) { |
| @@ -1482,7 +1752,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, · 41 unmodified lines |
| 1482 | | 1752 | |
| 1483 | npage = vfio_pin_pages_remote(dma, vaddr, | 1753 | npage = vfio_pin_pages_remote(dma, vaddr, |
| 1484 | n >> PAGE_SHIFT, | 1754 | n >> PAGE_SHIFT, |
| 1485 | &pfn, limit); | 1755 | &pfn, limit, |
| | 1756 | &batch); |
| 1486 | if (npage <= 0) { | 1757 | if (npage <= 0) { |
| 1487 | WARN_ON(!npage); | 1758 | WARN_ON(!npage); |
| 1488 | ret = (int)npage; | 1759 | ret = (int)npage; |
| @@ -1496,11 +1767,13 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, · 7 unmodified lines |
| 1496 | ret = iommu_map(domain->domain, iova, phys, | 1767 | ret = iommu_map(domain->domain, iova, phys, |
| 1497 | size, dma->prot | domain->prot); | 1768 | size, dma->prot | domain->prot); |
| 1498 | if (ret) { | 1769 | if (ret) { |
| 1499 | if (!dma->iommu_mapped) | 1770 | if (!dma->iommu_mapped) { |
| 1500 | vfio_unpin_pages_remote(dma, iova, | 1771 | vfio_unpin_pages_remote(dma, iova, |
| 1501 | phys >> PAGE_SHIFT, | 1772 | phys >> PAGE_SHIFT, |
| 1502 | size >> PAGE_SHIFT, | 1773 | size >> PAGE_SHIFT, |
| 1503 | true); | 1774 | true); |
| | 1775 | vfio_batch_unpin(&batch, dma); |
| | 1776 | } |
| 1504 | goto unwind; | 1777 | goto unwind; |
| 1505 | } | 1778 | } |
| 1506 | | 1779 | |
| @@ -1515,6 +1788,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, · 8 unmodified lines |
| 1515 | dma->iommu_mapped = true; | 1788 | dma->iommu_mapped = true; |
| 1516 | } | 1789 | } |
| 1517 | | 1790 | |
| | 1791 | vfio_batch_fini(&batch); |
| 1518 | return 0; | 1792 | return 0; |
| 1519 | | 1793 | |
| 1520 | unwind: | 1794 | unwind: |
| @@ -1555,6 +1829,7 @@ unwind: · 34 unmodified lines |
| 1555 | } | 1829 | } |
| 1556 | } | 1830 | } |
| 1557 | | 1831 | |
| | 1832 | vfio_batch_fini(&batch); |
| 1558 | return ret; | 1833 | return ret; |
| 1559 | } | 1834 | } |
| 1560 | | 1835 | |
| @@ -1622,33 +1897,6 @@ static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, · 61 unmodified lines |
| 1622 | return group; | 1897 | return group; |
| 1623 | } | 1898 | } |
| 1624 | | 1899 | |
| 1625 | static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu) | | |
| 1626 | { | | |
| 1627 | struct vfio_domain *domain; | | |
| 1628 | struct vfio_group *group; | | |
| 1629 | | | |
| 1630 | list_for_each_entry(domain, &iommu->domain_list, next) { | | |
| 1631 | list_for_each_entry(group, &domain->group_list, next) { | | |
| 1632 | if (!group->pinned_page_dirty_scope) { | | |
| 1633 | iommu->pinned_page_dirty_scope = false; | | |
| 1634 | return; | | |
| 1635 | } | | |
| 1636 | } | | |
| 1637 | } | | |
| 1638 | | | |
| 1639 | if (iommu->external_domain) { | | |
| 1640 | domain = iommu->external_domain; | | |
| 1641 | list_for_each_entry(group, &domain->group_list, next) { | | |
| 1642 | if (!group->pinned_page_dirty_scope) { | | |
| 1643 | iommu->pinned_page_dirty_scope = false; | | |
| 1644 | return; | | |
| 1645 | } | | |
| 1646 | } | | |
| 1647 | } | | |
| 1648 | | | |
| 1649 | iommu->pinned_page_dirty_scope = true; | | |
| 1650 | } | | |
| 1651 | | | |
| 1652 | static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, | 1900 | static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, |
| 1653 | phys_addr_t *base) | 1901 | phys_addr_t *base) |
| 1654 | { | 1902 | { |
| @@ -2057,8 +2305,6 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, · 402 unmodified lines |
| 2057 | * addition of a dirty tracking group. | 2305 | * addition of a dirty tracking group. |
| 2058 | */ | 2306 | */ |
| 2059 | group->pinned_page_dirty_scope = true; | 2307 | group->pinned_page_dirty_scope = true; |
| 2060 | if (!iommu->pinned_page_dirty_scope) | | |
| 2061 | update_pinned_page_dirty_scope(iommu); | | |
| 2062 | mutex_unlock(&iommu->lock); | 2308 | mutex_unlock(&iommu->lock); |
| 2063 | | 2309 | |
| 2064 | return 0; | 2310 | return 0; |
| @@ -2188,7 +2434,7 @@ done: · 123 unmodified lines |
| 2188 | * demotes the iommu scope until it declares itself dirty tracking | 2434 | * demotes the iommu scope until it declares itself dirty tracking |
| 2189 | * capable via the page pinning interface. | 2435 | * capable via the page pinning interface. |
| 2190 | */ | 2436 | */ |
| 2191 | iommu->pinned_page_dirty_scope = false; | 2437 | iommu->num_non_pinned_groups++; |
| 2192 | mutex_unlock(&iommu->lock); | 2438 | mutex_unlock(&iommu->lock); |
| 2193 | vfio_iommu_resv_free(&group_resv_regions); | 2439 | vfio_iommu_resv_free(&group_resv_regions); |
| 2194 | | 2440 | |
| @@ -2238,23 +2484,6 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) · 43 unmodified lines |
| 2238 | } | 2484 | } |
| 2239 | } | 2485 | } |
| 2240 | | 2486 | |
| 2241 | static void vfio_sanity_check_pfn_list(struct vfio_iommu *iommu) | | |
| 2242 | { | | |
| 2243 | struct rb_node *n; | | |
| 2244 | | | |
| 2245 | n = rb_first(&iommu->dma_list); | | |
| 2246 | for (; n; n = rb_next(n)) { | | |
| 2247 | struct vfio_dma *dma; | | |
| 2248 | | | |
| 2249 | dma = rb_entry(n, struct vfio_dma, node); | | |
| 2250 | | | |
| 2251 | if (WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list))) | | |
| 2252 | break; | | |
| 2253 | } | | |
| 2254 | /* mdev vendor driver must unregister notifier */ | | |
| 2255 | WARN_ON(iommu->notifier.head); | | |
| 2256 | } | | |
| 2257 | | | |
| 2258 | /* | 2487 | /* |
| 2259 | * Called when a domain is removed in detach. It is possible that | 2488 | * Called when a domain is removed in detach. It is possible that |
| 2260 | * the removed domain decided the iova aperture window. Modify the | 2489 | * the removed domain decided the iova aperture window. Modify the |
| @@ -2354,10 +2583,10 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 93 unmodified lines |
| 2354 | kfree(group); | 2583 | kfree(group); |
| 2355 | | 2584 | |
| 2356 | if (list_empty(&iommu->external_domain->group_list)) { | 2585 | if (list_empty(&iommu->external_domain->group_list)) { |
| 2357 | vfio_sanity_check_pfn_list(iommu); | 2586 | if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) { |
| 2358 | | 2587 | WARN_ON(iommu->notifier.head); |
| 2359 | if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) | | |
| 2360 | vfio_iommu_unmap_unpin_all(iommu); | 2588 | vfio_iommu_unmap_unpin_all(iommu); |
| | 2589 | } |
| 2361 | | 2590 | |
| 2362 | kfree(iommu->external_domain); | 2591 | kfree(iommu->external_domain); |
| 2363 | iommu->external_domain = NULL; | 2592 | iommu->external_domain = NULL; |
| @@ -2391,10 +2620,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, · 27 unmodified lines |
| 2391 | */ | 2620 | */ |
| 2392 | if (list_empty(&domain->group_list)) { | 2621 | if (list_empty(&domain->group_list)) { |
| 2393 | if (list_is_singular(&iommu->domain_list)) { | 2622 | if (list_is_singular(&iommu->domain_list)) { |
| 2394 | if (!iommu->external_domain) | 2623 | if (!iommu->external_domain) { |
| | 2624 | WARN_ON(iommu->notifier.head); |
| 2395 | vfio_iommu_unmap_unpin_all(iommu); | 2625 | vfio_iommu_unmap_unpin_all(iommu); |
| 2396 | else | 2626 | } else { |
| 2397 | vfio_iommu_unmap_unpin_reaccount(iommu); | 2627 | vfio_iommu_unmap_unpin_reaccount(iommu); |
| | 2628 | } |
| 2398 | } | 2629 | } |
| 2399 | iommu_domain_free(domain->domain); | 2630 | iommu_domain_free(domain->domain); |
| 2400 | list_del(&domain->next); | 2631 | list_del(&domain->next); |
| @@ -2415,8 +2646,11 @@ detach_group_done: · 14 unmodified lines |
| 2415 | * Removal of a group without dirty tracking may allow the iommu scope | 2646 | * Removal of a group without dirty tracking may allow the iommu scope |
| 2416 | * to be promoted. | 2647 | * to be promoted. |
| 2417 | */ | 2648 | */ |
| 2418 | if (update_dirty_scope) | 2649 | if (update_dirty_scope) { |
| 2419 | update_pinned_page_dirty_scope(iommu); | 2650 | iommu->num_non_pinned_groups--; |
| | 2651 | if (iommu->dirty_page_tracking) |
| | 2652 | vfio_iommu_populate_bitmap_full(iommu); |
| | 2653 | } |
| 2420 | mutex_unlock(&iommu->lock); | 2654 | mutex_unlock(&iommu->lock); |
| 2421 | } | 2655 | } |
| 2422 | | 2656 | |
| @@ -2446,8 +2680,10 @@ static void *vfio_iommu_type1_open(unsigned long arg) · 23 unmodified lines |
| 2446 | INIT_LIST_HEAD(&iommu->iova_list); | 2680 | INIT_LIST_HEAD(&iommu->iova_list); |
| 2447 | iommu->dma_list = RB_ROOT; | 2681 | iommu->dma_list = RB_ROOT; |
| 2448 | iommu->dma_avail = dma_entry_limit; | 2682 | iommu->dma_avail = dma_entry_limit; |
| | 2683 | iommu->container_open = true; |
| 2449 | mutex_init(&iommu->lock); | 2684 | mutex_init(&iommu->lock); |
| 2450 | BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier); | 2685 | BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier); |
| | 2686 | init_waitqueue_head(&iommu->vaddr_wait); |
| 2451 | | 2687 | |
| 2452 | return iommu; | 2688 | return iommu; |
| 2453 | } | 2689 | } |
| @@ -2475,7 +2711,6 @@ static void vfio_iommu_type1_release(void *iommu_data) · 21 unmodified lines |
| 2475 | | 2711 | |
| 2476 | if (iommu->external_domain) { | 2712 | if (iommu->external_domain) { |
| 2477 | vfio_release_domain(iommu->external_domain, true); | 2713 | vfio_release_domain(iommu->external_domain, true); |
| 2478 | vfio_sanity_check_pfn_list(iommu); | | |
| 2479 | kfree(iommu->external_domain); | 2714 | kfree(iommu->external_domain); |
| 2480 | } | 2715 | } |
| 2481 | | 2716 | |
| @@ -2517,6 +2752,8 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, · 35 unmodified lines |
| 2517 | case VFIO_TYPE1_IOMMU: | 2752 | case VFIO_TYPE1_IOMMU: |
| 2518 | case VFIO_TYPE1v2_IOMMU: | 2753 | case VFIO_TYPE1v2_IOMMU: |
| 2519 | case VFIO_TYPE1_NESTING_IOMMU: | 2754 | case VFIO_TYPE1_NESTING_IOMMU: |
| | 2755 | case VFIO_UNMAP_ALL: |
| | 2756 | case VFIO_UPDATE_VADDR: |
| 2520 | return 1; | 2757 | return 1; |
| 2521 | case VFIO_DMA_CC_IOMMU: | 2758 | case VFIO_DMA_CC_IOMMU: |
| 2522 | if (!iommu) | 2759 | if (!iommu) |
| @@ -2688,7 +2925,8 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, · 165 unmodified lines |
| 2688 | { | 2925 | { |
| 2689 | struct vfio_iommu_type1_dma_map map; | 2926 | struct vfio_iommu_type1_dma_map map; |
| 2690 | unsigned long minsz; | 2927 | unsigned long minsz; |
| 2691 | uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; | 2928 | uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE | |
| | 2929 | VFIO_DMA_MAP_FLAG_VADDR; |
| 2692 | | 2930 | |
| 2693 | minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); | 2931 | minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); |
| 2694 | | 2932 | |
| @@ -2706,6 +2944,9 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, · 11 unmodified lines |
| 2706 | { | 2944 | { |
| 2707 | struct vfio_iommu_type1_dma_unmap unmap; | 2945 | struct vfio_iommu_type1_dma_unmap unmap; |
| 2708 | struct vfio_bitmap bitmap = { 0 }; | 2946 | struct vfio_bitmap bitmap = { 0 }; |
| | 2947 | uint32_t mask = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP | |
| | 2948 | VFIO_DMA_UNMAP_FLAG_VADDR | |
| | 2949 | VFIO_DMA_UNMAP_FLAG_ALL; |
| 2709 | unsigned long minsz; | 2950 | unsigned long minsz; |
| 2710 | int ret; | 2951 | int ret; |
| 2711 | | 2952 | |
| @@ -2714,8 +2955,12 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, · 2 unmodified lines |
| 2714 | if (copy_from_user(&unmap, (void __user *)arg, minsz)) | 2955 | if (copy_from_user(&unmap, (void __user *)arg, minsz)) |
| 2715 | return -EFAULT; | 2956 | return -EFAULT; |
| 2716 | | 2957 | |
| 2717 | if (unmap.argsz < minsz || | 2958 | if (unmap.argsz < minsz || unmap.flags & ~mask) |
| 2718 | unmap.flags & ~VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) | 2959 | return -EINVAL; |
| | 2960 | |
| | 2961 | if ((unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && |
| | 2962 | (unmap.flags & (VFIO_DMA_UNMAP_FLAG_ALL | |
| | 2963 | VFIO_DMA_UNMAP_FLAG_VADDR))) |
| 2719 | return -EINVAL; | 2964 | return -EINVAL; |
| 2720 | | 2965 | |
| 2721 | if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { | 2966 | if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { |
| @@ -2906,12 +3151,13 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, · 184 unmodified lines |
| 2906 | struct vfio_dma *dma; | 3151 | struct vfio_dma *dma; |
| 2907 | bool kthread = current->mm == NULL; | 3152 | bool kthread = current->mm == NULL; |
| 2908 | size_t offset; | 3153 | size_t offset; |
| | 3154 | int ret; |
| 2909 | | 3155 | |
| 2910 | *copied = 0; | 3156 | *copied = 0; |
| 2911 | | 3157 | |
| 2912 | dma = vfio_find_dma(iommu, user_iova, 1); | 3158 | ret = vfio_find_dma_valid(iommu, user_iova, 1, &dma); |
| 2913 | if (!dma) | 3159 | if (ret < 0) |
| 2914 | return -EINVAL; | 3160 | return ret; |
| 2915 | | 3161 | |
| 2916 | if ((write && !(dma->prot & IOMMU_WRITE)) || | 3162 | if ((write && !(dma->prot & IOMMU_WRITE)) || |
| 2917 | !(dma->prot & IOMMU_READ)) | 3163 | !(dma->prot & IOMMU_READ)) |
| @@ -3003,6 +3249,19 @@ vfio_iommu_type1_group_iommu_domain(void *iommu_data, · 85 unmodified lines |
| 3003 | return domain; | 3249 | return domain; |
| 3004 | } | 3250 | } |
| 3005 | | 3251 | |
| | 3252 | static void vfio_iommu_type1_notify(void *iommu_data, |
| | 3253 | enum vfio_iommu_notify_type event) |
| | 3254 | { |
| | 3255 | struct vfio_iommu *iommu = iommu_data; |
| | 3256 | |
| | 3257 | if (event != VFIO_IOMMU_CONTAINER_CLOSE) |
| | 3258 | return; |
| | 3259 | mutex_lock(&iommu->lock); |
| | 3260 | iommu->container_open = false; |
| | 3261 | mutex_unlock(&iommu->lock); |
| | 3262 | wake_up_all(&iommu->vaddr_wait); |
| | 3263 | } |
| | 3264 | |
| 3006 | static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { | 3265 | static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { |
| 3007 | .name = "vfio-iommu-type1", | 3266 | .name = "vfio-iommu-type1", |
| 3008 | .owner = THIS_MODULE, | 3267 | .owner = THIS_MODULE, |
| @@ -3017,6 +3276,7 @@ static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { · 8 unmodified lines |
| 3017 | .unregister_notifier = vfio_iommu_type1_unregister_notifier, | 3276 | .unregister_notifier = vfio_iommu_type1_unregister_notifier, |
| 3018 | .dma_rw = vfio_iommu_type1_dma_rw, | 3277 | .dma_rw = vfio_iommu_type1_dma_rw, |
| 3019 | .group_iommu_domain = vfio_iommu_type1_group_iommu_domain, | 3278 | .group_iommu_domain = vfio_iommu_type1_group_iommu_domain, |
| | 3279 | .notify = vfio_iommu_type1_notify, |
| 3020 | }; | 3280 | }; |
| 3021 | | 3281 | |
| 3022 | static int __init vfio_iommu_type1_init(void) | 3282 | static int __init vfio_iommu_type1_init(void) |