Pinning one 4K page at a time is inefficient, so do it in batches of 512 instead. This is just an optimization with no functional change intended, and in particular the driver still calls iommu_map() with the largest physically contiguous range possible. Add two fields in vfio_batch to remember where to start between calls to vfio_pin_pages_remote(), and use vfio_batch_unpin() to handle remaining pages in the batch in case of error. qemu pins pages for guests around 8% faster on my test system, a two-node Broadwell server with 128G memory per node. The qemu process was bound to one node with its allocations constrained there as well. base test guest ---------------- ---------------- mem (GB) speedup avg sec (std) avg sec (std) 1 7.4% 0.61 (0.00) 0.56 (0.00) 2 8.3% 0.93 (0.00) 0.85 (0.00) 4 8.4% 1.46 (0.00) 1.34 (0.00) 8 8.6% 2.54 (0.01) 2.32 (0.00) 16 8.3% 4.66 (0.00) 4.27 (0.01) 32 8.3% 8.94 (0.01) 8.20 (0.01) 64 8.2% 17.47 (0.01) 16.04 (0.03) 120 8.5% 32.45 (0.13) 29.69 (0.01) perf diff confirms less time spent in pup. Here are the top ten functions: Baseline Delta Abs Symbol 78.63% +6.64% clear_page_erms 1.50% -1.50% __gup_longterm_locked 1.27% -0.78% __get_user_pages +0.76% kvm_zap_rmapp.constprop.0 0.54% -0.53% vmacache_find 0.55% -0.51% get_pfnblock_flags_mask 0.48% -0.48% __get_user_pages_remote +0.39% slot_rmap_walk_next +0.32% vfio_pin_map_dma +0.26% kvm_handle_hva_range ...
Unified · Split
| @@ -107,6 +107,8 @@ struct vfio_batch { · 106 unmodified lines | |||
| 107 | struct page **pages; /* for pin_user_pages_remote */ | 107 | struct page **pages; /* for pin_user_pages_remote */ |
| 108 | struct page *fallback_page; /* if pages alloc fails */ | 108 | struct page *fallback_page; /* if pages alloc fails */ |
| 109 | int capacity; /* length of pages array */ | 109 | int capacity; /* length of pages array */ |
| 110 | int size; /* of batch currently */ | ||
| 111 | int offset; /* of next entry in pages */ | ||
| 110 | }; | 112 | }; |
| 111 | 113 | ||
| 112 | struct vfio_group { | 114 | struct vfio_group { |
| @@ -469,6 +471,9 @@ static int put_pfn(unsigned long pfn, int prot) · 356 unmodified lines | |||
| 469 | 471 | ||
| 470 | static void vfio_batch_init(struct vfio_batch *batch) | 472 | static void vfio_batch_init(struct vfio_batch *batch) |
| 471 | { | 473 | { |
| 474 | batch->size = 0; | ||
| 475 | batch->offset = 0; | ||
| 476 | |||
| 472 | if (unlikely(disable_hugepages)) | 477 | if (unlikely(disable_hugepages)) |
| 473 | goto fallback; | 478 | goto fallback; |
| 474 | 479 | ||
| @@ -484,6 +489,17 @@ fallback: · 9 unmodified lines | |||
| 484 | batch->capacity = 1; | 489 | batch->capacity = 1; |
| 485 | } | 490 | } |
| 486 | 491 | ||
| 492 | static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma) | ||
| 493 | { | ||
| 494 | while (batch->size) { | ||
| 495 | unsigned long pfn = page_to_pfn(batch->pages[batch->offset]); | ||
| 496 | |||
| 497 | put_pfn(pfn, dma->prot); | ||
| 498 | batch->offset++; | ||
| 499 | batch->size--; | ||
| 500 | } | ||
| 501 | } | ||
| 502 | |||
| 487 | static void vfio_batch_fini(struct vfio_batch *batch) | 503 | static void vfio_batch_fini(struct vfio_batch *batch) |
| 488 | { | 504 | { |
| 489 | if (batch->capacity == VFIO_BATCH_MAX_CAPACITY) | 505 | if (batch->capacity == VFIO_BATCH_MAX_CAPACITY) |
| @@ -635,65 +651,88 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, · 145 unmodified lines | |||
| 635 | long npage, unsigned long *pfn_base, | 651 | long npage, unsigned long *pfn_base, |
| 636 | unsigned long limit, struct vfio_batch *batch) | 652 | unsigned long limit, struct vfio_batch *batch) |
| 637 | { | 653 | { |
| 638 | unsigned long pfn = 0; | 654 | unsigned long pfn; |
| 655 | struct mm_struct *mm = current->mm; | ||
| 639 | long ret, pinned = 0, lock_acct = 0; | 656 | long ret, pinned = 0, lock_acct = 0; |
| 640 | bool rsvd; | 657 | bool rsvd; |
| 641 | dma_addr_t iova = vaddr - dma->vaddr + dma->iova; | 658 | dma_addr_t iova = vaddr - dma->vaddr + dma->iova; |
| 642 | 659 | ||
| 643 | /* This code path is only user initiated */ | 660 | /* This code path is only user initiated */ |
| 644 | if (!current->mm) | 661 | if (!mm) |
| 645 | return -ENODEV; | 662 | return -ENODEV; |
| 646 | 663 | ||
| 647 | ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, pfn_base, | 664 | if (batch->size) { |
| 648 | batch->pages); | 665 | /* Leftover pages in batch from an earlier call. */ |
| 649 | if (ret < 0) | 666 | *pfn_base = page_to_pfn(batch->pages[batch->offset]); |
| 650 | return ret; | 667 | pfn = *pfn_base; |
| 651 | 668 | rsvd = is_invalid_reserved_pfn(*pfn_base); | |
| 652 | pinned++; | 669 | } else { |
| 653 | rsvd = is_invalid_reserved_pfn(*pfn_base); | 670 | *pfn_base = 0; |
| 654 | |||
| 655 | /* | ||
| 656 | * Reserved pages aren't counted against the user, externally pinned | ||
| 657 | * pages are already counted against the user. | ||
| 658 | */ | ||
| 659 | if (!rsvd && !vfio_find_vpfn(dma, iova)) { | ||
| 660 | if (!dma->lock_cap && current->mm->locked_vm + 1 > limit) { | ||
| 661 | put_pfn(*pfn_base, dma->prot); | ||
| 662 | pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, | ||
| 663 | limit << PAGE_SHIFT); | ||
| 664 | return -ENOMEM; | ||
| 665 | } | ||
| 666 | lock_acct++; | ||
| 667 | } | 671 | } |
| 668 | 672 | ||
| 669 | if (unlikely(disable_hugepages)) | 673 | while (npage) { |
| 670 | goto out; | 674 | if (!batch->size) { |
| 675 | /* Empty batch, so refill it. */ | ||
| 676 | long req_pages = min_t(long, npage, batch->capacity); | ||
| 671 | 677 | ||
| 672 | /* Lock all the consecutive pages from pfn_base */ | 678 | ret = vaddr_get_pfns(mm, vaddr, req_pages, dma->prot, |
| 673 | for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage; | 679 | &pfn, batch->pages); |
| 674 | pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) { | 680 | if (ret < 0) |
| 675 | ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, &pfn, | 681 | goto unpin_out; |
| 676 | batch->pages); | ||
| 677 | if (ret < 0) | ||
| 678 | break; | ||
| 679 | 682 | ||
| 680 | if (pfn != *pfn_base + pinned || | 683 | batch->size = ret; |
| 681 | rsvd != is_invalid_reserved_pfn(pfn)) { | 684 | batch->offset = 0; |
| 682 | put_pfn(pfn, dma->prot); | 685 | |
| 683 | break; | 686 | if (!*pfn_base) { |
| 687 | *pfn_base = pfn; | ||
| 688 | rsvd = is_invalid_reserved_pfn(*pfn_base); | ||
| 689 | } | ||
| 684 | } | 690 | } |
| 685 | 691 | ||
| 686 | if (!rsvd && !vfio_find_vpfn(dma, iova)) { | 692 | /* |
| 687 | if (!dma->lock_cap && | 693 | * pfn is preset for the first iteration of this inner loop and |
| 688 | current->mm->locked_vm + lock_acct + 1 > limit) { | 694 | * updated at the end to handle a VM_PFNMAP pfn. In that case, |
| 689 | put_pfn(pfn, dma->prot); | 695 | * batch->pages isn't valid (there's no struct page), so allow |
| 690 | pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", | 696 | * batch->pages to be touched only when there's more than one |
| 691 | __func__, limit << PAGE_SHIFT); | 697 | * pfn to check, which guarantees the pfns are from a |
| 692 | ret = -ENOMEM; | 698 | * !VM_PFNMAP vma. |
| 693 | goto unpin_out; | 699 | */ |
| 700 | while (true) { | ||
| 701 | if (pfn != *pfn_base + pinned || | ||
| 702 | rsvd != is_invalid_reserved_pfn(pfn)) | ||
| 703 | goto out; | ||
| 704 | |||
| 705 | /* | ||
| 706 | * Reserved pages aren't counted against the user, | ||
| 707 | * externally pinned pages are already counted against | ||
| 708 | * the user. | ||
| 709 | */ | ||
| 710 | if (!rsvd && !vfio_find_vpfn(dma, iova)) { | ||
| 711 | if (!dma->lock_cap && | ||
| 712 | mm->locked_vm + lock_acct + 1 > limit) { | ||
| 713 | pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", | ||
| 714 | __func__, limit << PAGE_SHIFT); | ||
| 715 | ret = -ENOMEM; | ||
| 716 | goto unpin_out; | ||
| 717 | } | ||
| 718 | lock_acct++; | ||
| 694 | } | 719 | } |
| 695 | lock_acct++; | 720 | |
| 721 | pinned++; | ||
| 722 | npage--; | ||
| 723 | vaddr += PAGE_SIZE; | ||
| 724 | iova += PAGE_SIZE; | ||
| 725 | batch->offset++; | ||
| 726 | batch->size--; | ||
| 727 | |||
| 728 | if (!batch->size) | ||
| 729 | break; | ||
| 730 | |||
| 731 | pfn = page_to_pfn(batch->pages[batch->offset]); | ||
| 696 | } | 732 | } |
| 733 | |||
| 734 | if (unlikely(disable_hugepages)) | ||
| 735 | break; | ||
| 697 | } | 736 | } |
| 698 | 737 | ||
| 699 | out: | 738 | out: |
| @@ -701,10 +740,11 @@ out: · 1 unmodified line | |||
| 701 | 740 | ||
| 702 | unpin_out: | 741 | unpin_out: |
| 703 | if (ret < 0) { | 742 | if (ret < 0) { |
| 704 | if (!rsvd) { | 743 | if (pinned && !rsvd) { |
| 705 | for (pfn = *pfn_base ; pinned ; pfn++, pinned--) | 744 | for (pfn = *pfn_base ; pinned ; pfn++, pinned--) |
| 706 | put_pfn(pfn, dma->prot); | 745 | put_pfn(pfn, dma->prot); |
| 707 | } | 746 | } |
| 747 | vfio_batch_unpin(batch, dma); | ||
| 708 | 748 | ||
| 709 | return ret; | 749 | return ret; |
| 710 | } | 750 | } |
| @@ -1463,6 +1503,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, · 752 unmodified lines | |||
| 1463 | if (ret) { | 1503 | if (ret) { |
| 1464 | vfio_unpin_pages_remote(dma, iova + dma->size, pfn, | 1504 | vfio_unpin_pages_remote(dma, iova + dma->size, pfn, |
| 1465 | npage, true); | 1505 | npage, true); |
| 1506 | vfio_batch_unpin(&batch, dma); | ||
| 1466 | break; | 1507 | break; |
| 1467 | } | 1508 | } |
| 1468 | 1509 | ||
| @@ -1726,11 +1767,13 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, · 257 unmodified lines | |||
| 1726 | ret = iommu_map(domain->domain, iova, phys, | 1767 | ret = iommu_map(domain->domain, iova, phys, |
| 1727 | size, dma->prot | domain->prot); | 1768 | size, dma->prot | domain->prot); |
| 1728 | if (ret) { | 1769 | if (ret) { |
| 1729 | if (!dma->iommu_mapped) | 1770 | if (!dma->iommu_mapped) { |
| 1730 | vfio_unpin_pages_remote(dma, iova, | 1771 | vfio_unpin_pages_remote(dma, iova, |
| 1731 | phys >> PAGE_SHIFT, | 1772 | phys >> PAGE_SHIFT, |
| 1732 | size >> PAGE_SHIFT, | 1773 | size >> PAGE_SHIFT, |
| 1733 | true); | 1774 | true); |
| 1775 | vfio_batch_unpin(&batch, dma); | ||
| 1776 | } | ||
| 1734 | goto unwind; | 1777 | goto unwind; |
| 1735 | } | 1778 | } |
| 1736 | 1779 | ||