History

vfio/type1: Batch page pinning

Pinning one 4K page at a time is inefficient, so do it in batches of 512 instead. This is just an optimization with no functional change intended, and in particular the driver still calls iommu_map() with the largest physically contiguous range possible. Add two fields in vfio_batch to remember where to start between calls to vfio_pin_pages_remote(), and use vfio_batch_unpin() to handle remaining pages in the batch in case of error. qemu pins pages for guests around 8% faster on my test system, a two-node Broadwell server with 128G memory per node. The qemu process was bound to one node with its allocations constrained there as well. base test guest ---------------- ---------------- mem (GB) speedup avg sec (std) avg sec (std) 1 7.4% 0.61 (0.00) 0.56 (0.00) 2 8.3% 0.93 (0.00) 0.85 (0.00) 4 8.4% 1.46 (0.00) 1.34 (0.00) 8 8.6% 2.54 (0.01) 2.32 (0.00) 16 8.3% 4.66 (0.00) 4.27 (0.01) 32 8.3% 8.94 (0.01) 8.20 (0.01) 64 8.2% 17.47 (0.01) 16.04 (0.03) 120 8.5% 32.45 (0.13) 29.69 (0.01) perf diff confirms less time spent in pup. Here are the top ten functions: Baseline Delta Abs Symbol 78.63% +6.64% clear_page_erms 1.50% -1.50% __gup_longterm_locked 1.27% -0.78% __get_user_pages +0.76% kvm_zap_rmapp.constprop.0 0.54% -0.53% vmacache_find 0.55% -0.51% get_pfnblock_flags_mask 0.48% -0.48% __get_user_pages_remote +0.39% slot_rmap_walk_next +0.32% vfio_pin_map_dma +0.26% kvm_handle_hva_range ...

Daniel Jordan committed 5 years ago

Commit 4d83de61 parent 4b6c33b1 file · +89 −46Browse files

Unified · Split

drivers/vfio/vfio_iommu_type1.c+89−46
@@ -107,6 +107,8 @@ struct vfio_batch { · 106 unmodified lines
107107 struct page **pages; /* for pin_user_pages_remote */
108108 struct page *fallback_page; /* if pages alloc fails */
109109 int capacity; /* length of pages array */
110+ int size; /* of batch currently */
111+ int offset; /* of next entry in pages */
110112 };
111113
112114 struct vfio_group {
@@ -469,6 +471,9 @@ static int put_pfn(unsigned long pfn, int prot) · 356 unmodified lines
469471
470472 static void vfio_batch_init(struct vfio_batch *batch)
471473 {
474+ batch->size = 0;
475+ batch->offset = 0;
476+
472477 if (unlikely(disable_hugepages))
473478 goto fallback;
474479
@@ -484,6 +489,17 @@ fallback: · 9 unmodified lines
484489 batch->capacity = 1;
485490 }
486491
492+static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma)
493+{
494+ while (batch->size) {
495+ unsigned long pfn = page_to_pfn(batch->pages[batch->offset]);
496+
497+ put_pfn(pfn, dma->prot);
498+ batch->offset++;
499+ batch->size--;
500+ }
501+}
502+
487503 static void vfio_batch_fini(struct vfio_batch *batch)
488504 {
489505 if (batch->capacity == VFIO_BATCH_MAX_CAPACITY)
@@ -635,65 +651,88 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, · 145 unmodified lines
635651 long npage, unsigned long *pfn_base,
636652 unsigned long limit, struct vfio_batch *batch)
637653 {
638− unsigned long pfn = 0;
654+ unsigned long pfn;
655+ struct mm_struct *mm = current->mm;
639656 long ret, pinned = 0, lock_acct = 0;
640657 bool rsvd;
641658 dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
642659
643660 /* This code path is only user initiated */
644− if (!current->mm)
661+ if (!mm)
645662 return -ENODEV;
646663
647− ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, pfn_base,
648− batch->pages);
649− if (ret < 0)
650− return ret;
651−
652− pinned++;
653− rsvd = is_invalid_reserved_pfn(*pfn_base);
654−
655− /*
656− * Reserved pages aren't counted against the user, externally pinned
657− * pages are already counted against the user.
658− */
659− if (!rsvd && !vfio_find_vpfn(dma, iova)) {
660− if (!dma->lock_cap && current->mm->locked_vm + 1 > limit) {
661− put_pfn(*pfn_base, dma->prot);
662− pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
663− limit << PAGE_SHIFT);
664− return -ENOMEM;
665− }
666− lock_acct++;
664+ if (batch->size) {
665+ /* Leftover pages in batch from an earlier call. */
666+ *pfn_base = page_to_pfn(batch->pages[batch->offset]);
667+ pfn = *pfn_base;
668+ rsvd = is_invalid_reserved_pfn(*pfn_base);
669+ } else {
670+ *pfn_base = 0;
667671 }
668672
669− if (unlikely(disable_hugepages))
670− goto out;
673+ while (npage) {
674+ if (!batch->size) {
675+ /* Empty batch, so refill it. */
676+ long req_pages = min_t(long, npage, batch->capacity);
671677
672− /* Lock all the consecutive pages from pfn_base */
673− for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage;
674− pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) {
675− ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, &pfn,
676− batch->pages);
677− if (ret < 0)
678− break;
678+ ret = vaddr_get_pfns(mm, vaddr, req_pages, dma->prot,
679+ &pfn, batch->pages);
680+ if (ret < 0)
681+ goto unpin_out;
679682
680− if (pfn != *pfn_base + pinned ||
681− rsvd != is_invalid_reserved_pfn(pfn)) {
682− put_pfn(pfn, dma->prot);
683− break;
683+ batch->size = ret;
684+ batch->offset = 0;
685+
686+ if (!*pfn_base) {
687+ *pfn_base = pfn;
688+ rsvd = is_invalid_reserved_pfn(*pfn_base);
689+ }
684690 }
685691
686− if (!rsvd && !vfio_find_vpfn(dma, iova)) {
687− if (!dma->lock_cap &&
688− current->mm->locked_vm + lock_acct + 1 > limit) {
689− put_pfn(pfn, dma->prot);
690− pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
691− __func__, limit << PAGE_SHIFT);
692− ret = -ENOMEM;
693− goto unpin_out;
692+ /*
693+ * pfn is preset for the first iteration of this inner loop and
694+ * updated at the end to handle a VM_PFNMAP pfn. In that case,
695+ * batch->pages isn't valid (there's no struct page), so allow
696+ * batch->pages to be touched only when there's more than one
697+ * pfn to check, which guarantees the pfns are from a
698+ * !VM_PFNMAP vma.
699+ */
700+ while (true) {
701+ if (pfn != *pfn_base + pinned ||
702+ rsvd != is_invalid_reserved_pfn(pfn))
703+ goto out;
704+
705+ /*
706+ * Reserved pages aren't counted against the user,
707+ * externally pinned pages are already counted against
708+ * the user.
709+ */
710+ if (!rsvd && !vfio_find_vpfn(dma, iova)) {
711+ if (!dma->lock_cap &&
712+ mm->locked_vm + lock_acct + 1 > limit) {
713+ pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
714+ __func__, limit << PAGE_SHIFT);
715+ ret = -ENOMEM;
716+ goto unpin_out;
717+ }
718+ lock_acct++;
694719 }
695− lock_acct++;
720+
721+ pinned++;
722+ npage--;
723+ vaddr += PAGE_SIZE;
724+ iova += PAGE_SIZE;
725+ batch->offset++;
726+ batch->size--;
727+
728+ if (!batch->size)
729+ break;
730+
731+ pfn = page_to_pfn(batch->pages[batch->offset]);
696732 }
733+
734+ if (unlikely(disable_hugepages))
735+ break;
697736 }
698737
699738 out:
@@ -701,10 +740,11 @@ out: · 1 unmodified line
701740
702741 unpin_out:
703742 if (ret < 0) {
704− if (!rsvd) {
743+ if (pinned && !rsvd) {
705744 for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
706745 put_pfn(pfn, dma->prot);
707746 }
747+ vfio_batch_unpin(batch, dma);
708748
709749 return ret;
710750 }
@@ -1463,6 +1503,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, · 752 unmodified lines
14631503 if (ret) {
14641504 vfio_unpin_pages_remote(dma, iova + dma->size, pfn,
14651505 npage, true);
1506+ vfio_batch_unpin(&batch, dma);
14661507 break;
14671508 }
14681509
@@ -1726,11 +1767,13 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, · 257 unmodified lines
17261767 ret = iommu_map(domain->domain, iova, phys,
17271768 size, dma->prot | domain->prot);
17281769 if (ret) {
1729− if (!dma->iommu_mapped)
1770+ if (!dma->iommu_mapped) {
17301771 vfio_unpin_pages_remote(dma, iova,
17311772 phys >> PAGE_SHIFT,
17321773 size >> PAGE_SHIFT,
17331774 true);
1775+ vfio_batch_unpin(&batch, dma);
1776+ }
17341777 goto unwind;
17351778 }
17361779