History

Merge tag 'for-linus-hmm' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull hmm updates from Jason Gunthorpe: "Ralph has been working on nouveau's use of hmm_range_fault() and migrate_vma() which resulted in this small series. It adds reporting of the page table order from hmm_range_fault() and some optimization of migrate_vma(): - Report the size of the page table mapping out of hmm_range_fault(). This makes it easier to establish a large/huge/etc mapping in the device's page table. - Allow devices to ignore the invalidations during migration in cases where the migration is not going to change pages. For instance migrating pages to a device does not require the device to invalidate pages already in the device. - Update nouveau and hmm_tests to use the above" * tag 'for-linus-hmm' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: mm/hmm/test: use the new migration invalidation nouveau/svm: use the new migration invalidation mm/notifier: add migration invalidation type mm/migrate: add a flags parameter to migrate_vma nouveau: fix storing invalid ptes nouveau/hmm: support mapping large sysmem pages nouveau: fix mapping 2MB sysmem pages nouveau/hmm: fault one page at a time mm/hmm: add tests for hmm_pfn_to_map_order() mm/hmm: provide the page mapping order in hmm_range_fault()

Linus Torvalds committed 6 years ago

Commit fffe3ae2 parents 8f7be62 7d17e8314 files · +410 −200Browse files

Unified · Split

14 files
  1. arch/powerpc/kvm/book3s_hv_uvmem.c +3 −1
  2. drivers/gpu/drm/nouveau/nouveau_dmem.c +15 −4
  3. drivers/gpu/drm/nouveau/nouveau_svm.c +107 −150
  4. drivers/gpu/drm/nouveau/nouveau_svm.h +12 −1
  5. drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +2 −3
  6. drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c +89 −2
  7. include/linux/hmm.h +22 −2
  8. include/linux/migrate.h +12 −4
  9. include/linux/mmu_notifier.h +6 −0
  10. lib/test_hmm.c +24 −23
  11. lib/test_hmm_uapi.h +4 −0
  12. mm/hmm.c +13 −3
  13. mm/migrate.c +11 −3
  14. tools/testing/selftests/vm/hmm-tests.c +90 −4
arch/powerpc/kvm/book3s_hv_uvmem.c+3−1
@@ -400,6 +400,7 @@ kvmppc_svm_page_in(struct vm_area_struct *vma, unsigned long start, · 399 unmodified lines
400 mig.end = end;400 mig.end = end;
401 mig.src = &src_pfn;401 mig.src = &src_pfn;
402 mig.dst = &dst_pfn;402 mig.dst = &dst_pfn;
403 mig.flags = MIGRATE_VMA_SELECT_SYSTEM;
403 404
404 /*405 /*
405 * We come here with mmap_lock write lock held just for406 * We come here with mmap_lock write lock held just for
@@ -577,7 +578,8 @@ kvmppc_svm_page_out(struct vm_area_struct *vma, unsigned long start, · 171 unmodified lines
577 mig.end = end;578 mig.end = end;
578 mig.src = &src_pfn;579 mig.src = &src_pfn;
579 mig.dst = &dst_pfn;580 mig.dst = &dst_pfn;
580 mig.src_owner = &kvmppc_uvmem_pgmap;581 mig.pgmap_owner = &kvmppc_uvmem_pgmap;
582 mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
581 583
582 mutex_lock(&kvm->arch.uvmem_lock);584 mutex_lock(&kvm->arch.uvmem_lock);
583 /* The requested page is already paged-out, nothing to do */585 /* The requested page is already paged-out, nothing to do */
drivers/gpu/drm/nouveau/nouveau_dmem.c+15−4
@@ -140,6 +140,7 @@ static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, · 139 unmodified lines
140{140{
141 struct device *dev = drm->dev->dev;141 struct device *dev = drm->dev->dev;
142 struct page *dpage, *spage;142 struct page *dpage, *spage;
143 struct nouveau_svmm *svmm;
143 144
144 spage = migrate_pfn_to_page(args->src[0]);145 spage = migrate_pfn_to_page(args->src[0]);
145 if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE))146 if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE))
@@ -154,14 +155,19 @@ static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, · 8 unmodified lines
154 if (dma_mapping_error(dev, *dma_addr))155 if (dma_mapping_error(dev, *dma_addr))
155 goto error_free_page;156 goto error_free_page;
156 157
158 svmm = spage->zone_device_data;
159 mutex_lock(&svmm->mutex);
160 nouveau_svmm_invalidate(svmm, args->start, args->end);
157 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr,161 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr,
158 NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage)))162 NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage)))
159 goto error_dma_unmap;163 goto error_dma_unmap;
164 mutex_unlock(&svmm->mutex);
160 165
161 args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;166 args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
162 return 0;167 return 0;
163 168
164error_dma_unmap:169error_dma_unmap:
170 mutex_unlock(&svmm->mutex);
165 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);171 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
166error_free_page:172error_free_page:
167 __free_page(dpage);173 __free_page(dpage);
@@ -182,7 +188,8 @@ static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) · 14 unmodified lines
182 .end = vmf->address + PAGE_SIZE,188 .end = vmf->address + PAGE_SIZE,
183 .src = &src,189 .src = &src,
184 .dst = &dst,190 .dst = &dst,
185 .src_owner = drm->dev,191 .pgmap_owner = drm->dev,
192 .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE,
186 };193 };
187 194
188 /*195 /*
@@ -530,7 +537,8 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 341 unmodified lines
530}537}
531 538
532static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm,539static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm,
533 unsigned long src, dma_addr_t *dma_addr, u64 *pfn)540 struct nouveau_svmm *svmm, unsigned long src,
541 dma_addr_t *dma_addr, u64 *pfn)
534{542{
535 struct device *dev = drm->dev->dev;543 struct device *dev = drm->dev->dev;
536 struct page *dpage, *spage;544 struct page *dpage, *spage;
@@ -560,6 +568,7 @@ static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm, · 23 unmodified lines
560 goto out_free_page;568 goto out_free_page;
561 }569 }
562 570
571 dpage->zone_device_data = svmm;
563 *pfn = NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_VRAM |572 *pfn = NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_VRAM |
564 ((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT);573 ((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT);
565 if (src & MIGRATE_PFN_WRITE)574 if (src & MIGRATE_PFN_WRITE)
@@ -583,8 +592,8 @@ static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm, · 17 unmodified lines
583 unsigned long addr = args->start, nr_dma = 0, i;592 unsigned long addr = args->start, nr_dma = 0, i;
584 593
585 for (i = 0; addr < args->end; i++) {594 for (i = 0; addr < args->end; i++) {
586 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, args->src[i],595 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, svmm,
587 dma_addrs + nr_dma, pfns + i);596 args->src[i], dma_addrs + nr_dma, pfns + i);
588 if (!dma_mapping_error(drm->dev->dev, dma_addrs[nr_dma]))597 if (!dma_mapping_error(drm->dev->dev, dma_addrs[nr_dma]))
589 nr_dma++;598 nr_dma++;
590 addr += PAGE_SIZE;599 addr += PAGE_SIZE;
@@ -615,6 +624,8 @@ nouveau_dmem_migrate_vma(struct nouveau_drm *drm, · 24 unmodified lines
615 struct migrate_vma args = {624 struct migrate_vma args = {
616 .vma = vma,625 .vma = vma,
617 .start = start,626 .start = start,
627 .pgmap_owner = drm->dev,
628 .flags = MIGRATE_VMA_SELECT_SYSTEM,
618 };629 };
619 unsigned long i;630 unsigned long i;
620 u64 *pfns;631 u64 *pfns;
drivers/gpu/drm/nouveau/nouveau_svm.c+107−150
@@ -93,17 +93,6 @@ nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst) · 92 unmodified lines
93 return NULL;93 return NULL;
94}94}
95 95
96struct nouveau_svmm {
97 struct mmu_notifier notifier;
98 struct nouveau_vmm *vmm;
99 struct {
100 unsigned long start;
101 unsigned long limit;
102 } unmanaged;
103
104 struct mutex mutex;
105};
106
107#define SVMM_DBG(s,f,a...) \96#define SVMM_DBG(s,f,a...) \
108 NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a)97 NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a)
109#define SVMM_ERR(s,f,a...) \98#define SVMM_ERR(s,f,a...) \
@@ -246,7 +235,7 @@ nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst) · 136 unmodified lines
246}235}
247 236
248/* Invalidate SVMM address-range on GPU. */237/* Invalidate SVMM address-range on GPU. */
249static void238void
250nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit)239nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit)
251{240{
252 if (limit > start) {241 if (limit > start) {
@@ -279,6 +268,14 @@ nouveau_svmm_invalidate_range_start(struct mmu_notifier *mn, · 26 unmodified lines
279 if (unlikely(!svmm->vmm))268 if (unlikely(!svmm->vmm))
280 goto out;269 goto out;
281 270
271 /*
272 * Ignore invalidation callbacks for device private pages since
273 * the invalidation is handled as part of the migration process.
274 */
275 if (update->event == MMU_NOTIFY_MIGRATE &&
276 update->migrate_pgmap_owner == svmm->vmm->cli->drm->dev)
277 goto out;
278
282 if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) {279 if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) {
283 if (start < svmm->unmanaged.start) {280 if (start < svmm->unmanaged.start) {
284 nouveau_svmm_invalidate(svmm, start,281 nouveau_svmm_invalidate(svmm, start,
@@ -514,53 +511,68 @@ static const struct mmu_interval_notifier_ops nouveau_svm_mni_ops = { · 229 unmodified lines
514};511};
515 512
516static void nouveau_hmm_convert_pfn(struct nouveau_drm *drm,513static void nouveau_hmm_convert_pfn(struct nouveau_drm *drm,
517 struct hmm_range *range, u64 *ioctl_addr)514 struct hmm_range *range,
515 struct nouveau_pfnmap_args *args)
518{516{
519 unsigned long i, npages;517 struct page *page;
520 518
521 /*519 /*
522 * The ioctl_addr prepared here is passed through nvif_object_ioctl()520 * The address prepared here is passed through nvif_object_ioctl()
523 * to an eventual DMA map in something like gp100_vmm_pgt_pfn()521 * to an eventual DMA map in something like gp100_vmm_pgt_pfn()
524 *522 *
525 * This is all just encoding the internal hmm representation into a523 * This is all just encoding the internal hmm representation into a
526 * different nouveau internal representation.524 * different nouveau internal representation.
527 */525 */
528 npages = (range->end - range->start) >> PAGE_SHIFT;526 if (!(range->hmm_pfns[0] & HMM_PFN_VALID)) {
529 for (i = 0; i < npages; ++i) {527 args->p.phys[0] = 0;
530 struct page *page;528 return;
531 529 }
532 if (!(range->hmm_pfns[i] & HMM_PFN_VALID)) {
533 ioctl_addr[i] = 0;
534 continue;
535 }
536 530
537 page = hmm_pfn_to_page(range->hmm_pfns[i]);531 page = hmm_pfn_to_page(range->hmm_pfns[0]);
538 if (is_device_private_page(page))532 /*
539 ioctl_addr[i] = nouveau_dmem_page_addr(page) |533 * Only map compound pages to the GPU if the CPU is also mapping the
540 NVIF_VMM_PFNMAP_V0_V |534 * page as a compound page. Otherwise, the PTE protections might not be
541 NVIF_VMM_PFNMAP_V0_VRAM;535 * consistent (e.g., CPU only maps part of a compound page).
542 else536 * Note that the underlying page might still be larger than the
543 ioctl_addr[i] = page_to_phys(page) |537 * CPU mapping (e.g., a PUD sized compound page partially mapped with
544 NVIF_VMM_PFNMAP_V0_V |538 * a PMD sized page table entry).
545 NVIF_VMM_PFNMAP_V0_HOST;539 */
546 if (range->hmm_pfns[i] & HMM_PFN_WRITE)540 if (hmm_pfn_to_map_order(range->hmm_pfns[0])) {
547 ioctl_addr[i] |= NVIF_VMM_PFNMAP_V0_W;541 unsigned long addr = args->p.addr;
542
543 args->p.page = hmm_pfn_to_map_order(range->hmm_pfns[0]) +
544 PAGE_SHIFT;
545 args->p.size = 1UL << args->p.page;
546 args->p.addr &= ~(args->p.size - 1);
547 page -= (addr - args->p.addr) >> PAGE_SHIFT;
548 }548 }
549 if (is_device_private_page(page))
550 args->p.phys[0] = nouveau_dmem_page_addr(page) |
551 NVIF_VMM_PFNMAP_V0_V |
552 NVIF_VMM_PFNMAP_V0_VRAM;
553 else
554 args->p.phys[0] = page_to_phys(page) |
555 NVIF_VMM_PFNMAP_V0_V |
556 NVIF_VMM_PFNMAP_V0_HOST;
557 if (range->hmm_pfns[0] & HMM_PFN_WRITE)
558 args->p.phys[0] |= NVIF_VMM_PFNMAP_V0_W;
549}559}
550 560
551static int nouveau_range_fault(struct nouveau_svmm *svmm,561static int nouveau_range_fault(struct nouveau_svmm *svmm,
552 struct nouveau_drm *drm, void *data, u32 size,562 struct nouveau_drm *drm,
553 unsigned long hmm_pfns[], u64 *ioctl_addr,563 struct nouveau_pfnmap_args *args, u32 size,
564 unsigned long hmm_flags,
554 struct svm_notifier *notifier)565 struct svm_notifier *notifier)
555{566{
556 unsigned long timeout =567 unsigned long timeout =
557 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);568 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
558 /* Have HMM fault pages within the fault window to the GPU. */569 /* Have HMM fault pages within the fault window to the GPU. */
570 unsigned long hmm_pfns[1];
559 struct hmm_range range = {571 struct hmm_range range = {
560 .notifier = &notifier->notifier,572 .notifier = &notifier->notifier,
561 .start = notifier->notifier.interval_tree.start,573 .start = notifier->notifier.interval_tree.start,
562 .end = notifier->notifier.interval_tree.last + 1,574 .end = notifier->notifier.interval_tree.last + 1,
563 .pfn_flags_mask = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE,575 .default_flags = hmm_flags,
564 .hmm_pfns = hmm_pfns,576 .hmm_pfns = hmm_pfns,
565 .dev_private_owner = drm->dev,577 .dev_private_owner = drm->dev,
566 };578 };
@@ -576,11 +588,6 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, · 9 unmodified lines
576 ret = hmm_range_fault(&range);588 ret = hmm_range_fault(&range);
577 mmap_read_unlock(mm);589 mmap_read_unlock(mm);
578 if (ret) {590 if (ret) {
579 /*
580 * FIXME: the input PFN_REQ flags are destroyed on
581 * -EBUSY, we need to regenerate them, also for the
582 * other continue below
583 */
584 if (ret == -EBUSY)591 if (ret == -EBUSY)
585 continue;592 continue;
586 return ret;593 return ret;
@@ -595,10 +602,10 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, · 8 unmodified lines
595 break;602 break;
596 }603 }
597 604
598 nouveau_hmm_convert_pfn(drm, &range, ioctl_addr);605 nouveau_hmm_convert_pfn(drm, &range, args);
599 606
600 svmm->vmm->vmm.object.client->super = true;607 svmm->vmm->vmm.object.client->super = true;
601 ret = nvif_object_ioctl(&svmm->vmm->vmm.object, data, size, NULL);608 ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL);
602 svmm->vmm->vmm.object.client->super = false;609 svmm->vmm->vmm.object.client->super = false;
603 mutex_unlock(&svmm->mutex);610 mutex_unlock(&svmm->mutex);
604 611
@@ -615,17 +622,12 @@ nouveau_svm_fault(struct nvif_notify *notify) · 10 unmodified lines
615 struct nvif_object *device = &svm->drm->client.device.object;622 struct nvif_object *device = &svm->drm->client.device.object;
616 struct nouveau_svmm *svmm;623 struct nouveau_svmm *svmm;
617 struct {624 struct {
618 struct {625 struct nouveau_pfnmap_args i;
619 struct nvif_ioctl_v0 i;626 u64 phys[1];
620 struct nvif_ioctl_mthd_v0 m;
621 struct nvif_vmm_pfnmap_v0 p;
622 } i;
623 u64 phys[16];
624 } args;627 } args;
625 unsigned long hmm_pfns[ARRAY_SIZE(args.phys)];628 unsigned long hmm_flags;
626 struct vm_area_struct *vma;
627 u64 inst, start, limit;629 u64 inst, start, limit;
628 int fi, fn, pi, fill;630 int fi, fn;
629 int replay = 0, ret;631 int replay = 0, ret;
630 632
631 /* Parse available fault buffer entries into a cache, and update633 /* Parse available fault buffer entries into a cache, and update
@@ -692,128 +694,83 @@ nouveau_svm_fault(struct nvif_notify *notify) · 60 unmodified lines
692 * window into a single update.694 * window into a single update.
693 */695 */
694 start = buffer->fault[fi]->addr;696 start = buffer->fault[fi]->addr;
695 limit = start + (ARRAY_SIZE(args.phys) << PAGE_SHIFT);697 limit = start + PAGE_SIZE;
696 if (start < svmm->unmanaged.limit)698 if (start < svmm->unmanaged.limit)
697 limit = min_t(u64, limit, svmm->unmanaged.start);699 limit = min_t(u64, limit, svmm->unmanaged.start);
698 SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit);
699 700
700 mm = svmm->notifier.mm;701 /*
701 if (!mmget_not_zero(mm)) {702 * Prepare the GPU-side update of all pages within the
702 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);703 * fault window, determining required pages and access
703 continue;704 * permissions based on pending faults.
704 }
705
706 /* Intersect fault window with the CPU VMA, cancelling
707 * the fault if the address is invalid.
708 */705 */
709 mmap_read_lock(mm);706 args.i.p.addr = start;
710 vma = find_vma_intersection(mm, start, limit);707 args.i.p.page = PAGE_SHIFT;
711 if (!vma) {708 args.i.p.size = PAGE_SIZE;
712 SVMM_ERR(svmm, "wndw %016llx-%016llx", start, limit);709 /*
713 mmap_read_unlock(mm);710 * Determine required permissions based on GPU fault
714 mmput(mm);711 * access flags.
715 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);712 * XXX: atomic?
716 continue;713 */
714 switch (buffer->fault[fi]->access) {
715 case 0: /* READ. */
716 hmm_flags = HMM_PFN_REQ_FAULT;
717 break;
718 case 3: /* PREFETCH. */
719 hmm_flags = 0;
720 break;
721 default:
722 hmm_flags = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE;
723 break;
717 }724 }
718 start = max_t(u64, start, vma->vm_start);
719 limit = min_t(u64, limit, vma->vm_end);
720 mmap_read_unlock(mm);
721 SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit);
722 725
723 if (buffer->fault[fi]->addr != start) {726 mm = svmm->notifier.mm;
724 SVMM_ERR(svmm, "addr %016llx", buffer->fault[fi]->addr);727 if (!mmget_not_zero(mm)) {
725 mmput(mm);
726 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);728 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
727 continue;729 continue;
728 }730 }
729 731
730 /* Prepare the GPU-side update of all pages within the732 notifier.svmm = svmm;
731 * fault window, determining required pages and access733 ret = mmu_interval_notifier_insert(&notifier.notifier, mm,
732 * permissions based on pending faults.734 args.i.p.addr, args.i.p.size,
733 */735 &nouveau_svm_mni_ops);
734 args.i.p.page = PAGE_SHIFT;736 if (!ret) {
735 args.i.p.addr = start;737 ret = nouveau_range_fault(svmm, svm->drm, &args.i,
736 for (fn = fi, pi = 0;;) {738 sizeof(args), hmm_flags, &notifier);
737 /* Determine required permissions based on GPU fault739 mmu_interval_notifier_remove(&notifier.notifier);
738 * access flags.740 }
739 *XXX: atomic?741 mmput(mm);
740 */
741 switch (buffer->fault[fn]->access) {
742 case 0: /* READ. */
743 hmm_pfns[pi++] = HMM_PFN_REQ_FAULT;
744 break;
745 case 3: /* PREFETCH. */
746 hmm_pfns[pi++] = 0;
747 break;
748 default:
749 hmm_pfns[pi++] = HMM_PFN_REQ_FAULT |
750 HMM_PFN_REQ_WRITE;
751 break;
752 }
753 args.i.p.size = pi << PAGE_SHIFT;
754 742
743 limit = args.i.p.addr + args.i.p.size;
744 for (fn = fi; ++fn < buffer->fault_nr; ) {
755 /* It's okay to skip over duplicate addresses from the745 /* It's okay to skip over duplicate addresses from the
756 * same SVMM as faults are ordered by access type such746 * same SVMM as faults are ordered by access type such
757 * that only the first one needs to be handled.747 * that only the first one needs to be handled.
758 *748 *
759 * ie. WRITE faults appear first, thus any handling of749 * ie. WRITE faults appear first, thus any handling of
760 * pending READ faults will already be satisfied.750 * pending READ faults will already be satisfied.
751 * But if a large page is mapped, make sure subsequent
752 * fault addresses have sufficient access permission.
761 */753 */
762 while (++fn < buffer->fault_nr &&754 if (buffer->fault[fn]->svmm != svmm ||
763 buffer->fault[fn]->svmm == svmm &&755 buffer->fault[fn]->addr >= limit ||
764 buffer->fault[fn ]->addr ==756 (buffer->fault[fi]->access == 0 /* READ. */ &&
765 buffer->fault[fn - 1]->addr);757 !(args.phys[0] & NVIF_VMM_PFNMAP_V0_V)) ||
766 758 (buffer->fault[fi]->access != 0 /* READ. */ &&
767 /* If the next fault is outside the window, or all GPU759 buffer->fault[fi]->access != 3 /* PREFETCH. */ &&
768 * faults have been dealt with, we're done here.760 !(args.phys[0] & NVIF_VMM_PFNMAP_V0_W)))
769 */
770 if (fn >= buffer->fault_nr ||
771 buffer->fault[fn]->svmm != svmm ||
772 buffer->fault[fn]->addr >= limit)
773 break;761 break;
774
775 /* Fill in the gap between this fault and the next. */
776 fill = (buffer->fault[fn ]->addr -
777 buffer->fault[fn - 1]->addr) >> PAGE_SHIFT;
778 while (--fill)
779 hmm_pfns[pi++] = 0;
780 }762 }
781 763
782 SVMM_DBG(svmm, "wndw %016llx-%016llx covering %d fault(s)",764 /* If handling failed completely, cancel all faults. */
783 args.i.p.addr,765 if (ret) {
784 args.i.p.addr + args.i.p.size, fn - fi);766 while (fi < fn) {
785 767 struct nouveau_svm_fault *fault =
786 notifier.svmm = svmm;768 buffer->fault[fi++];
787 ret = mmu_interval_notifier_insert(&notifier.notifier,
788 svmm->notifier.mm,
789 args.i.p.addr, args.i.p.size,
790 &nouveau_svm_mni_ops);
791 if (!ret) {
792 ret = nouveau_range_fault(
793 svmm, svm->drm, &args,
794 sizeof(args.i) + pi * sizeof(args.phys[0]),
795 hmm_pfns, args.phys, &notifier);
796 mmu_interval_notifier_remove(&notifier.notifier);
797 }
798 mmput(mm);
799 769
800 /* Cancel any faults in the window whose pages didn't manage
801 * to keep their valid bit, or stay writeable when required.
802 *
803 * If handling failed completely, cancel all faults.
804 */
805 while (fi < fn) {
806 struct nouveau_svm_fault *fault = buffer->fault[fi++];
807 pi = (fault->addr - args.i.p.addr) >> PAGE_SHIFT;
808 if (ret ||
809 !(args.phys[pi] & NVIF_VMM_PFNMAP_V0_V) ||
810 (!(args.phys[pi] & NVIF_VMM_PFNMAP_V0_W) &&
811 fault->access != 0 && fault->access != 3)) {
812 nouveau_svm_fault_cancel_fault(svm, fault);770 nouveau_svm_fault_cancel_fault(svm, fault);
813 continue;
814 }771 }
772 } else
815 replay++;773 replay++;
816 }
817 }774 }
818 775
819 /* Issue fault replay to the GPU. */776 /* Issue fault replay to the GPU. */
drivers/gpu/drm/nouveau/nouveau_svm.h+12−1
@@ -1,11 +1,21 @@
1#ifndef __NOUVEAU_SVM_H__1#ifndef __NOUVEAU_SVM_H__
2#define __NOUVEAU_SVM_H__2#define __NOUVEAU_SVM_H__
3#include <nvif/os.h>3#include <nvif/os.h>
4#include <linux/mmu_notifier.h>
4struct drm_device;5struct drm_device;
5struct drm_file;6struct drm_file;
6struct nouveau_drm;7struct nouveau_drm;
7 8
8struct nouveau_svmm;9struct nouveau_svmm {
10 struct mmu_notifier notifier;
11 struct nouveau_vmm *vmm;
12 struct {
13 unsigned long start;
14 unsigned long limit;
15 } unmanaged;
16
17 struct mutex mutex;
18};
9 19
10#if IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM)20#if IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM)
11void nouveau_svm_init(struct nouveau_drm *);21void nouveau_svm_init(struct nouveau_drm *);
@@ -19,6 +29,7 @@ int nouveau_svmm_join(struct nouveau_svmm *, u64 inst); · 7 unmodified lines
19void nouveau_svmm_part(struct nouveau_svmm *, u64 inst);29void nouveau_svmm_part(struct nouveau_svmm *, u64 inst);
20int nouveau_svmm_bind(struct drm_device *, void *, struct drm_file *);30int nouveau_svmm_bind(struct drm_device *, void *, struct drm_file *);
21 31
32void nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit);
22u64 *nouveau_pfns_alloc(unsigned long npages);33u64 *nouveau_pfns_alloc(unsigned long npages);
23void nouveau_pfns_free(u64 *pfns);34void nouveau_pfns_free(u64 *pfns);
24void nouveau_pfns_map(struct nouveau_svmm *svmm, struct mm_struct *mm,35void nouveau_pfns_map(struct nouveau_svmm *svmm, struct mm_struct *mm,
drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c+2−3
@@ -1204,7 +1204,6 @@ nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size) · 1203 unmodified lines
1204/*TODO:1204/*TODO:
1205 * - Avoid PT readback (for dma_unmap etc), this might end up being dealt1205 * - Avoid PT readback (for dma_unmap etc), this might end up being dealt
1206 * with inside HMM, which would be a lot nicer for us to deal with.1206 * with inside HMM, which would be a lot nicer for us to deal with.
1207 * - Multiple page sizes (particularly for huge page support).
1208 * - Support for systems without a 4KiB page size.1207 * - Support for systems without a 4KiB page size.
1209 */1208 */
1210int1209int
@@ -1220,8 +1219,8 @@ nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn) · 9 unmodified lines
1220 /* Only support mapping where the page size of the incoming page1219 /* Only support mapping where the page size of the incoming page
1221 * array matches a page size available for direct mapping.1220 * array matches a page size available for direct mapping.
1222 */1221 */
1223 while (page->shift && page->shift != shift &&1222 while (page->shift && (page->shift != shift ||
1224 page->desc->func->pfn == NULL)1223 page->desc->func->pfn == NULL))
1225 page++;1224 page++;
1226 1225
1227 if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||1226 if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c+89−2
@@ -79,8 +79,12 @@ gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, · 78 unmodified lines
79 dma_addr_t addr;79 dma_addr_t addr;
80 80
81 nvkm_kmap(pt->memory);81 nvkm_kmap(pt->memory);
82 while (ptes--) {82 for (; ptes; ptes--, map->pfn++) {
83 u64 data = 0;83 u64 data = 0;
84
85 if (!(*map->pfn & NVKM_VMM_PFN_V))
86 continue;
87
84 if (!(*map->pfn & NVKM_VMM_PFN_W))88 if (!(*map->pfn & NVKM_VMM_PFN_W))
85 data |= BIT_ULL(6); /* RO. */89 data |= BIT_ULL(6); /* RO. */
86 90
@@ -100,7 +104,6 @@ gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, · 13 unmodified lines
100 }104 }
101 105
102 VMM_WO064(pt, vmm, ptei++ * 8, data);106 VMM_WO064(pt, vmm, ptei++ * 8, data);
103 map->pfn++;
104 }107 }
105 nvkm_done(pt->memory);108 nvkm_done(pt->memory);
106}109}
@@ -258,12 +261,96 @@ gp100_vmm_pd0_unmap(struct nvkm_vmm *vmm, · 151 unmodified lines
258 VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes);261 VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes);
259}262}
260 263
264static void
265gp100_vmm_pd0_pfn_unmap(struct nvkm_vmm *vmm,
266 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
267{
268 struct device *dev = vmm->mmu->subdev.device->dev;
269 dma_addr_t addr;
270
271 nvkm_kmap(pt->memory);
272 while (ptes--) {
273 u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 0);
274 u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 4);
275 u64 data = (u64)datahi << 32 | datalo;
276
277 if ((data & (3ULL << 1)) != 0) {
278 addr = (data >> 8) << 12;
279 dma_unmap_page(dev, addr, 1UL << 21, DMA_BIDIRECTIONAL);
280 }
281 ptei++;
282 }
283 nvkm_done(pt->memory);
284}
285
286static bool
287gp100_vmm_pd0_pfn_clear(struct nvkm_vmm *vmm,
288 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
289{
290 bool dma = false;
291
292 nvkm_kmap(pt->memory);
293 while (ptes--) {
294 u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 0);
295 u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 4);
296 u64 data = (u64)datahi << 32 | datalo;
297
298 if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) {
299 VMM_WO064(pt, vmm, ptei * 16, data & ~BIT_ULL(0));
300 dma = true;
301 }
302 ptei++;
303 }
304 nvkm_done(pt->memory);
305 return dma;
306}
307
308static void
309gp100_vmm_pd0_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
310 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
311{
312 struct device *dev = vmm->mmu->subdev.device->dev;
313 dma_addr_t addr;
314
315 nvkm_kmap(pt->memory);
316 for (; ptes; ptes--, map->pfn++) {
317 u64 data = 0;
318
319 if (!(*map->pfn & NVKM_VMM_PFN_V))
320 continue;
321
322 if (!(*map->pfn & NVKM_VMM_PFN_W))
323 data |= BIT_ULL(6); /* RO. */
324
325 if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) {
326 addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT;
327 addr = dma_map_page(dev, pfn_to_page(addr), 0,
328 1UL << 21, DMA_BIDIRECTIONAL);
329 if (!WARN_ON(dma_mapping_error(dev, addr))) {
330 data |= addr >> 4;
331 data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */
332 data |= BIT_ULL(3); /* VOL. */
333 data |= BIT_ULL(0); /* VALID. */
334 }
335 } else {
336 data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4;
337 data |= BIT_ULL(0); /* VALID. */
338 }
339
340 VMM_WO064(pt, vmm, ptei++ * 16, data);
341 }
342 nvkm_done(pt->memory);
343}
344
261static const struct nvkm_vmm_desc_func345static const struct nvkm_vmm_desc_func
262gp100_vmm_desc_pd0 = {346gp100_vmm_desc_pd0 = {
263 .unmap = gp100_vmm_pd0_unmap,347 .unmap = gp100_vmm_pd0_unmap,
264 .sparse = gp100_vmm_pd0_sparse,348 .sparse = gp100_vmm_pd0_sparse,
265 .pde = gp100_vmm_pd0_pde,349 .pde = gp100_vmm_pd0_pde,
266 .mem = gp100_vmm_pd0_mem,350 .mem = gp100_vmm_pd0_mem,
351 .pfn = gp100_vmm_pd0_pfn,
352 .pfn_clear = gp100_vmm_pd0_pfn_clear,
353 .pfn_unmap = gp100_vmm_pd0_pfn_unmap,
267};354};
268 355
269static void356static void
include/linux/hmm.h+22−2
@@ -37,16 +37,17 @@ · 36 unmodified lines
37 * will fail. Must be combined with HMM_PFN_REQ_FAULT.37 * will fail. Must be combined with HMM_PFN_REQ_FAULT.
38 */38 */
39enum hmm_pfn_flags {39enum hmm_pfn_flags {
40 /* Output flags */40 /* Output fields and flags */
41 HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),41 HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
42 HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),42 HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
43 HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),43 HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
44 HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
44 45
45 /* Input flags */46 /* Input flags */
46 HMM_PFN_REQ_FAULT = HMM_PFN_VALID,47 HMM_PFN_REQ_FAULT = HMM_PFN_VALID,
47 HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,48 HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,
48 49
49 HMM_PFN_FLAGS = HMM_PFN_VALID | HMM_PFN_WRITE | HMM_PFN_ERROR,50 HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,
50};51};
51 52
52/*53/*
@@ -61,6 +62,25 @@ static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn) · 8 unmodified lines
61 return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);62 return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);
62}63}
63 64
65/*
66 * hmm_pfn_to_map_order() - return the CPU mapping size order
67 *
68 * This is optionally useful to optimize processing of the pfn result
69 * array. It indicates that the page starts at the order aligned VA and is
70 * 1<<order bytes long. Every pfn within an high order page will have the
71 * same pfn flags, both access protections and the map_order. The caller must
72 * be careful with edge cases as the start and end VA of the given page may
73 * extend past the range used with hmm_range_fault().
74 *
75 * This must be called under the caller 'user_lock' after a successful
76 * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
77 * already.
78 */
79static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn)
80{
81 return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F;
82}
83
64/*84/*
65 * struct hmm_range - track invalidation lock on virtual address range85 * struct hmm_range - track invalidation lock on virtual address range
66 *86 *
include/linux/migrate.h+12−4
@@ -180,6 +180,11 @@ static inline unsigned long migrate_pfn(unsigned long pfn) · 179 unmodified lines
180 return (pfn << MIGRATE_PFN_SHIFT) | MIGRATE_PFN_VALID;180 return (pfn << MIGRATE_PFN_SHIFT) | MIGRATE_PFN_VALID;
181}181}
182 182
183enum migrate_vma_direction {
184 MIGRATE_VMA_SELECT_SYSTEM = 1 << 0,
185 MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1,
186};
187
183struct migrate_vma {188struct migrate_vma {
184 struct vm_area_struct *vma;189 struct vm_area_struct *vma;
185 /*190 /*
@@ -199,11 +204,14 @@ struct migrate_vma { · 13 unmodified lines
199 204
200 /*205 /*
201 * Set to the owner value also stored in page->pgmap->owner for206 * Set to the owner value also stored in page->pgmap->owner for
202 * migrating out of device private memory. If set only device207 * migrating out of device private memory. The flags also need to
203 * private pages with this owner are migrated. If not set208 * be set to MIGRATE_VMA_SELECT_DEVICE_PRIVATE.
204 * device private pages are not migrated at all.209 * The caller should always set this field when using mmu notifier
210 * callbacks to avoid device MMU invalidations for device private
211 * pages that are not being migrated.
205 */212 */
206 void *src_owner;213 void *pgmap_owner;
214 unsigned long flags;
207};215};
208 216
209int migrate_vma_setup(struct migrate_vma *args);217int migrate_vma_setup(struct migrate_vma *args);
include/linux/mmu_notifier.h+6−0
@@ -38,6 +38,10 @@ struct mmu_interval_notifier; · 37 unmodified lines
38 *38 *
39 * @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal39 * @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal
40 * that the mm refcount is zero and the range is no longer accessible.40 * that the mm refcount is zero and the range is no longer accessible.
41 *
42 * @MMU_NOTIFY_MIGRATE: used during migrate_vma_collect() invalidate to signal
43 * a device driver to possibly ignore the invalidation if the
44 * migrate_pgmap_owner field matches the driver's device private pgmap owner.
41 */45 */
42enum mmu_notifier_event {46enum mmu_notifier_event {
43 MMU_NOTIFY_UNMAP = 0,47 MMU_NOTIFY_UNMAP = 0,
@@ -46,6 +50,7 @@ enum mmu_notifier_event { · 2 unmodified lines
46 MMU_NOTIFY_PROTECTION_PAGE,50 MMU_NOTIFY_PROTECTION_PAGE,
47 MMU_NOTIFY_SOFT_DIRTY,51 MMU_NOTIFY_SOFT_DIRTY,
48 MMU_NOTIFY_RELEASE,52 MMU_NOTIFY_RELEASE,
53 MMU_NOTIFY_MIGRATE,
49};54};
50 55
51#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)56#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
@@ -264,6 +269,7 @@ struct mmu_notifier_range { · 212 unmodified lines
264 unsigned long end;269 unsigned long end;
265 unsigned flags;270 unsigned flags;
266 enum mmu_notifier_event event;271 enum mmu_notifier_event event;
272 void *migrate_pgmap_owner;
267};273};
268 274
269static inline int mm_has_notifiers(struct mm_struct *mm)275static inline int mm_has_notifiers(struct mm_struct *mm)
lib/test_hmm.c+24−23
@@ -214,6 +214,14 @@ static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, · 213 unmodified lines
214{214{
215 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);215 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
216 216
217 /*
218 * Ignore invalidation callbacks for device private pages since
219 * the invalidation is handled as part of the migration process.
220 */
221 if (range->event == MMU_NOTIFY_MIGRATE &&
222 range->migrate_pgmap_owner == dmirror->mdevice)
223 return true;
224
217 if (mmu_notifier_range_blockable(range))225 if (mmu_notifier_range_blockable(range))
218 mutex_lock(&dmirror->mutex);226 mutex_lock(&dmirror->mutex);
219 else if (!mutex_trylock(&dmirror->mutex))227 else if (!mutex_trylock(&dmirror->mutex))
@@ -585,15 +593,6 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, · 365 unmodified lines
585 */593 */
586 spage = migrate_pfn_to_page(*src);594 spage = migrate_pfn_to_page(*src);
587 595
588 /*
589 * Don't migrate device private pages from our own driver or
590 * others. For our own we would do a device private memory copy
591 * not a migration and for others, we would need to fault the
592 * other device's page into system memory first.
593 */
594 if (spage && is_zone_device_page(spage))
595 continue;
596
597 dpage = dmirror_devmem_alloc_page(mdevice);596 dpage = dmirror_devmem_alloc_page(mdevice);
598 if (!dpage)597 if (!dpage)
599 continue;598 continue;
@@ -702,7 +701,8 @@ static int dmirror_migrate(struct dmirror *dmirror, · 102 unmodified lines
702 args.dst = dst_pfns;701 args.dst = dst_pfns;
703 args.start = addr;702 args.start = addr;
704 args.end = next;703 args.end = next;
705 args.src_owner = NULL;704 args.pgmap_owner = dmirror->mdevice;
705 args.flags = MIGRATE_VMA_SELECT_SYSTEM;
706 ret = migrate_vma_setup(&args);706 ret = migrate_vma_setup(&args);
707 if (ret)707 if (ret)
708 goto out;708 goto out;
@@ -766,6 +766,10 @@ static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, · 57 unmodified lines
766 *perm |= HMM_DMIRROR_PROT_WRITE;766 *perm |= HMM_DMIRROR_PROT_WRITE;
767 else767 else
768 *perm |= HMM_DMIRROR_PROT_READ;768 *perm |= HMM_DMIRROR_PROT_READ;
769 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
770 *perm |= HMM_DMIRROR_PROT_PMD;
771 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
772 *perm |= HMM_DMIRROR_PROT_PUD;
769}773}
770 774
771static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,775static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
@@ -987,7 +991,7 @@ static void dmirror_devmem_free(struct page *page) · 215 unmodified lines
987}991}
988 992
989static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,993static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
990 struct dmirror_device *mdevice)994 struct dmirror *dmirror)
991{995{
992 const unsigned long *src = args->src;996 const unsigned long *src = args->src;
993 unsigned long *dst = args->dst;997 unsigned long *dst = args->dst;
@@ -1009,6 +1013,7 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, · 15 unmodified lines
1009 continue;1013 continue;
1010 1014
1011 lock_page(dpage);1015 lock_page(dpage);
1016 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
1012 copy_highpage(dpage, spage);1017 copy_highpage(dpage, spage);
1013 *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;1018 *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
1014 if (*src & MIGRATE_PFN_WRITE)1019 if (*src & MIGRATE_PFN_WRITE)
@@ -1017,15 +1022,6 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, · 2 unmodified lines
1017 return 0;1022 return 0;
1018}1023}
1019 1024
1020static void dmirror_devmem_fault_finalize_and_map(struct migrate_vma *args,
1021 struct dmirror *dmirror)
1022{
1023 /* Invalidate the device's page table mapping. */
1024 mutex_lock(&dmirror->mutex);
1025 dmirror_do_update(dmirror, args->start, args->end);
1026 mutex_unlock(&dmirror->mutex);
1027}
1028
1029static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)1025static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
1030{1026{
1031 struct migrate_vma args;1027 struct migrate_vma args;
@@ -1049,16 +1045,21 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) · 17 unmodified lines
1049 args.end = args.start + PAGE_SIZE;1045 args.end = args.start + PAGE_SIZE;
1050 args.src = &src_pfns;1046 args.src = &src_pfns;
1051 args.dst = &dst_pfns;1047 args.dst = &dst_pfns;
1052 args.src_owner = dmirror->mdevice;1048 args.pgmap_owner = dmirror->mdevice;
1049 args.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
1053 1050
1054 if (migrate_vma_setup(&args))1051 if (migrate_vma_setup(&args))
1055 return VM_FAULT_SIGBUS;1052 return VM_FAULT_SIGBUS;
1056 1053
1057 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror->mdevice);1054 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
1058 if (ret)1055 if (ret)
1059 return ret;1056 return ret;
1060 migrate_vma_pages(&args);1057 migrate_vma_pages(&args);
1061 dmirror_devmem_fault_finalize_and_map(&args, dmirror);1058 /*
1059 * No device finalize step is needed since
1060 * dmirror_devmem_fault_alloc_and_copy() will have already
1061 * invalidated the device page table.
1062 */
1062 migrate_vma_finalize(&args);1063 migrate_vma_finalize(&args);
1063 return 0;1064 return 0;
1064}1065}
lib/test_hmm_uapi.h+4−0
@@ -40,6 +40,8 @@ struct hmm_dmirror_cmd { · 39 unmodified lines
40 * HMM_DMIRROR_PROT_NONE: unpopulated PTE or PTE with no access40 * HMM_DMIRROR_PROT_NONE: unpopulated PTE or PTE with no access
41 * HMM_DMIRROR_PROT_READ: read-only PTE41 * HMM_DMIRROR_PROT_READ: read-only PTE
42 * HMM_DMIRROR_PROT_WRITE: read/write PTE42 * HMM_DMIRROR_PROT_WRITE: read/write PTE
43 * HMM_DMIRROR_PROT_PMD: PMD sized page is fully mapped by same permissions
44 * HMM_DMIRROR_PROT_PUD: PUD sized page is fully mapped by same permissions
43 * HMM_DMIRROR_PROT_ZERO: special read-only zero page45 * HMM_DMIRROR_PROT_ZERO: special read-only zero page
44 * HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL: Migrated device private page on the46 * HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL: Migrated device private page on the
45 * device the ioctl() is made47 * device the ioctl() is made
@@ -51,6 +53,8 @@ enum { · 5 unmodified lines
51 HMM_DMIRROR_PROT_NONE = 0x00,53 HMM_DMIRROR_PROT_NONE = 0x00,
52 HMM_DMIRROR_PROT_READ = 0x01,54 HMM_DMIRROR_PROT_READ = 0x01,
53 HMM_DMIRROR_PROT_WRITE = 0x02,55 HMM_DMIRROR_PROT_WRITE = 0x02,
56 HMM_DMIRROR_PROT_PMD = 0x04,
57 HMM_DMIRROR_PROT_PUD = 0x08,
54 HMM_DMIRROR_PROT_ZERO = 0x10,58 HMM_DMIRROR_PROT_ZERO = 0x10,
55 HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL = 0x20,59 HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL = 0x20,
56 HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE = 0x30,60 HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE = 0x30,
mm/hmm.c+13−3
@@ -165,12 +165,19 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, · 164 unmodified lines
165 return hmm_pfns_fill(addr, end, range, 0);165 return hmm_pfns_fill(addr, end, range, 0);
166}166}
167 167
168static inline unsigned long hmm_pfn_flags_order(unsigned long order)
169{
170 return order << HMM_PFN_ORDER_SHIFT;
171}
172
168static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range,173static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range,
169 pmd_t pmd)174 pmd_t pmd)
170{175{
171 if (pmd_protnone(pmd))176 if (pmd_protnone(pmd))
172 return 0;177 return 0;
173 return pmd_write(pmd) ? (HMM_PFN_VALID | HMM_PFN_WRITE) : HMM_PFN_VALID;178 return (pmd_write(pmd) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :
179 HMM_PFN_VALID) |
180 hmm_pfn_flags_order(PMD_SHIFT - PAGE_SHIFT);
174}181}
175 182
176#ifdef CONFIG_TRANSPARENT_HUGEPAGE183#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -389,7 +396,9 @@ static inline unsigned long pud_to_hmm_pfn_flags(struct hmm_range *range, · 212 unmodified lines
389{396{
390 if (!pud_present(pud))397 if (!pud_present(pud))
391 return 0;398 return 0;
392 return pud_write(pud) ? (HMM_PFN_VALID | HMM_PFN_WRITE) : HMM_PFN_VALID;399 return (pud_write(pud) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :
400 HMM_PFN_VALID) |
401 hmm_pfn_flags_order(PUD_SHIFT - PAGE_SHIFT);
393}402}
394 403
395static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,404static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
@@ -474,7 +483,8 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, · 78 unmodified lines
474 483
475 i = (start - range->start) >> PAGE_SHIFT;484 i = (start - range->start) >> PAGE_SHIFT;
476 pfn_req_flags = range->hmm_pfns[i];485 pfn_req_flags = range->hmm_pfns[i];
477 cpu_flags = pte_to_hmm_pfn_flags(range, entry);486 cpu_flags = pte_to_hmm_pfn_flags(range, entry) |
487 hmm_pfn_flags_order(huge_page_order(hstate_vma(vma)));
478 required_fault =488 required_fault =
479 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);489 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);
480 if (required_fault) {490 if (required_fault) {
mm/migrate.c+11−3
@@ -2276,7 +2276,9 @@ again: · 2275 unmodified lines
2276 goto next;2276 goto next;
2277 2277
2278 page = device_private_entry_to_page(entry);2278 page = device_private_entry_to_page(entry);
2279 if (page->pgmap->owner != migrate->src_owner)2279 if (!(migrate->flags &
2280 MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
2281 page->pgmap->owner != migrate->pgmap_owner)
2280 goto next;2282 goto next;
2281 2283
2282 mpfn = migrate_pfn(page_to_pfn(page)) |2284 mpfn = migrate_pfn(page_to_pfn(page)) |
@@ -2284,7 +2286,7 @@ again: · 1 unmodified line
2284 if (is_write_device_private_entry(entry))2286 if (is_write_device_private_entry(entry))
2285 mpfn |= MIGRATE_PFN_WRITE;2287 mpfn |= MIGRATE_PFN_WRITE;
2286 } else {2288 } else {
2287 if (migrate->src_owner)2289 if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
2288 goto next;2290 goto next;
2289 pfn = pte_pfn(pte);2291 pfn = pte_pfn(pte);
2290 if (is_zero_pfn(pfn)) {2292 if (is_zero_pfn(pfn)) {
@@ -2379,8 +2381,14 @@ static void migrate_vma_collect(struct migrate_vma *migrate) · 88 unmodified lines
2379{2381{
2380 struct mmu_notifier_range range;2382 struct mmu_notifier_range range;
2381 2383
2382 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL,2384 /*
2385 * Note that the pgmap_owner is passed to the mmu notifier callback so
2386 * that the registered device driver can skip invalidating device
2387 * private page mappings that won't be migrated.
2388 */
2389 mmu_notifier_range_init(&range, MMU_NOTIFY_MIGRATE, 0, migrate->vma,
2383 migrate->vma->vm_mm, migrate->start, migrate->end);2390 migrate->vma->vm_mm, migrate->start, migrate->end);
2391 range.migrate_pgmap_owner = migrate->pgmap_owner;
2384 mmu_notifier_invalidate_range_start(&range);2392 mmu_notifier_invalidate_range_start(&range);
2385 2393
2386 walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,2394 walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
tools/testing/selftests/vm/hmm-tests.c+90−4
@@ -881,8 +881,9 @@ TEST_F(hmm, migrate) · 880 unmodified lines
881}881}
882 882
883/*883/*
884 * Migrate anonymous memory to device private memory and fault it back to system884 * Migrate anonymous memory to device private memory and fault some of it back
885 * memory.885 * to system memory, then try migrating the resulting mix of system and device
886 * private memory to the device.
886 */887 */
887TEST_F(hmm, migrate_fault)888TEST_F(hmm, migrate_fault)
888{889{
@@ -924,8 +925,17 @@ TEST_F(hmm, migrate_fault) · 35 unmodified lines
924 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)925 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
925 ASSERT_EQ(ptr[i], i);926 ASSERT_EQ(ptr[i], i);
926 927
927 /* Fault pages back to system memory and check them. */928 /* Fault half the pages back to system memory and check them. */
928 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)929 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
930 ASSERT_EQ(ptr[i], i);
931
932 /* Migrate memory to the device again. */
933 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages);
934 ASSERT_EQ(ret, 0);
935 ASSERT_EQ(buffer->cpages, npages);
936
937 /* Check what the device read. */
938 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
929 ASSERT_EQ(ptr[i], i);939 ASSERT_EQ(ptr[i], i);
930 940
931 hmm_buffer_free(buffer);941 hmm_buffer_free(buffer);
@@ -1291,6 +1301,82 @@ TEST_F(hmm2, snapshot) · 359 unmodified lines
1291 hmm_buffer_free(buffer);1301 hmm_buffer_free(buffer);
1292}1302}
1293 1303
1304/*
1305 * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that
1306 * should be mapped by a large page table entry.
1307 */
1308TEST_F(hmm, compound)
1309{
1310 struct hmm_buffer *buffer;
1311 unsigned long npages;
1312 unsigned long size;
1313 int *ptr;
1314 unsigned char *m;
1315 int ret;
1316 long pagesizes[4];
1317 int n, idx;
1318 unsigned long i;
1319
1320 /* Skip test if we can't allocate a hugetlbfs page. */
1321
1322 n = gethugepagesizes(pagesizes, 4);
1323 if (n <= 0)
1324 return;
1325 for (idx = 0; --n > 0; ) {
1326 if (pagesizes[n] < pagesizes[idx])
1327 idx = n;
1328 }
1329 size = ALIGN(TWOMEG, pagesizes[idx]);
1330 npages = size >> self->page_shift;
1331
1332 buffer = malloc(sizeof(*buffer));
1333 ASSERT_NE(buffer, NULL);
1334
1335 buffer->ptr = get_hugepage_region(size, GHR_STRICT);
1336 if (buffer->ptr == NULL) {
1337 free(buffer);
1338 return;
1339 }
1340
1341 buffer->size = size;
1342 buffer->mirror = malloc(npages);
1343 ASSERT_NE(buffer->mirror, NULL);
1344
1345 /* Initialize the pages the device will snapshot in buffer->ptr. */
1346 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1347 ptr[i] = i;
1348
1349 /* Simulate a device snapshotting CPU pagetables. */
1350 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1351 ASSERT_EQ(ret, 0);
1352 ASSERT_EQ(buffer->cpages, npages);
1353
1354 /* Check what the device saw. */
1355 m = buffer->mirror;
1356 for (i = 0; i < npages; ++i)
1357 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE |
1358 HMM_DMIRROR_PROT_PMD);
1359
1360 /* Make the region read-only. */
1361 ret = mprotect(buffer->ptr, size, PROT_READ);
1362 ASSERT_EQ(ret, 0);
1363
1364 /* Simulate a device snapshotting CPU pagetables. */
1365 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1366 ASSERT_EQ(ret, 0);
1367 ASSERT_EQ(buffer->cpages, npages);
1368
1369 /* Check what the device saw. */
1370 m = buffer->mirror;
1371 for (i = 0; i < npages; ++i)
1372 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ |
1373 HMM_DMIRROR_PROT_PMD);
1374
1375 free_hugepage_region(buffer->ptr);
1376 buffer->ptr = NULL;
1377 hmm_buffer_free(buffer);
1378}
1379
1294/*1380/*
1295 * Test two devices reading the same memory (double mapped).1381 * Test two devices reading the same memory (double mapped).
1296 */1382 */