Merge tag 'for-linus-hmm' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull hmm updates from Jason Gunthorpe: "Ralph has been working on nouveau's use of hmm_range_fault() and migrate_vma() which resulted in this small series. It adds reporting of the page table order from hmm_range_fault() and some optimization of migrate_vma(): - Report the size of the page table mapping out of hmm_range_fault(). This makes it easier to establish a large/huge/etc mapping in the device's page table. - Allow devices to ignore the invalidations during migration in cases where the migration is not going to change pages. For instance migrating pages to a device does not require the device to invalidate pages already in the device. - Update nouveau and hmm_tests to use the above" * tag 'for-linus-hmm' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: mm/hmm/test: use the new migration invalidation nouveau/svm: use the new migration invalidation mm/notifier: add migration invalidation type mm/migrate: add a flags parameter to migrate_vma nouveau: fix storing invalid ptes nouveau/hmm: support mapping large sysmem pages nouveau: fix mapping 2MB sysmem pages nouveau/hmm: fault one page at a time mm/hmm: add tests for hmm_pfn_to_map_order() mm/hmm: provide the page mapping order in hmm_range_fault()

Linus Torvalds committed 6 years ago

Commit fffe3ae2 parents 8f7be62 7d17e8314 files · +410 −200Browse files

Unified · Split

14 files

arch/powerpc/kvm/book3s_hv_uvmem.c +3 −1
drivers/gpu/drm/nouveau/nouveau_dmem.c +15 −4
drivers/gpu/drm/nouveau/nouveau_svm.c +107 −150
drivers/gpu/drm/nouveau/nouveau_svm.h +12 −1
drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +2 −3
drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c +89 −2
include/linux/hmm.h +22 −2
include/linux/migrate.h +12 −4
include/linux/mmu_notifier.h +6 −0
lib/test_hmm.c +24 −23
lib/test_hmm_uapi.h +4 −0
mm/hmm.c +13 −3
mm/migrate.c +11 −3
tools/testing/selftests/vm/hmm-tests.c +90 −4

arch/powerpc/kvm/book3s_hv_uvmem.c+3−1

@@ -400,6 +400,7 @@ kvmppc_svm_page_in(struct vm_area_struct *vma, unsigned long start, · 399 unmodified lines
400	400	mig.end = end;
401	401	mig.src = &src_pfn;
402	402	mig.dst = &dst_pfn;
	403	+ mig.flags = MIGRATE_VMA_SELECT_SYSTEM;
403	404
404	405	/*
405	406	* We come here with mmap_lock write lock held just for
@@ -577,7 +578,8 @@ kvmppc_svm_page_out(struct vm_area_struct *vma, unsigned long start, · 171 unmodified lines
577	578	mig.end = end;
578	579	mig.src = &src_pfn;
579	580	mig.dst = &dst_pfn;
580		− mig.src_owner = &kvmppc_uvmem_pgmap;
	581	+ mig.pgmap_owner = &kvmppc_uvmem_pgmap;
	582	+ mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
581	583
582	584	mutex_lock(&kvm->arch.uvmem_lock);
583	585	/* The requested page is already paged-out, nothing to do */

drivers/gpu/drm/nouveau/nouveau_dmem.c+15−4

@@ -140,6 +140,7 @@ static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, · 139 unmodified lines
140	140	{
141	141	struct device *dev = drm->dev->dev;
142	142	struct page dpage, spage;
	143	+ struct nouveau_svmm *svmm;
143	144
144	145	spage = migrate_pfn_to_page(args->src[0]);
145	146	if (!spage \|\| !(args->src[0] & MIGRATE_PFN_MIGRATE))
@@ -154,14 +155,19 @@ static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, · 8 unmodified lines
154	155	if (dma_mapping_error(dev, *dma_addr))
155	156	goto error_free_page;
156	157
	158	+ svmm = spage->zone_device_data;
	159	+ mutex_lock(&svmm->mutex);
	160	+ nouveau_svmm_invalidate(svmm, args->start, args->end);
157	161	if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr,
158	162	NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage)))
159	163	goto error_dma_unmap;
	164	+ mutex_unlock(&svmm->mutex);
160	165
161	166	args->dst[0] = migrate_pfn(page_to_pfn(dpage)) \| MIGRATE_PFN_LOCKED;
162	167	return 0;
163	168
164	169	error_dma_unmap:
	170	+ mutex_unlock(&svmm->mutex);
165	171	dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
166	172	error_free_page:
167	173	__free_page(dpage);
@@ -182,7 +188,8 @@ static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) · 14 unmodified lines
182	188	.end = vmf->address + PAGE_SIZE,
183	189	.src = &src,
184	190	.dst = &dst,
185		− .src_owner = drm->dev,
	191	+ .pgmap_owner = drm->dev,
	192	+ .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE,
186	193	};
187	194
188	195	/*
@@ -530,7 +537,8 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 341 unmodified lines
530	537	}
531	538
532	539	static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm,
533		− unsigned long src, dma_addr_t dma_addr, u64 pfn)
	540	+ struct nouveau_svmm *svmm, unsigned long src,
	541	+ dma_addr_t dma_addr, u64 pfn)
534	542	{
535	543	struct device *dev = drm->dev->dev;
536	544	struct page dpage, spage;
@@ -560,6 +568,7 @@ static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm, · 23 unmodified lines
560	568	goto out_free_page;
561	569	}
562	570
	571	+ dpage->zone_device_data = svmm;
563	572	*pfn = NVIF_VMM_PFNMAP_V0_V \| NVIF_VMM_PFNMAP_V0_VRAM \|
564	573	((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT);
565	574	if (src & MIGRATE_PFN_WRITE)
@@ -583,8 +592,8 @@ static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm, · 17 unmodified lines
583	592	unsigned long addr = args->start, nr_dma = 0, i;
584	593
585	594	for (i = 0; addr < args->end; i++) {
586		− args->dst[i] = nouveau_dmem_migrate_copy_one(drm, args->src[i],
587		− dma_addrs + nr_dma, pfns + i);
	595	+ args->dst[i] = nouveau_dmem_migrate_copy_one(drm, svmm,
	596	+ args->src[i], dma_addrs + nr_dma, pfns + i);
588	597	if (!dma_mapping_error(drm->dev->dev, dma_addrs[nr_dma]))
589	598	nr_dma++;
590	599	addr += PAGE_SIZE;
@@ -615,6 +624,8 @@ nouveau_dmem_migrate_vma(struct nouveau_drm *drm, · 24 unmodified lines
615	624	struct migrate_vma args = {
616	625	.vma = vma,
617	626	.start = start,
	627	+ .pgmap_owner = drm->dev,
	628	+ .flags = MIGRATE_VMA_SELECT_SYSTEM,
618	629	};
619	630	unsigned long i;
620	631	u64 *pfns;

drivers/gpu/drm/nouveau/nouveau_svm.c+107−150

@@ -93,17 +93,6 @@ nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst) · 92 unmodified lines
93	93	return NULL;
94	94	}
95	95
96		−struct nouveau_svmm {
97		− struct mmu_notifier notifier;
98		− struct nouveau_vmm *vmm;
99		− struct {
100		− unsigned long start;
101		− unsigned long limit;
102		− } unmanaged;
103		−
104		− struct mutex mutex;
105		−};
106		−
107	96	#define SVMM_DBG(s,f,a...) \
108	97	NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a)
109	98	#define SVMM_ERR(s,f,a...) \
@@ -246,7 +235,7 @@ nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst) · 136 unmodified lines
246	235	}
247	236
248	237	/* Invalidate SVMM address-range on GPU. */
249		−static void
	238	+void
250	239	nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit)
251	240	{
252	241	if (limit > start) {
@@ -279,6 +268,14 @@ nouveau_svmm_invalidate_range_start(struct mmu_notifier *mn, · 26 unmodified lines
279	268	if (unlikely(!svmm->vmm))
280	269	goto out;
281	270
	271	+ /*
	272	+ * Ignore invalidation callbacks for device private pages since
	273	+ * the invalidation is handled as part of the migration process.
	274	+ */
	275	+ if (update->event == MMU_NOTIFY_MIGRATE &&
	276	+ update->migrate_pgmap_owner == svmm->vmm->cli->drm->dev)
	277	+ goto out;
	278	+
282	279	if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) {
283	280	if (start < svmm->unmanaged.start) {
284	281	nouveau_svmm_invalidate(svmm, start,
@@ -514,53 +511,68 @@ static const struct mmu_interval_notifier_ops nouveau_svm_mni_ops = { · 229 unmodified lines
514	511	};
515	512
516	513	static void nouveau_hmm_convert_pfn(struct nouveau_drm *drm,
517		− struct hmm_range range, u64 ioctl_addr)
	514	+ struct hmm_range *range,
	515	+ struct nouveau_pfnmap_args *args)
518	516	{
519		− unsigned long i, npages;
	517	+ struct page *page;
520	518
521	519	/*
522		− * The ioctl_addr prepared here is passed through nvif_object_ioctl()
	520	+ * The address prepared here is passed through nvif_object_ioctl()
523	521	* to an eventual DMA map in something like gp100_vmm_pgt_pfn()
524	522	*
525	523	* This is all just encoding the internal hmm representation into a
526	524	* different nouveau internal representation.
527	525	*/
528		− npages = (range->end - range->start) >> PAGE_SHIFT;
529		− for (i = 0; i < npages; ++i) {
530		− struct page *page;
531		−
532		− if (!(range->hmm_pfns[i] & HMM_PFN_VALID)) {
533		− ioctl_addr[i] = 0;
534		− continue;
535		− }
	526	+ if (!(range->hmm_pfns[0] & HMM_PFN_VALID)) {
	527	+ args->p.phys[0] = 0;
	528	+ return;
	529	+ }
536	530
537		− page = hmm_pfn_to_page(range->hmm_pfns[i]);
538		− if (is_device_private_page(page))
539		− ioctl_addr[i] = nouveau_dmem_page_addr(page) \|
540		− NVIF_VMM_PFNMAP_V0_V \|
541		− NVIF_VMM_PFNMAP_V0_VRAM;
542		− else
543		− ioctl_addr[i] = page_to_phys(page) \|
544		− NVIF_VMM_PFNMAP_V0_V \|
545		− NVIF_VMM_PFNMAP_V0_HOST;
546		− if (range->hmm_pfns[i] & HMM_PFN_WRITE)
547		− ioctl_addr[i] \|= NVIF_VMM_PFNMAP_V0_W;
	531	+ page = hmm_pfn_to_page(range->hmm_pfns[0]);
	532	+ /*
	533	+ * Only map compound pages to the GPU if the CPU is also mapping the
	534	+ * page as a compound page. Otherwise, the PTE protections might not be
	535	+ * consistent (e.g., CPU only maps part of a compound page).
	536	+ * Note that the underlying page might still be larger than the
	537	+ * CPU mapping (e.g., a PUD sized compound page partially mapped with
	538	+ * a PMD sized page table entry).
	539	+ */
	540	+ if (hmm_pfn_to_map_order(range->hmm_pfns[0])) {
	541	+ unsigned long addr = args->p.addr;
	542	+
	543	+ args->p.page = hmm_pfn_to_map_order(range->hmm_pfns[0]) +
	544	+ PAGE_SHIFT;
	545	+ args->p.size = 1UL << args->p.page;
	546	+ args->p.addr &= ~(args->p.size - 1);
	547	+ page -= (addr - args->p.addr) >> PAGE_SHIFT;
548	548	}
	549	+ if (is_device_private_page(page))
	550	+ args->p.phys[0] = nouveau_dmem_page_addr(page) \|
	551	+ NVIF_VMM_PFNMAP_V0_V \|
	552	+ NVIF_VMM_PFNMAP_V0_VRAM;
	553	+ else
	554	+ args->p.phys[0] = page_to_phys(page) \|
	555	+ NVIF_VMM_PFNMAP_V0_V \|
	556	+ NVIF_VMM_PFNMAP_V0_HOST;
	557	+ if (range->hmm_pfns[0] & HMM_PFN_WRITE)
	558	+ args->p.phys[0] \|= NVIF_VMM_PFNMAP_V0_W;
549	559	}
550	560
551	561	static int nouveau_range_fault(struct nouveau_svmm *svmm,
552		− struct nouveau_drm drm, void data, u32 size,
553		− unsigned long hmm_pfns[], u64 *ioctl_addr,
	562	+ struct nouveau_drm *drm,
	563	+ struct nouveau_pfnmap_args *args, u32 size,
	564	+ unsigned long hmm_flags,
554	565	struct svm_notifier *notifier)
555	566	{
556	567	unsigned long timeout =
557	568	jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
558	569	/* Have HMM fault pages within the fault window to the GPU. */
	570	+ unsigned long hmm_pfns[1];
559	571	struct hmm_range range = {
560	572	.notifier = &notifier->notifier,
561	573	.start = notifier->notifier.interval_tree.start,
562	574	.end = notifier->notifier.interval_tree.last + 1,
563		− .pfn_flags_mask = HMM_PFN_REQ_FAULT \| HMM_PFN_REQ_WRITE,
	575	+ .default_flags = hmm_flags,
564	576	.hmm_pfns = hmm_pfns,
565	577	.dev_private_owner = drm->dev,
566	578	};
@@ -576,11 +588,6 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, · 9 unmodified lines
576	588	ret = hmm_range_fault(&range);
577	589	mmap_read_unlock(mm);
578	590	if (ret) {
579		− /*
580		− * FIXME: the input PFN_REQ flags are destroyed on
581		− * -EBUSY, we need to regenerate them, also for the
582		− * other continue below
583		− */
584	591	if (ret == -EBUSY)
585	592	continue;
586	593	return ret;
@@ -595,10 +602,10 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, · 8 unmodified lines
595	602	break;
596	603	}
597	604
598		− nouveau_hmm_convert_pfn(drm, &range, ioctl_addr);
	605	+ nouveau_hmm_convert_pfn(drm, &range, args);
599	606
600	607	svmm->vmm->vmm.object.client->super = true;
601		− ret = nvif_object_ioctl(&svmm->vmm->vmm.object, data, size, NULL);
	608	+ ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL);
602	609	svmm->vmm->vmm.object.client->super = false;
603	610	mutex_unlock(&svmm->mutex);
604	611
@@ -615,17 +622,12 @@ nouveau_svm_fault(struct nvif_notify *notify) · 10 unmodified lines
615	622	struct nvif_object *device = &svm->drm->client.device.object;
616	623	struct nouveau_svmm *svmm;
617	624	struct {
618		− struct {
619		− struct nvif_ioctl_v0 i;
620		− struct nvif_ioctl_mthd_v0 m;
621		− struct nvif_vmm_pfnmap_v0 p;
622		− } i;
623		− u64 phys[16];
	625	+ struct nouveau_pfnmap_args i;
	626	+ u64 phys[1];
624	627	} args;
625		− unsigned long hmm_pfns[ARRAY_SIZE(args.phys)];
626		− struct vm_area_struct *vma;
	628	+ unsigned long hmm_flags;
627	629	u64 inst, start, limit;
628		− int fi, fn, pi, fill;
	630	+ int fi, fn;
629	631	int replay = 0, ret;
630	632
631	633	/* Parse available fault buffer entries into a cache, and update
@@ -692,128 +694,83 @@ nouveau_svm_fault(struct nvif_notify *notify) · 60 unmodified lines
692	694	* window into a single update.
693	695	*/
694	696	start = buffer->fault[fi]->addr;
695		− limit = start + (ARRAY_SIZE(args.phys) << PAGE_SHIFT);
	697	+ limit = start + PAGE_SIZE;
696	698	if (start < svmm->unmanaged.limit)
697	699	limit = min_t(u64, limit, svmm->unmanaged.start);
698		− SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit);
699	700
700		− mm = svmm->notifier.mm;
701		− if (!mmget_not_zero(mm)) {
702		− nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
703		− continue;
704		− }
705		−
706		− /* Intersect fault window with the CPU VMA, cancelling
707		− * the fault if the address is invalid.
	701	+ /*
	702	+ * Prepare the GPU-side update of all pages within the
	703	+ * fault window, determining required pages and access
	704	+ * permissions based on pending faults.
708	705	*/
709		− mmap_read_lock(mm);
710		− vma = find_vma_intersection(mm, start, limit);
711		− if (!vma) {
712		− SVMM_ERR(svmm, "wndw %016llx-%016llx", start, limit);
713		− mmap_read_unlock(mm);
714		− mmput(mm);
715		− nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
716		− continue;
	706	+ args.i.p.addr = start;
	707	+ args.i.p.page = PAGE_SHIFT;
	708	+ args.i.p.size = PAGE_SIZE;
	709	+ /*
	710	+ * Determine required permissions based on GPU fault
	711	+ * access flags.
	712	+ * XXX: atomic?
	713	+ */
	714	+ switch (buffer->fault[fi]->access) {
	715	+ case 0: /* READ. */
	716	+ hmm_flags = HMM_PFN_REQ_FAULT;
	717	+ break;
	718	+ case 3: /* PREFETCH. */
	719	+ hmm_flags = 0;
	720	+ break;
	721	+ default:
	722	+ hmm_flags = HMM_PFN_REQ_FAULT \| HMM_PFN_REQ_WRITE;
	723	+ break;
717	724	}
718		− start = max_t(u64, start, vma->vm_start);
719		− limit = min_t(u64, limit, vma->vm_end);
720		− mmap_read_unlock(mm);
721		− SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit);
722	725
723		− if (buffer->fault[fi]->addr != start) {
724		− SVMM_ERR(svmm, "addr %016llx", buffer->fault[fi]->addr);
725		− mmput(mm);
	726	+ mm = svmm->notifier.mm;
	727	+ if (!mmget_not_zero(mm)) {
726	728	nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
727	729	continue;
728	730	}
729	731
730		− /* Prepare the GPU-side update of all pages within the
731		− * fault window, determining required pages and access
732		− * permissions based on pending faults.
733		− */
734		− args.i.p.page = PAGE_SHIFT;
735		− args.i.p.addr = start;
736		− for (fn = fi, pi = 0;;) {
737		− /* Determine required permissions based on GPU fault
738		− * access flags.
739		− *XXX: atomic?
740		− */
741		− switch (buffer->fault[fn]->access) {
742		− case 0: /* READ. */
743		− hmm_pfns[pi++] = HMM_PFN_REQ_FAULT;
744		− break;
745		− case 3: /* PREFETCH. */
746		− hmm_pfns[pi++] = 0;
747		− break;
748		− default:
749		− hmm_pfns[pi++] = HMM_PFN_REQ_FAULT \|
750		− HMM_PFN_REQ_WRITE;
751		− break;
752		− }
753		− args.i.p.size = pi << PAGE_SHIFT;
	732	+ notifier.svmm = svmm;
	733	+ ret = mmu_interval_notifier_insert(&notifier.notifier, mm,
	734	+ args.i.p.addr, args.i.p.size,
	735	+ &nouveau_svm_mni_ops);
	736	+ if (!ret) {
	737	+ ret = nouveau_range_fault(svmm, svm->drm, &args.i,
	738	+ sizeof(args), hmm_flags, &notifier);
	739	+ mmu_interval_notifier_remove(&notifier.notifier);
	740	+ }
	741	+ mmput(mm);
754	742
	743	+ limit = args.i.p.addr + args.i.p.size;
	744	+ for (fn = fi; ++fn < buffer->fault_nr; ) {
755	745	/* It's okay to skip over duplicate addresses from the
756	746	* same SVMM as faults are ordered by access type such
757	747	* that only the first one needs to be handled.
758	748	*
759	749	* ie. WRITE faults appear first, thus any handling of
760	750	* pending READ faults will already be satisfied.
	751	+ * But if a large page is mapped, make sure subsequent
	752	+ * fault addresses have sufficient access permission.
761	753	*/
762		− while (++fn < buffer->fault_nr &&
763		− buffer->fault[fn]->svmm == svmm &&
764		− buffer->fault[fn ]->addr ==
765		− buffer->fault[fn - 1]->addr);
766		−
767		− /* If the next fault is outside the window, or all GPU
768		− * faults have been dealt with, we're done here.
769		− */
770		− if (fn >= buffer->fault_nr \|\|
771		− buffer->fault[fn]->svmm != svmm \|\|
772		− buffer->fault[fn]->addr >= limit)
	754	+ if (buffer->fault[fn]->svmm != svmm \|\|
	755	+ buffer->fault[fn]->addr >= limit \|\|
	756	+ (buffer->fault[fi]->access == 0 /* READ. */ &&
	757	+ !(args.phys[0] & NVIF_VMM_PFNMAP_V0_V)) \|\|
	758	+ (buffer->fault[fi]->access != 0 /* READ. */ &&
	759	+ buffer->fault[fi]->access != 3 /* PREFETCH. */ &&
	760	+ !(args.phys[0] & NVIF_VMM_PFNMAP_V0_W)))
773	761	break;
774		−
775		− /* Fill in the gap between this fault and the next. */
776		− fill = (buffer->fault[fn ]->addr -
777		− buffer->fault[fn - 1]->addr) >> PAGE_SHIFT;
778		− while (--fill)
779		− hmm_pfns[pi++] = 0;
780	762	}
781	763
782		− SVMM_DBG(svmm, "wndw %016llx-%016llx covering %d fault(s)",
783		− args.i.p.addr,
784		− args.i.p.addr + args.i.p.size, fn - fi);
785		−
786		− notifier.svmm = svmm;
787		− ret = mmu_interval_notifier_insert(&notifier.notifier,
788		− svmm->notifier.mm,
789		− args.i.p.addr, args.i.p.size,
790		− &nouveau_svm_mni_ops);
791		− if (!ret) {
792		− ret = nouveau_range_fault(
793		− svmm, svm->drm, &args,
794		− sizeof(args.i) + pi * sizeof(args.phys[0]),
795		− hmm_pfns, args.phys, &notifier);
796		− mmu_interval_notifier_remove(&notifier.notifier);
797		− }
798		− mmput(mm);
	764	+ /* If handling failed completely, cancel all faults. */
	765	+ if (ret) {
	766	+ while (fi < fn) {
	767	+ struct nouveau_svm_fault *fault =
	768	+ buffer->fault[fi++];
799	769
800		− /* Cancel any faults in the window whose pages didn't manage
801		− * to keep their valid bit, or stay writeable when required.
802		− *
803		− * If handling failed completely, cancel all faults.
804		− */
805		− while (fi < fn) {
806		− struct nouveau_svm_fault *fault = buffer->fault[fi++];
807		− pi = (fault->addr - args.i.p.addr) >> PAGE_SHIFT;
808		− if (ret \|\|
809		− !(args.phys[pi] & NVIF_VMM_PFNMAP_V0_V) \|\|
810		− (!(args.phys[pi] & NVIF_VMM_PFNMAP_V0_W) &&
811		− fault->access != 0 && fault->access != 3)) {
812	770	nouveau_svm_fault_cancel_fault(svm, fault);
813		− continue;
814	771	}
	772	+ } else
815	773	replay++;
816		− }
817	774	}
818	775
819	776	/* Issue fault replay to the GPU. */

drivers/gpu/drm/nouveau/nouveau_svm.h+12−1

@@ -1,11 +1,21 @@
1	1	#ifndef __NOUVEAU_SVM_H__
2	2	#define __NOUVEAU_SVM_H__
3	3	#include <nvif/os.h>
	4	+#include <linux/mmu_notifier.h>
4	5	struct drm_device;
5	6	struct drm_file;
6	7	struct nouveau_drm;
7	8
8		−struct nouveau_svmm;
	9	+struct nouveau_svmm {
	10	+ struct mmu_notifier notifier;
	11	+ struct nouveau_vmm *vmm;
	12	+ struct {
	13	+ unsigned long start;
	14	+ unsigned long limit;
	15	+ } unmanaged;
	16	+
	17	+ struct mutex mutex;
	18	+};
9	19
10	20	#if IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM)
11	21	void nouveau_svm_init(struct nouveau_drm *);
@@ -19,6 +29,7 @@ int nouveau_svmm_join(struct nouveau_svmm *, u64 inst); · 7 unmodified lines
19	29	void nouveau_svmm_part(struct nouveau_svmm *, u64 inst);
20	30	int nouveau_svmm_bind(struct drm_device , void , struct drm_file *);
21	31
	32	+void nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit);
22	33	u64 *nouveau_pfns_alloc(unsigned long npages);
23	34	void nouveau_pfns_free(u64 *pfns);
24	35	void nouveau_pfns_map(struct nouveau_svmm svmm, struct mm_struct mm,

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c+2−3

@@ -1204,7 +1204,6 @@ nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size) · 1203 unmodified lines
1204	1204	/*TODO:
1205	1205	* - Avoid PT readback (for dma_unmap etc), this might end up being dealt
1206	1206	* with inside HMM, which would be a lot nicer for us to deal with.
1207		− * - Multiple page sizes (particularly for huge page support).
1208	1207	* - Support for systems without a 4KiB page size.
1209	1208	*/
1210	1209	int
@@ -1220,8 +1219,8 @@ nvkm_vmm_pfn_map(struct nvkm_vmm vmm, u8 shift, u64 addr, u64 size, u64 pfn) · 9 unmodified lines
1220	1219	/* Only support mapping where the page size of the incoming page
1221	1220	* array matches a page size available for direct mapping.
1222	1221	*/
1223		− while (page->shift && page->shift != shift &&
1224		− page->desc->func->pfn == NULL)
	1222	+ while (page->shift && (page->shift != shift \|\|
	1223	+ page->desc->func->pfn == NULL))
1225	1224	page++;
1226	1225
1227	1226	if (!page->shift \|\| !IS_ALIGNED(addr, 1ULL << shift) \|\|

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c+89−2

@@ -79,8 +79,12 @@ gp100_vmm_pgt_pfn(struct nvkm_vmm vmm, struct nvkm_mmu_pt pt, · 78 unmodified lines
79	79	dma_addr_t addr;
80	80
81	81	nvkm_kmap(pt->memory);
82		− while (ptes--) {
	82	+ for (; ptes; ptes--, map->pfn++) {
83	83	u64 data = 0;
	84	+
	85	+ if (!(*map->pfn & NVKM_VMM_PFN_V))
	86	+ continue;
	87	+
84	88	if (!(*map->pfn & NVKM_VMM_PFN_W))
85	89	data \|= BIT_ULL(6); /* RO. */
86	90
@@ -100,7 +104,6 @@ gp100_vmm_pgt_pfn(struct nvkm_vmm vmm, struct nvkm_mmu_pt pt, · 13 unmodified lines
100	104	}
101	105
102	106	VMM_WO064(pt, vmm, ptei++ * 8, data);
103		− map->pfn++;
104	107	}
105	108	nvkm_done(pt->memory);
106	109	}
@@ -258,12 +261,96 @@ gp100_vmm_pd0_unmap(struct nvkm_vmm *vmm, · 151 unmodified lines
258	261	VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes);
259	262	}
260	263
	264	+static void
	265	+gp100_vmm_pd0_pfn_unmap(struct nvkm_vmm *vmm,
	266	+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
	267	+{
	268	+ struct device *dev = vmm->mmu->subdev.device->dev;
	269	+ dma_addr_t addr;
	270	+
	271	+ nvkm_kmap(pt->memory);
	272	+ while (ptes--) {
	273	+ u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 0);
	274	+ u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 4);
	275	+ u64 data = (u64)datahi << 32 \| datalo;
	276	+
	277	+ if ((data & (3ULL << 1)) != 0) {
	278	+ addr = (data >> 8) << 12;
	279	+ dma_unmap_page(dev, addr, 1UL << 21, DMA_BIDIRECTIONAL);
	280	+ }
	281	+ ptei++;
	282	+ }
	283	+ nvkm_done(pt->memory);
	284	+}
	285	+
	286	+static bool
	287	+gp100_vmm_pd0_pfn_clear(struct nvkm_vmm *vmm,
	288	+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
	289	+{
	290	+ bool dma = false;
	291	+
	292	+ nvkm_kmap(pt->memory);
	293	+ while (ptes--) {
	294	+ u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 0);
	295	+ u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 4);
	296	+ u64 data = (u64)datahi << 32 \| datalo;
	297	+
	298	+ if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) {
	299	+ VMM_WO064(pt, vmm, ptei * 16, data & ~BIT_ULL(0));
	300	+ dma = true;
	301	+ }
	302	+ ptei++;
	303	+ }
	304	+ nvkm_done(pt->memory);
	305	+ return dma;
	306	+}
	307	+
	308	+static void
	309	+gp100_vmm_pd0_pfn(struct nvkm_vmm vmm, struct nvkm_mmu_pt pt,
	310	+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
	311	+{
	312	+ struct device *dev = vmm->mmu->subdev.device->dev;
	313	+ dma_addr_t addr;
	314	+
	315	+ nvkm_kmap(pt->memory);
	316	+ for (; ptes; ptes--, map->pfn++) {
	317	+ u64 data = 0;
	318	+
	319	+ if (!(*map->pfn & NVKM_VMM_PFN_V))
	320	+ continue;
	321	+
	322	+ if (!(*map->pfn & NVKM_VMM_PFN_W))
	323	+ data \|= BIT_ULL(6); /* RO. */
	324	+
	325	+ if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) {
	326	+ addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT;
	327	+ addr = dma_map_page(dev, pfn_to_page(addr), 0,
	328	+ 1UL << 21, DMA_BIDIRECTIONAL);
	329	+ if (!WARN_ON(dma_mapping_error(dev, addr))) {
	330	+ data \|= addr >> 4;
	331	+ data \|= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */
	332	+ data \|= BIT_ULL(3); /* VOL. */
	333	+ data \|= BIT_ULL(0); /* VALID. */
	334	+ }
	335	+ } else {
	336	+ data \|= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4;
	337	+ data \|= BIT_ULL(0); /* VALID. */
	338	+ }
	339	+
	340	+ VMM_WO064(pt, vmm, ptei++ * 16, data);
	341	+ }
	342	+ nvkm_done(pt->memory);
	343	+}
	344	+
261	345	static const struct nvkm_vmm_desc_func
262	346	gp100_vmm_desc_pd0 = {
263	347	.unmap = gp100_vmm_pd0_unmap,
264	348	.sparse = gp100_vmm_pd0_sparse,
265	349	.pde = gp100_vmm_pd0_pde,
266	350	.mem = gp100_vmm_pd0_mem,
	351	+ .pfn = gp100_vmm_pd0_pfn,
	352	+ .pfn_clear = gp100_vmm_pd0_pfn_clear,
	353	+ .pfn_unmap = gp100_vmm_pd0_pfn_unmap,
267	354	};
268	355
269	356	static void

include/linux/hmm.h+22−2

@@ -37,16 +37,17 @@ · 36 unmodified lines
37	37	* will fail. Must be combined with HMM_PFN_REQ_FAULT.
38	38	*/
39	39	enum hmm_pfn_flags {
40		− /* Output flags */
	40	+ /* Output fields and flags */
41	41	HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
42	42	HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
43	43	HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
	44	+ HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
44	45
45	46	/* Input flags */
46	47	HMM_PFN_REQ_FAULT = HMM_PFN_VALID,
47	48	HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,
48	49
49		− HMM_PFN_FLAGS = HMM_PFN_VALID \| HMM_PFN_WRITE \| HMM_PFN_ERROR,
	50	+ HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,
50	51	};
51	52
52	53	/*
@@ -61,6 +62,25 @@ static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn) · 8 unmodified lines
61	62	return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);
62	63	}
63	64
	65	+/*
	66	+ * hmm_pfn_to_map_order() - return the CPU mapping size order
	67	+ *
	68	+ * This is optionally useful to optimize processing of the pfn result
	69	+ * array. It indicates that the page starts at the order aligned VA and is
	70	+ * 1<<order bytes long. Every pfn within an high order page will have the
	71	+ * same pfn flags, both access protections and the map_order. The caller must
	72	+ * be careful with edge cases as the start and end VA of the given page may
	73	+ * extend past the range used with hmm_range_fault().
	74	+ *
	75	+ * This must be called under the caller 'user_lock' after a successful
	76	+ * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
	77	+ * already.
	78	+ */
	79	+static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn)
	80	+{
	81	+ return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F;
	82	+}
	83	+
64	84	/*
65	85	* struct hmm_range - track invalidation lock on virtual address range
66	86	*

include/linux/migrate.h+12−4

@@ -180,6 +180,11 @@ static inline unsigned long migrate_pfn(unsigned long pfn) · 179 unmodified lines
180	180	return (pfn << MIGRATE_PFN_SHIFT) \| MIGRATE_PFN_VALID;
181	181	}
182	182
	183	+enum migrate_vma_direction {
	184	+ MIGRATE_VMA_SELECT_SYSTEM = 1 << 0,
	185	+ MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1,
	186	+};
	187	+
183	188	struct migrate_vma {
184	189	struct vm_area_struct *vma;
185	190	/*
@@ -199,11 +204,14 @@ struct migrate_vma { · 13 unmodified lines
199	204
200	205	/*
201	206	* Set to the owner value also stored in page->pgmap->owner for
202		− * migrating out of device private memory. If set only device
203		− * private pages with this owner are migrated. If not set
204		− * device private pages are not migrated at all.
	207	+ * migrating out of device private memory. The flags also need to
	208	+ * be set to MIGRATE_VMA_SELECT_DEVICE_PRIVATE.
	209	+ * The caller should always set this field when using mmu notifier
	210	+ * callbacks to avoid device MMU invalidations for device private
	211	+ * pages that are not being migrated.
205	212	*/
206		− void *src_owner;
	213	+ void *pgmap_owner;
	214	+ unsigned long flags;
207	215	};
208	216
209	217	int migrate_vma_setup(struct migrate_vma *args);

include/linux/mmu_notifier.h+6−0

@@ -38,6 +38,10 @@ struct mmu_interval_notifier; · 37 unmodified lines
38	38	*
39	39	* @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal
40	40	* that the mm refcount is zero and the range is no longer accessible.
	41	+ *
	42	+ * @MMU_NOTIFY_MIGRATE: used during migrate_vma_collect() invalidate to signal
	43	+ * a device driver to possibly ignore the invalidation if the
	44	+ * migrate_pgmap_owner field matches the driver's device private pgmap owner.
41	45	*/
42	46	enum mmu_notifier_event {
43	47	MMU_NOTIFY_UNMAP = 0,
@@ -46,6 +50,7 @@ enum mmu_notifier_event { · 2 unmodified lines
46	50	MMU_NOTIFY_PROTECTION_PAGE,
47	51	MMU_NOTIFY_SOFT_DIRTY,
48	52	MMU_NOTIFY_RELEASE,
	53	+ MMU_NOTIFY_MIGRATE,
49	54	};
50	55
51	56	#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
@@ -264,6 +269,7 @@ struct mmu_notifier_range { · 212 unmodified lines
264	269	unsigned long end;
265	270	unsigned flags;
266	271	enum mmu_notifier_event event;
	272	+ void *migrate_pgmap_owner;
267	273	};
268	274
269	275	static inline int mm_has_notifiers(struct mm_struct *mm)

lib/test_hmm.c+24−23

@@ -214,6 +214,14 @@ static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, · 213 unmodified lines
214	214	{
215	215	struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
216	216
	217	+ /*
	218	+ * Ignore invalidation callbacks for device private pages since
	219	+ * the invalidation is handled as part of the migration process.
	220	+ */
	221	+ if (range->event == MMU_NOTIFY_MIGRATE &&
	222	+ range->migrate_pgmap_owner == dmirror->mdevice)
	223	+ return true;
	224	+
217	225	if (mmu_notifier_range_blockable(range))
218	226	mutex_lock(&dmirror->mutex);
219	227	else if (!mutex_trylock(&dmirror->mutex))
@@ -585,15 +593,6 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, · 365 unmodified lines
585	593	*/
586	594	spage = migrate_pfn_to_page(*src);
587	595
588		− /*
589		− * Don't migrate device private pages from our own driver or
590		− * others. For our own we would do a device private memory copy
591		− * not a migration and for others, we would need to fault the
592		− * other device's page into system memory first.
593		− */
594		− if (spage && is_zone_device_page(spage))
595		− continue;
596		−
597	596	dpage = dmirror_devmem_alloc_page(mdevice);
598	597	if (!dpage)
599	598	continue;
@@ -702,7 +701,8 @@ static int dmirror_migrate(struct dmirror *dmirror, · 102 unmodified lines
702	701	args.dst = dst_pfns;
703	702	args.start = addr;
704	703	args.end = next;
705		− args.src_owner = NULL;
	704	+ args.pgmap_owner = dmirror->mdevice;
	705	+ args.flags = MIGRATE_VMA_SELECT_SYSTEM;
706	706	ret = migrate_vma_setup(&args);
707	707	if (ret)
708	708	goto out;
@@ -766,6 +766,10 @@ static void dmirror_mkentry(struct dmirror dmirror, struct hmm_range range, · 57 unmodified lines
766	766	*perm \|= HMM_DMIRROR_PROT_WRITE;
767	767	else
768	768	*perm \|= HMM_DMIRROR_PROT_READ;
	769	+ if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
	770	+ *perm \|= HMM_DMIRROR_PROT_PMD;
	771	+ else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
	772	+ *perm \|= HMM_DMIRROR_PROT_PUD;
769	773	}
770	774
771	775	static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
@@ -987,7 +991,7 @@ static void dmirror_devmem_free(struct page *page) · 215 unmodified lines
987	991	}
988	992
989	993	static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
990		− struct dmirror_device *mdevice)
	994	+ struct dmirror *dmirror)
991	995	{
992	996	const unsigned long *src = args->src;
993	997	unsigned long *dst = args->dst;
@@ -1009,6 +1013,7 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, · 15 unmodified lines
1009	1013	continue;
1010	1014
1011	1015	lock_page(dpage);
	1016	+ xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
1012	1017	copy_highpage(dpage, spage);
1013	1018	*dst = migrate_pfn(page_to_pfn(dpage)) \| MIGRATE_PFN_LOCKED;
1014	1019	if (*src & MIGRATE_PFN_WRITE)
@@ -1017,15 +1022,6 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, · 2 unmodified lines
1017	1022	return 0;
1018	1023	}
1019	1024
1020		−static void dmirror_devmem_fault_finalize_and_map(struct migrate_vma *args,
1021		− struct dmirror *dmirror)
1022		−{
1023		− /* Invalidate the device's page table mapping. */
1024		− mutex_lock(&dmirror->mutex);
1025		− dmirror_do_update(dmirror, args->start, args->end);
1026		− mutex_unlock(&dmirror->mutex);
1027		−}
1028		−
1029	1025	static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
1030	1026	{
1031	1027	struct migrate_vma args;
@@ -1049,16 +1045,21 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) · 17 unmodified lines
1049	1045	args.end = args.start + PAGE_SIZE;
1050	1046	args.src = &src_pfns;
1051	1047	args.dst = &dst_pfns;
1052		− args.src_owner = dmirror->mdevice;
	1048	+ args.pgmap_owner = dmirror->mdevice;
	1049	+ args.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
1053	1050
1054	1051	if (migrate_vma_setup(&args))
1055	1052	return VM_FAULT_SIGBUS;
1056	1053
1057		− ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror->mdevice);
	1054	+ ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
1058	1055	if (ret)
1059	1056	return ret;
1060	1057	migrate_vma_pages(&args);
1061		− dmirror_devmem_fault_finalize_and_map(&args, dmirror);
	1058	+ /*
	1059	+ * No device finalize step is needed since
	1060	+ * dmirror_devmem_fault_alloc_and_copy() will have already
	1061	+ * invalidated the device page table.
	1062	+ */
1062	1063	migrate_vma_finalize(&args);
1063	1064	return 0;
1064	1065	}

lib/test_hmm_uapi.h+4−0

@@ -40,6 +40,8 @@ struct hmm_dmirror_cmd { · 39 unmodified lines
40	40	* HMM_DMIRROR_PROT_NONE: unpopulated PTE or PTE with no access
41	41	* HMM_DMIRROR_PROT_READ: read-only PTE
42	42	* HMM_DMIRROR_PROT_WRITE: read/write PTE
	43	+ * HMM_DMIRROR_PROT_PMD: PMD sized page is fully mapped by same permissions
	44	+ * HMM_DMIRROR_PROT_PUD: PUD sized page is fully mapped by same permissions
43	45	* HMM_DMIRROR_PROT_ZERO: special read-only zero page
44	46	* HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL: Migrated device private page on the
45	47	* device the ioctl() is made
@@ -51,6 +53,8 @@ enum { · 5 unmodified lines
51	53	HMM_DMIRROR_PROT_NONE = 0x00,
52	54	HMM_DMIRROR_PROT_READ = 0x01,
53	55	HMM_DMIRROR_PROT_WRITE = 0x02,
	56	+ HMM_DMIRROR_PROT_PMD = 0x04,
	57	+ HMM_DMIRROR_PROT_PUD = 0x08,
54	58	HMM_DMIRROR_PROT_ZERO = 0x10,
55	59	HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL = 0x20,
56	60	HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE = 0x30,

mm/hmm.c+13−3

@@ -165,12 +165,19 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, · 164 unmodified lines
165	165	return hmm_pfns_fill(addr, end, range, 0);
166	166	}
167	167
	168	+static inline unsigned long hmm_pfn_flags_order(unsigned long order)
	169	+{
	170	+ return order << HMM_PFN_ORDER_SHIFT;
	171	+}
	172	+
168	173	static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range,
169	174	pmd_t pmd)
170	175	{
171	176	if (pmd_protnone(pmd))
172	177	return 0;
173		− return pmd_write(pmd) ? (HMM_PFN_VALID \| HMM_PFN_WRITE) : HMM_PFN_VALID;
	178	+ return (pmd_write(pmd) ? (HMM_PFN_VALID \| HMM_PFN_WRITE) :
	179	+ HMM_PFN_VALID) \|
	180	+ hmm_pfn_flags_order(PMD_SHIFT - PAGE_SHIFT);
174	181	}
175	182
176	183	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -389,7 +396,9 @@ static inline unsigned long pud_to_hmm_pfn_flags(struct hmm_range *range, · 212 unmodified lines
389	396	{
390	397	if (!pud_present(pud))
391	398	return 0;
392		− return pud_write(pud) ? (HMM_PFN_VALID \| HMM_PFN_WRITE) : HMM_PFN_VALID;
	399	+ return (pud_write(pud) ? (HMM_PFN_VALID \| HMM_PFN_WRITE) :
	400	+ HMM_PFN_VALID) \|
	401	+ hmm_pfn_flags_order(PUD_SHIFT - PAGE_SHIFT);
393	402	}
394	403
395	404	static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
@@ -474,7 +483,8 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, · 78 unmodified lines
474	483
475	484	i = (start - range->start) >> PAGE_SHIFT;
476	485	pfn_req_flags = range->hmm_pfns[i];
477		− cpu_flags = pte_to_hmm_pfn_flags(range, entry);
	486	+ cpu_flags = pte_to_hmm_pfn_flags(range, entry) \|
	487	+ hmm_pfn_flags_order(huge_page_order(hstate_vma(vma)));
478	488	required_fault =
479	489	hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);
480	490	if (required_fault) {

mm/migrate.c+11−3

@@ -2276,7 +2276,9 @@ again: · 2275 unmodified lines
2276	2276	goto next;
2277	2277
2278	2278	page = device_private_entry_to_page(entry);
2279		− if (page->pgmap->owner != migrate->src_owner)
	2279	+ if (!(migrate->flags &
	2280	+ MIGRATE_VMA_SELECT_DEVICE_PRIVATE) \|\|
	2281	+ page->pgmap->owner != migrate->pgmap_owner)
2280	2282	goto next;
2281	2283
2282	2284	mpfn = migrate_pfn(page_to_pfn(page)) \|
@@ -2284,7 +2286,7 @@ again: · 1 unmodified line
2284	2286	if (is_write_device_private_entry(entry))
2285	2287	mpfn \|= MIGRATE_PFN_WRITE;
2286	2288	} else {
2287		− if (migrate->src_owner)
	2289	+ if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
2288	2290	goto next;
2289	2291	pfn = pte_pfn(pte);
2290	2292	if (is_zero_pfn(pfn)) {
@@ -2379,8 +2381,14 @@ static void migrate_vma_collect(struct migrate_vma *migrate) · 88 unmodified lines
2379	2381	{
2380	2382	struct mmu_notifier_range range;
2381	2383
2382		− mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL,
	2384	+ /*
	2385	+ * Note that the pgmap_owner is passed to the mmu notifier callback so
	2386	+ * that the registered device driver can skip invalidating device
	2387	+ * private page mappings that won't be migrated.
	2388	+ */
	2389	+ mmu_notifier_range_init(&range, MMU_NOTIFY_MIGRATE, 0, migrate->vma,
2383	2390	migrate->vma->vm_mm, migrate->start, migrate->end);
	2391	+ range.migrate_pgmap_owner = migrate->pgmap_owner;
2384	2392	mmu_notifier_invalidate_range_start(&range);
2385	2393
2386	2394	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,

tools/testing/selftests/vm/hmm-tests.c+90−4

@@ -881,8 +881,9 @@ TEST_F(hmm, migrate) · 880 unmodified lines
881	881	}
882	882
883	883	/*
884		− * Migrate anonymous memory to device private memory and fault it back to system
885		− * memory.
	884	+ * Migrate anonymous memory to device private memory and fault some of it back
	885	+ * to system memory, then try migrating the resulting mix of system and device
	886	+ * private memory to the device.
886	887	*/
887	888	TEST_F(hmm, migrate_fault)
888	889	{
@@ -924,8 +925,17 @@ TEST_F(hmm, migrate_fault) · 35 unmodified lines
924	925	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
925	926	ASSERT_EQ(ptr[i], i);
926	927
927		− /* Fault pages back to system memory and check them. */
928		− for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
	928	+ /* Fault half the pages back to system memory and check them. */
	929	+ for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
	930	+ ASSERT_EQ(ptr[i], i);
	931	+
	932	+ /* Migrate memory to the device again. */
	933	+ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages);
	934	+ ASSERT_EQ(ret, 0);
	935	+ ASSERT_EQ(buffer->cpages, npages);
	936	+
	937	+ /* Check what the device read. */
	938	+ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
929	939	ASSERT_EQ(ptr[i], i);
930	940
931	941	hmm_buffer_free(buffer);
@@ -1291,6 +1301,82 @@ TEST_F(hmm2, snapshot) · 359 unmodified lines
1291	1301	hmm_buffer_free(buffer);
1292	1302	}
1293	1303
	1304	+/*
	1305	+ * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that
	1306	+ * should be mapped by a large page table entry.
	1307	+ */
	1308	+TEST_F(hmm, compound)
	1309	+{
	1310	+ struct hmm_buffer *buffer;
	1311	+ unsigned long npages;
	1312	+ unsigned long size;
	1313	+ int *ptr;
	1314	+ unsigned char *m;
	1315	+ int ret;
	1316	+ long pagesizes[4];
	1317	+ int n, idx;
	1318	+ unsigned long i;
	1319	+
	1320	+ /* Skip test if we can't allocate a hugetlbfs page. */
	1321	+
	1322	+ n = gethugepagesizes(pagesizes, 4);
	1323	+ if (n <= 0)
	1324	+ return;
	1325	+ for (idx = 0; --n > 0; ) {
	1326	+ if (pagesizes[n] < pagesizes[idx])
	1327	+ idx = n;
	1328	+ }
	1329	+ size = ALIGN(TWOMEG, pagesizes[idx]);
	1330	+ npages = size >> self->page_shift;
	1331	+
	1332	+ buffer = malloc(sizeof(*buffer));
	1333	+ ASSERT_NE(buffer, NULL);
	1334	+
	1335	+ buffer->ptr = get_hugepage_region(size, GHR_STRICT);
	1336	+ if (buffer->ptr == NULL) {
	1337	+ free(buffer);
	1338	+ return;
	1339	+ }
	1340	+
	1341	+ buffer->size = size;
	1342	+ buffer->mirror = malloc(npages);
	1343	+ ASSERT_NE(buffer->mirror, NULL);
	1344	+
	1345	+ /* Initialize the pages the device will snapshot in buffer->ptr. */
	1346	+ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
	1347	+ ptr[i] = i;
	1348	+
	1349	+ /* Simulate a device snapshotting CPU pagetables. */
	1350	+ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
	1351	+ ASSERT_EQ(ret, 0);
	1352	+ ASSERT_EQ(buffer->cpages, npages);
	1353	+
	1354	+ /* Check what the device saw. */
	1355	+ m = buffer->mirror;
	1356	+ for (i = 0; i < npages; ++i)
	1357	+ ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE \|
	1358	+ HMM_DMIRROR_PROT_PMD);
	1359	+
	1360	+ /* Make the region read-only. */
	1361	+ ret = mprotect(buffer->ptr, size, PROT_READ);
	1362	+ ASSERT_EQ(ret, 0);
	1363	+
	1364	+ /* Simulate a device snapshotting CPU pagetables. */
	1365	+ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
	1366	+ ASSERT_EQ(ret, 0);
	1367	+ ASSERT_EQ(buffer->cpages, npages);
	1368	+
	1369	+ /* Check what the device saw. */
	1370	+ m = buffer->mirror;
	1371	+ for (i = 0; i < npages; ++i)
	1372	+ ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ \|
	1373	+ HMM_DMIRROR_PROT_PMD);
	1374	+
	1375	+ free_hugepage_region(buffer->ptr);
	1376	+ buffer->ptr = NULL;
	1377	+ hmm_buffer_free(buffer);
	1378	+}
	1379	+
1294	1380	/*
1295	1381	* Test two devices reading the same memory (double mapped).
1296	1382	*/