Merge branch 'hmm-devmem-cleanup.4' into rdma.git hmm

Christoph Hellwig says: ==================== Below is a series that cleans up the dev_pagemap interface so that it is more easily usable, which removes the need to wrap it in hmm and thus allowing to kill a lot of code Changes since v3: - pull in "mm/swap: Fix release_pages() when releasing devmap pages" and rebase the other patches on top of that - fold the hmm_devmem_add_resource into the DEVICE_PUBLIC memory removal patch - remove _vm_normal_page as it isn't needed without DEVICE_PUBLIC memory - pick up various ACKs Changes since v2: - fix nvdimm kunit build - add a new memory type for device dax - fix a few issues in intermediate patches that didn't show up in the end result - incorporate feedback from Michal Hocko, including killing of the DEVICE_PUBLIC memory type entirely Changes since v1: - rebase - also switch p2pdma to the internal refcount - add type checking for pgmap->type - rename the migrate method to migrate_to_ram - cleanup the altmap_valid flag - various tidbits from the reviews ==================== Conflicts resolved by: - Keeping Ira's version of the code in swap.c - Using the delete for the section in hmm.rst - Using the delete for the devmap code in hmm.c and .h * branch 'hmm-devmem-cleanup.4': (24 commits) mm: don't select MIGRATE_VMA_HELPER from HMM_MIRROR mm: remove the HMM config option mm: sort out the DEVICE_PRIVATE Kconfig mess mm: simplify ZONE_DEVICE page private data mm: remove hmm_devmem_add mm: remove hmm_vma_alloc_locked_page nouveau: use devm_memremap_pages directly nouveau: use alloc_page_vma directly PCI/P2PDMA: use the dev_pagemap internal refcount device-dax: use the dev_pagemap internal refcount memremap: provide an optional internal refcount in struct dev_pagemap memremap: replace the altmap_valid field with a PGMAP_ALTMAP_VALID flag memremap: remove the data field in struct dev_pagemap memremap: add a migrate_to_ram method to struct dev_pagemap_ops memremap: lift the devmap_enable manipulation into devm_memremap_pages memremap: pass a struct dev_pagemap to ->kill and ->cleanup memremap: move dev_pagemap callbacks into a separate structure memremap: validate the pagemap type passed to devm_memremap_pages mm: factor out a devm_request_free_mem_region helper mm: export alloc_pages_vma ...

Jason Gunthorpe committed 7 years ago

Commit cc5dfd52 parents 9ec3f4c b6b346a33 files · +370 −1013Browse files

Unified · Split

33 files

Documentation/vm/hmm.rst +0 −27
arch/powerpc/mm/mem.c +1 −9
arch/x86/mm/init_64.c +2 −6
drivers/dax/dax-private.h +0 −4
drivers/dax/device.c +1 −40
drivers/dax/pmem/core.c +1 −1
drivers/gpu/drm/nouveau/Kconfig +3 −3
drivers/gpu/drm/nouveau/nouveau_dmem.c +47 −56
drivers/nvdimm/pfn_devs.c +1 −2
drivers/nvdimm/pmem.c +19 −32
drivers/pci/p2pdma.c +4 −48
fs/proc/task_mmu.c +1 −1
include/linux/hmm.h +1 −197
include/linux/ioport.h +2 −1
include/linux/memremap.h +49 −26
include/linux/mm.h +2 −26
include/linux/mm_types.h +2 −2
include/linux/swapops.h +0 −15
kernel/memremap.c +118 −76
kernel/resource.c +39 −0
mm/Kconfig +4 −46
mm/Makefile +1 −1
mm/gup.c +0 −7
mm/hmm.c +0 −284
mm/madvise.c +1 −1
mm/memcontrol.c +6 −7
mm/memory-failure.c +1 −5
mm/memory.c +5 −44
mm/memory_hotplug.c +2 −4
mm/mempolicy.c +1 −0
mm/migrate.c +4 −24
mm/page_alloc.c +6 −7
tools/testing/nvdimm/test/iomap.c +46 −11

Documentation/vm/hmm.rst+0−27

@@ -336,33 +336,6 @@ directly using struct page for device memory which left most kernel code paths · 335 unmodified lines
336	336	unaware of the difference. We only need to make sure that no one ever tries to
337	337	map those pages from the CPU side.
338	338
339		−HMM provides a set of helpers to register and hotplug device memory as a new
340		−region needing a struct page. This is offered through a very simple API::
341		−
342		− struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops,
343		− struct device *device,
344		− unsigned long size);
345		− void hmm_devmem_remove(struct hmm_devmem *devmem);
346		−
347		−The hmm_devmem_ops is where most of the important things are::
348		−
349		− struct hmm_devmem_ops {
350		− void (free)(struct hmm_devmem devmem, struct page *page);
351		− vm_fault_t (fault)(struct hmm_devmem devmem,
352		− struct vm_area_struct *vma,
353		− unsigned long addr,
354		− struct page *page,
355		− unsigned flags,
356		− pmd_t *pmdp);
357		− };
358		−
359		−The first callback (free()) happens when the last reference on a device page is
360		−dropped. This means the device page is now free and no longer used by anyone.
361		−The second callback happens whenever the CPU tries to access a device page
362		−which it cannot do. This second callback must trigger a migration back to
363		−system memory.
364		−
365		−
366	339	Migration to and from device memory
367	340	===================================
368	341

arch/powerpc/mm/mem.c+1−9

@@ -131,17 +131,9 @@ void __ref arch_remove_memory(int nid, u64 start, u64 size, · 130 unmodified lines
131	131	{
132	132	unsigned long start_pfn = start >> PAGE_SHIFT;
133	133	unsigned long nr_pages = size >> PAGE_SHIFT;
134		− struct page *page;
	134	+ struct page *page = pfn_to_page(start_pfn) + vmem_altmap_offset(altmap);
135	135	int ret;
136	136
137		− /*
138		− * If we have an altmap then we need to skip over any reserved PFNs
139		− * when querying the zone.
140		− */
141		− page = pfn_to_page(start_pfn);
142		− if (altmap)
143		− page += vmem_altmap_offset(altmap);
144		−
145	137	__remove_pages(page_zone(page), start_pfn, nr_pages, altmap);
146	138
147	139	/* Remove htab bolted mappings for this section of memory */

arch/x86/mm/init_64.c+2−6

@@ -1213,13 +1213,9 @@ void __ref arch_remove_memory(int nid, u64 start, u64 size, · 1212 unmodified lines
1213	1213	{
1214	1214	unsigned long start_pfn = start >> PAGE_SHIFT;
1215	1215	unsigned long nr_pages = size >> PAGE_SHIFT;
1216		− struct page *page = pfn_to_page(start_pfn);
1217		− struct zone *zone;
	1216	+ struct page *page = pfn_to_page(start_pfn) + vmem_altmap_offset(altmap);
	1217	+ struct zone *zone = page_zone(page);
1218	1218
1219		− /* With altmap the first mapped page is offset from @start */
1220		− if (altmap)
1221		− page += vmem_altmap_offset(altmap);
1222		− zone = page_zone(page);
1223	1219	__remove_pages(zone, start_pfn, nr_pages, altmap);
1224	1220	kernel_physical_mapping_remove(start, start + size);
1225	1221	}

drivers/dax/dax-private.h+0−4

@@ -43,8 +43,6 @@ struct dax_region { · 42 unmodified lines
43	43	* @target_node: effective numa node if dev_dax memory range is onlined
44	44	* @dev - device core
45	45	* @pgmap - pgmap for memmap setup / lifetime (driver owned)
46		− * @ref: pgmap reference count (driver owned)
47		− * @cmp: @ref final put completion (driver owned)
48	46	*/
49	47	struct dev_dax {
50	48	struct dax_region *region;
@@ -52,8 +50,6 @@ struct dev_dax { · 1 unmodified line
52	50	int target_node;
53	51	struct device dev;
54	52	struct dev_pagemap pgmap;
55		− struct percpu_ref ref;
56		− struct completion cmp;
57	53	};
58	54
59	55	static inline struct dev_dax to_dev_dax(struct device dev)

drivers/dax/device.c+1−40

@@ -14,37 +14,6 @@ · 13 unmodified lines
14	14	#include "dax-private.h"
15	15	#include "bus.h"
16	16
17		−static struct dev_dax ref_to_dev_dax(struct percpu_ref ref)
18		−{
19		− return container_of(ref, struct dev_dax, ref);
20		−}
21		−
22		−static void dev_dax_percpu_release(struct percpu_ref *ref)
23		−{
24		− struct dev_dax *dev_dax = ref_to_dev_dax(ref);
25		−
26		− dev_dbg(&dev_dax->dev, "%s\n", __func__);
27		− complete(&dev_dax->cmp);
28		−}
29		−
30		−static void dev_dax_percpu_exit(struct percpu_ref *ref)
31		−{
32		− struct dev_dax *dev_dax = ref_to_dev_dax(ref);
33		−
34		− dev_dbg(&dev_dax->dev, "%s\n", __func__);
35		− wait_for_completion(&dev_dax->cmp);
36		− percpu_ref_exit(ref);
37		−}
38		−
39		−static void dev_dax_percpu_kill(struct percpu_ref *data)
40		−{
41		− struct percpu_ref *ref = data;
42		− struct dev_dax *dev_dax = ref_to_dev_dax(ref);
43		−
44		− dev_dbg(&dev_dax->dev, "%s\n", __func__);
45		− percpu_ref_kill(ref);
46		−}
47		−
48	17	static int check_vma(struct dev_dax dev_dax, struct vm_area_struct vma,
49	18	const char *func)
50	19	{
@@ -459,15 +428,7 @@ int dev_dax_probe(struct device *dev) · 408 unmodified lines
459	428	return -EBUSY;
460	429	}
461	430
462		− init_completion(&dev_dax->cmp);
463		− rc = percpu_ref_init(&dev_dax->ref, dev_dax_percpu_release, 0,
464		− GFP_KERNEL);
465		− if (rc)
466		− return rc;
467		−
468		− dev_dax->pgmap.ref = &dev_dax->ref;
469		− dev_dax->pgmap.kill = dev_dax_percpu_kill;
470		− dev_dax->pgmap.cleanup = dev_dax_percpu_exit;
	431	+ dev_dax->pgmap.type = MEMORY_DEVICE_DEVDAX;
471	432	addr = devm_memremap_pages(dev, &dev_dax->pgmap);
472	433	if (IS_ERR(addr))
473	434	return PTR_ERR(addr);

drivers/dax/pmem/core.c+1−1

@@ -16,7 +16,7 @@ struct dev_dax __dax_pmem_probe(struct device dev, enum dev_dax_subsys subsys) · 15 unmodified lines
16	16	struct dev_dax *dev_dax;
17	17	struct nd_namespace_io *nsio;
18	18	struct dax_region *dax_region;
19		− struct dev_pagemap pgmap = { 0 };
	19	+ struct dev_pagemap pgmap = { };
20	20	struct nd_namespace_common *ndns;
21	21	struct nd_dax *nd_dax = to_nd_dax(dev);
22	22	struct nd_pfn *nd_pfn = &nd_dax->nd_pfn;

drivers/gpu/drm/nouveau/Kconfig+3−3

@@ -84,11 +84,11 @@ config DRM_NOUVEAU_BACKLIGHT · 83 unmodified lines
84	84
85	85	config DRM_NOUVEAU_SVM
86	86	bool "(EXPERIMENTAL) Enable SVM (Shared Virtual Memory) support"
87		− depends on ARCH_HAS_HMM
	87	+ depends on DEVICE_PRIVATE
88	88	depends on DRM_NOUVEAU
	89	+ depends on HMM_MIRROR
89	90	depends on STAGING
90		− select HMM_MIRROR
91		− select DEVICE_PRIVATE
	91	+ select MIGRATE_VMA_HELPER
92	92	default n
93	93	help
94	94	Say Y here if you want to enable experimental support for

drivers/gpu/drm/nouveau/nouveau_dmem.c+47−56

@@ -72,7 +72,8 @@ struct nouveau_dmem_migrate { · 71 unmodified lines
72	72	};
73	73
74	74	struct nouveau_dmem {
75		− struct hmm_devmem *devmem;
	75	+ struct nouveau_drm *drm;
	76	+ struct dev_pagemap pagemap;
76	77	struct nouveau_dmem_migrate migrate;
77	78	struct list_head chunk_free;
78	79	struct list_head chunk_full;
@@ -80,6 +81,11 @@ struct nouveau_dmem { · 1 unmodified line
80	81	struct mutex mutex;
81	82	};
82	83
	84	+static inline struct nouveau_dmem page_to_dmem(struct page page)
	85	+{
	86	+ return container_of(page->pgmap, struct nouveau_dmem, pagemap);
	87	+}
	88	+
83	89	struct nouveau_dmem_fault {
84	90	struct nouveau_drm *drm;
85	91	struct nouveau_fence *fence;
@@ -96,14 +102,10 @@ struct nouveau_migrate { · 10 unmodified lines
96	102	unsigned long dma_nr;
97	103	};
98	104
99		−static void
100		−nouveau_dmem_free(struct hmm_devmem devmem, struct page page)
	105	+static void nouveau_dmem_page_free(struct page *page)
101	106	{
102		− struct nouveau_dmem_chunk *chunk;
103		− unsigned long idx;
104		−
105		− chunk = (void *)hmm_devmem_page_get_drvdata(page);
106		− idx = page_to_pfn(page) - chunk->pfn_first;
	107	+ struct nouveau_dmem_chunk *chunk = page->zone_device_data;
	108	+ unsigned long idx = page_to_pfn(page) - chunk->pfn_first;
107	109
108	110	/*
109	111	* FIXME:
@@ -148,11 +150,12 @@ nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, · 38 unmodified lines
148	150	if (!spage \|\| !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
149	151	continue;
150	152
151		− dpage = hmm_vma_alloc_locked_page(vma, addr);
	153	+ dpage = alloc_page_vma(GFP_HIGHUSER, vma, addr);
152	154	if (!dpage) {
153	155	dst_pfns[i] = MIGRATE_PFN_ERROR;
154	156	continue;
155	157	}
	158	+ lock_page(dpage);
156	159
157	160	dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) \|
158	161	MIGRATE_PFN_LOCKED;
@@ -194,7 +197,7 @@ nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, · 35 unmodified lines
194	197
195	198	dst_addr = fault->dma[fault->npages++];
196	199
197		− chunk = (void *)hmm_devmem_page_get_drvdata(spage);
	200	+ chunk = spage->zone_device_data;
198	201	src_addr = page_to_pfn(spage) - chunk->pfn_first;
199	202	src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
200	203
@@ -259,29 +262,21 @@ static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = { · 58 unmodified lines
259	262	.finalize_and_map = nouveau_dmem_fault_finalize_and_map,
260	263	};
261	264
262		−static vm_fault_t
263		−nouveau_dmem_fault(struct hmm_devmem *devmem,
264		− struct vm_area_struct *vma,
265		− unsigned long addr,
266		− const struct page *page,
267		− unsigned int flags,
268		− pmd_t *pmdp)
	265	+static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
269	266	{
270		− struct drm_device *drm_dev = dev_get_drvdata(devmem->device);
	267	+ struct nouveau_dmem *dmem = page_to_dmem(vmf->page);
271	268	unsigned long src[1] = {0}, dst[1] = {0};
272		− struct nouveau_dmem_fault fault = {0};
	269	+ struct nouveau_dmem_fault fault = { .drm = dmem->drm };
273	270	int ret;
274	271
275		−
276		−
277	272	/*
278	273	* FIXME what we really want is to find some heuristic to migrate more
279	274	* than just one page on CPU fault. When such fault happens it is very
280	275	* likely that more surrounding page will CPU fault too.
281	276	*/
282		− fault.drm = nouveau_drm(drm_dev);
283		− ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,
284		− addr + PAGE_SIZE, src, dst, &fault);
	277	+ ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vmf->vma,
	278	+ vmf->address, vmf->address + PAGE_SIZE,
	279	+ src, dst, &fault);
285	280	if (ret)
286	281	return VM_FAULT_SIGBUS;
287	282
@@ -291,10 +286,9 @@ nouveau_dmem_fault(struct hmm_devmem *devmem, · 3 unmodified lines
291	286	return 0;
292	287	}
293	288
294		−static const struct hmm_devmem_ops
295		−nouveau_dmem_devmem_ops = {
296		− .free = nouveau_dmem_free,
297		− .fault = nouveau_dmem_fault,
	289	+static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = {
	290	+ .page_free = nouveau_dmem_page_free,
	291	+ .migrate_to_ram = nouveau_dmem_migrate_to_ram,
298	292	};
299	293
300	294	static int
@@ -580,7 +574,8 @@ void · 279 unmodified lines
580	574	nouveau_dmem_init(struct nouveau_drm *drm)
581	575	{
582	576	struct device *device = drm->dev->dev;
583		− unsigned long i, size;
	577	+ struct resource *res;
	578	+ unsigned long i, size, pfn_first;
584	579	int ret;
585	580
586	581	/* This only make sense on PASCAL or newer */
@@ -590,6 +585,7 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 3 unmodified lines
590	585	if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
591	586	return;
592	587
	588	+ drm->dmem->drm = drm;
593	589	mutex_init(&drm->dmem->mutex);
594	590	INIT_LIST_HEAD(&drm->dmem->chunk_free);
595	591	INIT_LIST_HEAD(&drm->dmem->chunk_full);
@@ -599,11 +595,8 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 3 unmodified lines
599	595
600	596	/* Initialize migration dma helpers before registering memory */
601	597	ret = nouveau_dmem_migrate_init(drm);
602		− if (ret) {
603		− kfree(drm->dmem);
604		− drm->dmem = NULL;
605		− return;
606		− }
	598	+ if (ret)
	599	+ goto out_free;
607	600
608	601	/*
609	602	* FIXME we need some kind of policy to decide how much VRAM we
@@ -611,14 +604,16 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 1 unmodified line
611	604	* and latter if we want to do thing like over commit then we
612	605	* could revisit this.
613	606	*/
614		− drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,
615		− device, size);
616		− if (IS_ERR(drm->dmem->devmem)) {
617		− kfree(drm->dmem);
618		− drm->dmem = NULL;
619		− return;
620		− }
621		−
	607	+ res = devm_request_free_mem_region(device, &iomem_resource, size);
	608	+ if (IS_ERR(res))
	609	+ goto out_free;
	610	+ drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
	611	+ drm->dmem->pagemap.res = *res;
	612	+ drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops;
	613	+ if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap)))
	614	+ goto out_free;
	615	+
	616	+ pfn_first = res->start >> PAGE_SHIFT;
622	617	for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
623	618	struct nouveau_dmem_chunk *chunk;
624	619	struct page *page;
@@ -631,17 +626,19 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 6 unmodified lines
631	626	}
632	627
633	628	chunk->drm = drm;
634		− chunk->pfn_first = drm->dmem->devmem->pfn_first;
635		− chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
	629	+ chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES);
636	630	list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
637	631
638	632	page = pfn_to_page(chunk->pfn_first);
639		− for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {
640		− hmm_devmem_page_set_drvdata(page, (long)chunk);
641		− }
	633	+ for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page)
	634	+ page->zone_device_data = chunk;
642	635	}
643	636
644	637	NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
	638	+ return;
	639	+out_free:
	640	+ kfree(drm->dmem);
	641	+ drm->dmem = NULL;
645	642	}
646	643
647	644	static void
@@ -697,7 +694,7 @@ nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma, · 49 unmodified lines
697	694	if (!dpage \|\| dst_pfns[i] == MIGRATE_PFN_ERROR)
698	695	continue;
699	696
700		− chunk = (void *)hmm_devmem_page_get_drvdata(dpage);
	697	+ chunk = dpage->zone_device_data;
701	698	dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
702	699	dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
703	700
@@ -832,13 +829,7 @@ out: · 128 unmodified lines
832	829	static inline bool
833	830	nouveau_dmem_page(struct nouveau_drm drm, struct page page)
834	831	{
835		− if (!is_device_private_page(page))
836		− return false;
837		−
838		− if (drm->dmem->devmem != page->pgmap->data)
839		− return false;
840		−
841		− return true;
	832	+ return is_device_private_page(page) && drm->dmem == page_to_dmem(page);
842	833	}
843	834
844	835	void
@@ -867,7 +858,7 @@ nouveau_dmem_convert_pfn(struct nouveau_drm *drm, · 22 unmodified lines
867	858	continue;
868	859	}
869	860
870		− chunk = (void *)hmm_devmem_page_get_drvdata(page);
	861	+ chunk = page->zone_device_data;
871	862	addr = page_to_pfn(page) - chunk->pfn_first;
872	863	addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
873	864

drivers/nvdimm/pfn_devs.c+1−2

@@ -622,7 +622,6 @@ static int __nvdimm_setup_pfn(struct nd_pfn nd_pfn, struct dev_pagemap pgmap) · 621 unmodified lines
622	622	if (offset < reserve)
623	623	return -EINVAL;
624	624	nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns);
625		− pgmap->altmap_valid = false;
626	625	} else if (nd_pfn->mode == PFN_MODE_PMEM) {
627	626	nd_pfn->npfns = PFN_SECTION_ALIGN_UP((resource_size(res)
628	627	- offset) / PAGE_SIZE);
@@ -634,7 +633,7 @@ static int __nvdimm_setup_pfn(struct nd_pfn nd_pfn, struct dev_pagemap pgmap) · 5 unmodified lines
634	633	memcpy(altmap, &__altmap, sizeof(*altmap));
635	634	altmap->free = PHYS_PFN(offset - reserve);
636	635	altmap->alloc = 0;
637		− pgmap->altmap_valid = true;
	636	+ pgmap->flags \|= PGMAP_ALTMAP_VALID;
638	637	} else
639	638	return -ENXIO;
640	639

drivers/nvdimm/pmem.c+19−32

@@ -303,24 +303,24 @@ static const struct attribute_group *pmem_attribute_groups[] = { · 302 unmodified lines
303	303	NULL,
304	304	};
305	305
306		−static void __pmem_release_queue(struct percpu_ref *ref)
	306	+static void pmem_pagemap_cleanup(struct dev_pagemap *pgmap)
307	307	{
308		− struct request_queue *q;
	308	+ struct request_queue *q =
	309	+ container_of(pgmap->ref, struct request_queue, q_usage_counter);
309	310
310		− q = container_of(ref, typeof(*q), q_usage_counter);
311	311	blk_cleanup_queue(q);
312	312	}
313	313
314		−static void pmem_release_queue(void *ref)
	314	+static void pmem_release_queue(void *pgmap)
315	315	{
316		− __pmem_release_queue(ref);
	316	+ pmem_pagemap_cleanup(pgmap);
317	317	}
318	318
319		−static void pmem_freeze_queue(struct percpu_ref *ref)
	319	+static void pmem_pagemap_kill(struct dev_pagemap *pgmap)
320	320	{
321		− struct request_queue *q;
	321	+ struct request_queue *q =
	322	+ container_of(pgmap->ref, struct request_queue, q_usage_counter);
322	323
323		− q = container_of(ref, typeof(*q), q_usage_counter);
324	324	blk_freeze_queue_start(q);
325	325	}
326	326
@@ -334,26 +334,16 @@ static void pmem_release_disk(void *__pmem) · 7 unmodified lines
334	334	put_disk(pmem->disk);
335	335	}
336	336
337		−static void pmem_release_pgmap_ops(void *__pgmap)
338		−{
339		− dev_pagemap_put_ops();
340		−}
341		−
342		−static void fsdax_pagefree(struct page page, void data)
	337	+static void pmem_pagemap_page_free(struct page *page)
343	338	{
344	339	wake_up_var(&page->_refcount);
345	340	}
346	341
347		−static int setup_pagemap_fsdax(struct device dev, struct dev_pagemap pgmap)
348		−{
349		− dev_pagemap_get_ops();
350		− if (devm_add_action_or_reset(dev, pmem_release_pgmap_ops, pgmap))
351		− return -ENOMEM;
352		− pgmap->type = MEMORY_DEVICE_FS_DAX;
353		− pgmap->page_free = fsdax_pagefree;
354		−
355		− return 0;
356		−}
	342	+static const struct dev_pagemap_ops fsdax_pagemap_ops = {
	343	+ .page_free = pmem_pagemap_page_free,
	344	+ .kill = pmem_pagemap_kill,
	345	+ .cleanup = pmem_pagemap_cleanup,
	346	+};
357	347
358	348	static int pmem_attach_disk(struct device *dev,
359	349	struct nd_namespace_common *ndns)
@@ -409,11 +399,9 @@ static int pmem_attach_disk(struct device *dev, · 49 unmodified lines
409	399
410	400	pmem->pfn_flags = PFN_DEV;
411	401	pmem->pgmap.ref = &q->q_usage_counter;
412		− pmem->pgmap.kill = pmem_freeze_queue;
413		− pmem->pgmap.cleanup = __pmem_release_queue;
414	402	if (is_nd_pfn(dev)) {
415		− if (setup_pagemap_fsdax(dev, &pmem->pgmap))
416		− return -ENOMEM;
	403	+ pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;
	404	+ pmem->pgmap.ops = &fsdax_pagemap_ops;
417	405	addr = devm_memremap_pages(dev, &pmem->pgmap);
418	406	pfn_sb = nd_pfn->pfn_sb;
419	407	pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);
@@ -424,15 +412,14 @@ static int pmem_attach_disk(struct device *dev, · 4 unmodified lines
424	412	bb_res.start += pmem->data_offset;
425	413	} else if (pmem_should_map_pages(dev)) {
426	414	memcpy(&pmem->pgmap.res, &nsio->res, sizeof(pmem->pgmap.res));
427		− pmem->pgmap.altmap_valid = false;
428		− if (setup_pagemap_fsdax(dev, &pmem->pgmap))
429		− return -ENOMEM;
	415	+ pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;
	416	+ pmem->pgmap.ops = &fsdax_pagemap_ops;
430	417	addr = devm_memremap_pages(dev, &pmem->pgmap);
431	418	pmem->pfn_flags \|= PFN_MAP;
432	419	memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));
433	420	} else {
434	421	if (devm_add_action_or_reset(dev, pmem_release_queue,
435		− &q->q_usage_counter))
	422	+ &pmem->pgmap))
436	423	return -ENOMEM;
437	424	addr = devm_memremap(dev, pmem->phys_addr,
438	425	pmem->size, ARCH_MEMREMAP_PMEM);

drivers/pci/p2pdma.c+4−48

@@ -25,12 +25,6 @@ struct pci_p2pdma { · 24 unmodified lines
25	25	bool p2pmem_published;
26	26	};
27	27
28		−struct p2pdma_pagemap {
29		− struct dev_pagemap pgmap;
30		− struct percpu_ref ref;
31		− struct completion ref_done;
32		−};
33		−
34	28	static ssize_t size_show(struct device dev, struct device_attribute attr,
35	29	char *buf)
36	30	{
@@ -79,31 +73,6 @@ static const struct attribute_group p2pmem_group = { · 42 unmodified lines
79	73	.name = "p2pmem",
80	74	};
81	75
82		−static struct p2pdma_pagemap to_p2p_pgmap(struct percpu_ref ref)
83		−{
84		− return container_of(ref, struct p2pdma_pagemap, ref);
85		−}
86		−
87		−static void pci_p2pdma_percpu_release(struct percpu_ref *ref)
88		−{
89		− struct p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(ref);
90		−
91		− complete(&p2p_pgmap->ref_done);
92		−}
93		−
94		−static void pci_p2pdma_percpu_kill(struct percpu_ref *ref)
95		−{
96		− percpu_ref_kill(ref);
97		−}
98		−
99		−static void pci_p2pdma_percpu_cleanup(struct percpu_ref *ref)
100		−{
101		− struct p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(ref);
102		−
103		− wait_for_completion(&p2p_pgmap->ref_done);
104		− percpu_ref_exit(&p2p_pgmap->ref);
105		−}
106		−
107	76	static void pci_p2pdma_release(void *data)
108	77	{
109	78	struct pci_dev *pdev = data;
@@ -166,7 +135,6 @@ out: · 56 unmodified lines
166	135	int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
167	136	u64 offset)
168	137	{
169		− struct p2pdma_pagemap *p2p_pgmap;
170	138	struct dev_pagemap *pgmap;
171	139	void *addr;
172	140	int error;
@@ -189,27 +157,15 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, · 16 unmodified lines
189	157	return error;
190	158	}
191	159
192		− p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL);
193		− if (!p2p_pgmap)
	160	+ pgmap = devm_kzalloc(&pdev->dev, sizeof(*pgmap), GFP_KERNEL);
	161	+ if (!pgmap)
194	162	return -ENOMEM;
195		−
196		− init_completion(&p2p_pgmap->ref_done);
197		− error = percpu_ref_init(&p2p_pgmap->ref,
198		− pci_p2pdma_percpu_release, 0, GFP_KERNEL);
199		− if (error)
200		− goto pgmap_free;
201		−
202		− pgmap = &p2p_pgmap->pgmap;
203		−
204	163	pgmap->res.start = pci_resource_start(pdev, bar) + offset;
205	164	pgmap->res.end = pgmap->res.start + size - 1;
206	165	pgmap->res.flags = pci_resource_flags(pdev, bar);
207		− pgmap->ref = &p2p_pgmap->ref;
208	166	pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;
209	167	pgmap->pci_p2pdma_bus_offset = pci_bus_address(pdev, bar) -
210	168	pci_resource_start(pdev, bar);
211		− pgmap->kill = pci_p2pdma_percpu_kill;
212		− pgmap->cleanup = pci_p2pdma_percpu_cleanup;
213	169
214	170	addr = devm_memremap_pages(&pdev->dev, pgmap);
215	171	if (IS_ERR(addr)) {
@@ -220,7 +176,7 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, · 4 unmodified lines
220	176	error = gen_pool_add_owner(pdev->p2pdma->pool, (unsigned long)addr,
221	177	pci_bus_address(pdev, bar) + offset,
222	178	resource_size(&pgmap->res), dev_to_node(&pdev->dev),
223		− &p2p_pgmap->ref);
	179	+ pgmap->ref);
224	180	if (error)
225	181	goto pages_free;
226	182
@@ -232,7 +188,7 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, · 5 unmodified lines
232	188	pages_free:
233	189	devm_memunmap_pages(&pdev->dev, pgmap);
234	190	pgmap_free:
235		− devm_kfree(&pdev->dev, p2p_pgmap);
	191	+ devm_kfree(&pdev->dev, pgmap);
236	192	return error;
237	193	}
238	194	EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource);

fs/proc/task_mmu.c+1−1

@@ -1279,7 +1279,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, · 1278 unmodified lines
1279	1279	if (pm->show_pfn)
1280	1280	frame = pte_pfn(pte);
1281	1281	flags \|= PM_PRESENT;
1282		− page = _vm_normal_page(vma, addr, pte, true);
	1282	+ page = vm_normal_page(vma, addr, pte);
1283	1283	if (pte_soft_dirty(pte))
1284	1284	flags \|= PM_SOFT_DIRTY;
1285	1285	} else if (is_swap_pte(pte)) {

include/linux/hmm.h+1−197

@@ -62,7 +62,7 @@ · 61 unmodified lines
62	62	#include <linux/kconfig.h>
63	63	#include <asm/pgtable.h>
64	64
65		−#if IS_ENABLED(CONFIG_HMM)
	65	+#ifdef CONFIG_HMM_MIRROR
66	66
67	67	#include <linux/device.h>
68	68	#include <linux/migrate.h>
@@ -324,9 +324,6 @@ static inline uint64_t hmm_pfn_from_pfn(const struct hmm_range *range, · 255 unmodified lines
324	324	return hmm_device_entry_from_pfn(range, pfn);
325	325	}
326	326
327		−
328		−
329		−#if IS_ENABLED(CONFIG_HMM_MIRROR)
330	327	/*
331	328	* Mirroring: how to synchronize device page table with CPU page table.
332	329	*
@@ -550,197 +547,4 @@ static inline void hmm_mm_init(struct mm_struct *mm) · 217 unmodified lines
550	547	static inline void hmm_mm_init(struct mm_struct *mm) {}
551	548	#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
552	549
553		−#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) \|\| IS_ENABLED(CONFIG_DEVICE_PUBLIC)
554		−struct hmm_devmem;
555		−
556		−struct page hmm_vma_alloc_locked_page(struct vm_area_struct vma,
557		− unsigned long addr);
558		−
559		−/*
560		− * struct hmm_devmem_ops - callback for ZONE_DEVICE memory events
561		− *
562		− * @free: call when refcount on page reach 1 and thus is no longer use
563		− * @fault: call when there is a page fault to unaddressable memory
564		− *
565		− * Both callback happens from page_free() and page_fault() callback of struct
566		− * dev_pagemap respectively. See include/linux/memremap.h for more details on
567		− * those.
568		− *
569		− * The hmm_devmem_ops callback are just here to provide a coherent and
570		− * uniq API to device driver and device driver should not register their
571		− * own page_free() or page_fault() but rely on the hmm_devmem_ops call-
572		− * back.
573		− */
574		−struct hmm_devmem_ops {
575		− /*
576		− * free() - free a device page
577		− * @devmem: device memory structure (see struct hmm_devmem)
578		− * @page: pointer to struct page being freed
579		− *
580		− * Call back occurs whenever a device page refcount reach 1 which
581		− * means that no one is holding any reference on the page anymore
582		− * (ZONE_DEVICE page have an elevated refcount of 1 as default so
583		− * that they are not release to the general page allocator).
584		− *
585		− * Note that callback has exclusive ownership of the page (as no
586		− * one is holding any reference).
587		− */
588		− void (free)(struct hmm_devmem devmem, struct page *page);
589		− /*
590		− * fault() - CPU page fault or get user page (GUP)
591		− * @devmem: device memory structure (see struct hmm_devmem)
592		− * @vma: virtual memory area containing the virtual address
593		− * @addr: virtual address that faulted or for which there is a GUP
594		− * @page: pointer to struct page backing virtual address (unreliable)
595		− * @flags: FAULT_FLAG_* (see include/linux/mm.h)
596		− * @pmdp: page middle directory
597		− * Return: VM_FAULT_MINOR/MAJOR on success or one of VM_FAULT_ERROR
598		− * on error
599		− *
600		− * The callback occurs whenever there is a CPU page fault or GUP on a
601		− * virtual address. This means that the device driver must migrate the
602		− * page back to regular memory (CPU accessible).
603		− *
604		− * The device driver is free to migrate more than one page from the
605		− * fault() callback as an optimization. However if the device decides
606		− * to migrate more than one page it must always priotirize the faulting
607		− * address over the others.
608		− *
609		− * The struct page pointer is only given as a hint to allow quick
610		− * lookup of internal device driver data. A concurrent migration
611		− * might have already freed that page and the virtual address might
612		− * no longer be backed by it. So it should not be modified by the
613		− * callback.
614		− *
615		− * Note that mmap semaphore is held in read mode at least when this
616		− * callback occurs, hence the vma is valid upon callback entry.
617		− */
618		− vm_fault_t (fault)(struct hmm_devmem devmem,
619		− struct vm_area_struct *vma,
620		− unsigned long addr,
621		− const struct page *page,
622		− unsigned int flags,
623		− pmd_t *pmdp);
624		−};
625		−
626		−/*
627		− * struct hmm_devmem - track device memory
628		− *
629		− * @completion: completion object for device memory
630		− * @pfn_first: first pfn for this resource (set by hmm_devmem_add())
631		− * @pfn_last: last pfn for this resource (set by hmm_devmem_add())
632		− * @resource: IO resource reserved for this chunk of memory
633		− * @pagemap: device page map for that chunk
634		− * @device: device to bind resource to
635		− * @ops: memory operations callback
636		− * @ref: per CPU refcount
637		− * @page_fault: callback when CPU fault on an unaddressable device page
638		− *
639		− * This is a helper structure for device drivers that do not wish to implement
640		− * the gory details related to hotplugging new memoy and allocating struct
641		− * pages.
642		− *
643		− * Device drivers can directly use ZONE_DEVICE memory on their own if they
644		− * wish to do so.
645		− *
646		− * The page_fault() callback must migrate page back, from device memory to
647		− * system memory, so that the CPU can access it. This might fail for various
648		− * reasons (device issues, device have been unplugged, ...). When such error
649		− * conditions happen, the page_fault() callback must return VM_FAULT_SIGBUS and
650		− * set the CPU page table entry to "poisoned".
651		− *
652		− * Note that because memory cgroup charges are transferred to the device memory,
653		− * this should never fail due to memory restrictions. However, allocation
654		− * of a regular system page might still fail because we are out of memory. If
655		− * that happens, the page_fault() callback must return VM_FAULT_OOM.
656		− *
657		− * The page_fault() callback can also try to migrate back multiple pages in one
658		− * chunk, as an optimization. It must, however, prioritize the faulting address
659		− * over all the others.
660		− */
661		−typedef vm_fault_t (dev_page_fault_t)(struct vm_area_struct vma,
662		− unsigned long addr,
663		− const struct page *page,
664		− unsigned int flags,
665		− pmd_t *pmdp);
666		−
667		−struct hmm_devmem {
668		− struct completion completion;
669		− unsigned long pfn_first;
670		− unsigned long pfn_last;
671		− struct resource *resource;
672		− struct device *device;
673		− struct dev_pagemap pagemap;
674		− const struct hmm_devmem_ops *ops;
675		− struct percpu_ref ref;
676		− dev_page_fault_t page_fault;
677		−};
678		−
679		−/*
680		− * To add (hotplug) device memory, HMM assumes that there is no real resource
681		− * that reserves a range in the physical address space (this is intended to be
682		− * use by unaddressable device memory). It will reserve a physical range big
683		− * enough and allocate struct page for it.
684		− *
685		− * The device driver can wrap the hmm_devmem struct inside a private device
686		− * driver struct.
687		− */
688		−struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops,
689		− struct device *device,
690		− unsigned long size);
691		−struct hmm_devmem hmm_devmem_add_resource(const struct hmm_devmem_ops ops,
692		− struct device *device,
693		− struct resource *res);
694		−
695		−/*
696		− * hmm_devmem_page_set_drvdata - set per-page driver data field
697		− *
698		− * @page: pointer to struct page
699		− * @data: driver data value to set
700		− *
701		− * Because page can not be on lru we have an unsigned long that driver can use
702		− * to store a per page field. This just a simple helper to do that.
703		− */
704		−static inline void hmm_devmem_page_set_drvdata(struct page *page,
705		− unsigned long data)
706		−{
707		− page->hmm_data = data;
708		−}
709		−
710		−/*
711		− * hmm_devmem_page_get_drvdata - get per page driver data field
712		− *
713		− * @page: pointer to struct page
714		− * Return: driver data value
715		− */
716		−static inline unsigned long hmm_devmem_page_get_drvdata(const struct page *page)
717		−{
718		− return page->hmm_data;
719		−}
720		−
721		−
722		−/*
723		− * struct hmm_device - fake device to hang device memory onto
724		− *
725		− * @device: device struct
726		− * @minor: device minor number
727		− */
728		−struct hmm_device {
729		− struct device device;
730		− unsigned int minor;
731		−};
732		−
733		−/*
734		− * A device driver that wants to handle multiple devices memory through a
735		− * single fake device can use hmm_device to do so. This is purely a helper and
736		− * it is not strictly needed, in order to make use of any HMM functionality.
737		− */
738		−struct hmm_device hmm_device_new(void drvdata);
739		−void hmm_device_put(struct hmm_device *hmm_device);
740		−#endif /* CONFIG_DEVICE_PRIVATE \|\| CONFIG_DEVICE_PUBLIC */
741		−#else /* IS_ENABLED(CONFIG_HMM) */
742		−static inline void hmm_mm_destroy(struct mm_struct *mm) {}
743		−static inline void hmm_mm_init(struct mm_struct *mm) {}
744		−#endif /* IS_ENABLED(CONFIG_HMM) */
745		−
746	550	#endif /* LINUX_HMM_H */

include/linux/ioport.h+2−1

@@ -132,7 +132,6 @@ enum { · 131 unmodified lines
132	132	IORES_DESC_PERSISTENT_MEMORY = 4,
133	133	IORES_DESC_PERSISTENT_MEMORY_LEGACY = 5,
134	134	IORES_DESC_DEVICE_PRIVATE_MEMORY = 6,
135		− IORES_DESC_DEVICE_PUBLIC_MEMORY = 7,
136	135	};
137	136
138	137	/* helpers to define resources */
@@ -286,6 +285,8 @@ static inline bool resource_overlaps(struct resource r1, struct resource r2) · 147 unmodified lines
286	285	return (r1->start <= r2->end && r1->end >= r2->start);
287	286	}
288	287
	288	+struct resource devm_request_free_mem_region(struct device dev,
	289	+ struct resource *base, unsigned long size);
289	290
290	291	#endif /* __ASSEMBLY__ */
291	292	#endif /* _LINUX_IOPORT_H */

include/linux/memremap.h+49−26

@@ -37,13 +37,6 @@ struct vmem_altmap { · 36 unmodified lines
37	37	* A more complete discussion of unaddressable memory may be found in
38	38	* include/linux/hmm.h and Documentation/vm/hmm.rst.
39	39	*
40		− * MEMORY_DEVICE_PUBLIC:
41		− * Device memory that is cache coherent from device and CPU point of view. This
42		− * is use on platform that have an advance system bus (like CAPI or CCIX). A
43		− * driver can hotplug the device memory using ZONE_DEVICE and with that memory
44		− * type. Any page of a process can be migrated to such memory. However no one
45		− * should be allow to pin such memory so that it can always be evicted.
46		− *
47	40	* MEMORY_DEVICE_FS_DAX:
48	41	* Host memory that has similar access semantics as System RAM i.e. DMA
49	42	* coherent and supports page pinning. In support of coordinating page
@@ -52,54 +45,84 @@ struct vmem_altmap { · 2 unmodified lines
52	45	* wakeup is used to coordinate physical address space management (ex:
53	46	* fs truncate/hole punch) vs pinned pages (ex: device dma).
54	47	*
	48	+ * MEMORY_DEVICE_DEVDAX:
	49	+ * Host memory that has similar access semantics as System RAM i.e. DMA
	50	+ * coherent and supports page pinning. In contrast to
	51	+ * MEMORY_DEVICE_FS_DAX, this memory is access via a device-dax
	52	+ * character device.
	53	+ *
55	54	* MEMORY_DEVICE_PCI_P2PDMA:
56	55	* Device memory residing in a PCI BAR intended for use with Peer-to-Peer
57	56	* transactions.
58	57	*/
59	58	enum memory_type {
	59	+ /* 0 is reserved to catch uninitialized type fields */
60	60	MEMORY_DEVICE_PRIVATE = 1,
61		− MEMORY_DEVICE_PUBLIC,
62	61	MEMORY_DEVICE_FS_DAX,
	62	+ MEMORY_DEVICE_DEVDAX,
63	63	MEMORY_DEVICE_PCI_P2PDMA,
64	64	};
65	65
66		−/*
67		− * Additional notes about MEMORY_DEVICE_PRIVATE may be found in
68		− * include/linux/hmm.h and Documentation/vm/hmm.rst. There is also a brief
69		− * explanation in include/linux/memory_hotplug.h.
70		− *
71		− * The page_free() callback is called once the page refcount reaches 1
72		− * (ZONE_DEVICE pages never reach 0 refcount unless there is a refcount bug.
73		− * This allows the device driver to implement its own memory management.)
74		− */
75		−typedef void (dev_page_free_t)(struct page page, void *data);
	66	+struct dev_pagemap_ops {
	67	+ /*
	68	+ * Called once the page refcount reaches 1. (ZONE_DEVICE pages never
	69	+ * reach 0 refcount unless there is a refcount bug. This allows the
	70	+ * device driver to implement its own memory management.)
	71	+ */
	72	+ void (page_free)(struct page page);
	73	+
	74	+ /*
	75	+ * Transition the refcount in struct dev_pagemap to the dead state.
	76	+ */
	77	+ void (kill)(struct dev_pagemap pgmap);
	78	+
	79	+ /*
	80	+ * Wait for refcount in struct dev_pagemap to be idle and reap it.
	81	+ */
	82	+ void (cleanup)(struct dev_pagemap pgmap);
	83	+
	84	+ /*
	85	+ * Used for private (un-addressable) device memory only. Must migrate
	86	+ * the page back to a CPU accessible page.
	87	+ */
	88	+ vm_fault_t (migrate_to_ram)(struct vm_fault vmf);
	89	+};
	90	+
	91	+#define PGMAP_ALTMAP_VALID (1 << 0)
76	92
77	93	/**
78	94	* struct dev_pagemap - metadata for ZONE_DEVICE mappings
79		− * @page_free: free page callback when page refcount reaches 1
80	95	* @altmap: pre-allocated/reserved memory for vmemmap allocations
81	96	* @res: physical address range covered by @ref
82	97	* @ref: reference count that pins the devm_memremap_pages() mapping
83		− * @kill: callback to transition @ref to the dead state
84		− * @cleanup: callback to wait for @ref to be idle and reap it
	98	+ * @internal_ref: internal reference if @ref is not provided by the caller
	99	+ * @done: completion for @internal_ref
85	100	* @dev: host device of the mapping for debug
86	101	* @data: private data pointer for page_free()
87	102	* @type: memory type: see MEMORY_* in memory_hotplug.h
	103	+ * @flags: PGMAP_* flags to specify defailed behavior
	104	+ * @ops: method table
88	105	*/
89	106	struct dev_pagemap {
90		− dev_page_free_t page_free;
91	107	struct vmem_altmap altmap;
92		− bool altmap_valid;
93	108	struct resource res;
94	109	struct percpu_ref *ref;
95		− void (kill)(struct percpu_ref ref);
96		− void (cleanup)(struct percpu_ref ref);
	110	+ struct percpu_ref internal_ref;
	111	+ struct completion done;
97	112	struct device *dev;
98		− void *data;
99	113	enum memory_type type;
	114	+ unsigned int flags;
100	115	u64 pci_p2pdma_bus_offset;
	116	+ const struct dev_pagemap_ops *ops;
101	117	};
102	118
	119	+static inline struct vmem_altmap pgmap_altmap(struct dev_pagemap pgmap)
	120	+{
	121	+ if (pgmap->flags & PGMAP_ALTMAP_VALID)
	122	+ return &pgmap->altmap;
	123	+ return NULL;
	124	+}
	125	+
103	126	#ifdef CONFIG_ZONE_DEVICE
104	127	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap);
105	128	void devm_memunmap_pages(struct device dev, struct dev_pagemap pgmap);

include/linux/mm.h+2−26

@@ -932,8 +932,6 @@ static inline bool is_zone_device_page(const struct page *page) · 931 unmodified lines
932	932	#endif
933	933
934	934	#ifdef CONFIG_DEV_PAGEMAP_OPS
935		−void dev_pagemap_get_ops(void);
936		−void dev_pagemap_put_ops(void);
937	935	void __put_devmap_managed_page(struct page *page);
938	936	DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
939	937	static inline bool put_devmap_managed_page(struct page *page)
@@ -944,7 +942,6 @@ static inline bool put_devmap_managed_page(struct page *page) · 4 unmodified lines
944	942	return false;
945	943	switch (page->pgmap->type) {
946	944	case MEMORY_DEVICE_PRIVATE:
947		− case MEMORY_DEVICE_PUBLIC:
948	945	case MEMORY_DEVICE_FS_DAX:
949	946	__put_devmap_managed_page(page);
950	947	return true;
@@ -960,12 +957,6 @@ static inline bool is_device_private_page(const struct page *page) · 9 unmodified lines
960	957	page->pgmap->type == MEMORY_DEVICE_PRIVATE;
961	958	}
962	959
963		−static inline bool is_device_public_page(const struct page *page)
964		−{
965		− return is_zone_device_page(page) &&
966		− page->pgmap->type == MEMORY_DEVICE_PUBLIC;
967		−}
968		−
969	960	#ifdef CONFIG_PCI_P2PDMA
970	961	static inline bool is_pci_p2pdma_page(const struct page *page)
971	962	{
@@ -980,14 +971,6 @@ static inline bool is_pci_p2pdma_page(const struct page *page) · 8 unmodified lines
980	971	#endif /* CONFIG_PCI_P2PDMA */
981	972
982	973	#else /* CONFIG_DEV_PAGEMAP_OPS */
983		−static inline void dev_pagemap_get_ops(void)
984		−{
985		−}
986		−
987		−static inline void dev_pagemap_put_ops(void)
988		−{
989		−}
990		−
991	974	static inline bool put_devmap_managed_page(struct page *page)
992	975	{
993	976	return false;
@@ -998,11 +981,6 @@ static inline bool is_device_private_page(const struct page *page) · 4 unmodified lines
998	981	return false;
999	982	}
1000	983
1001		−static inline bool is_device_public_page(const struct page *page)
1002		−{
1003		− return false;
1004		−}
1005		−
1006	984	static inline bool is_pci_p2pdma_page(const struct page *page)
1007	985	{
1008	986	return false;
@@ -1431,10 +1409,8 @@ struct zap_details { · 422 unmodified lines
1431	1409	pgoff_t last_index; /* Highest page->index to unmap */
1432	1410	};
1433	1411
1434		−struct page _vm_normal_page(struct vm_area_struct vma, unsigned long addr,
1435		− pte_t pte, bool with_public_device);
1436		−#define vm_normal_page(vma, addr, pte) _vm_normal_page(vma, addr, pte, false)
1437		−
	1412	+struct page vm_normal_page(struct vm_area_struct vma, unsigned long addr,
	1413	+ pte_t pte);
1438	1414	struct page vm_normal_page_pmd(struct vm_area_struct vma, unsigned long addr,
1439	1415	pmd_t pmd);
1440	1416

include/linux/mm_types.h+2−2

@@ -158,7 +158,7 @@ struct page { · 157 unmodified lines
158	158	struct { /* ZONE_DEVICE pages */
159	159	/** @pgmap: Points to the hosting device page map. */
160	160	struct dev_pagemap *pgmap;
161		− unsigned long hmm_data;
	161	+ void *zone_device_data;
162	162	unsigned long _zd_pad_1; /* uses mapping */
163	163	};
164	164
@@ -501,7 +501,7 @@ struct mm_struct { · 336 unmodified lines
501	501	#endif
502	502	struct work_struct async_put_work;
503	503
504		−#if IS_ENABLED(CONFIG_HMM)
	504	+#ifdef CONFIG_HMM_MIRROR
505	505	/* HMM needs to track a few things per mm */
506	506	struct hmm *hmm;
507	507	#endif

include/linux/swapops.h+0−15

@@ -129,12 +129,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 128 unmodified lines
129	129	{
130	130	return pfn_to_page(swp_offset(entry));
131	131	}
132		−
133		−vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
134		− unsigned long addr,
135		− swp_entry_t entry,
136		− unsigned int flags,
137		− pmd_t *pmdp);
138	132	#else /* CONFIG_DEVICE_PRIVATE */
139	133	static inline swp_entry_t make_device_private_entry(struct page *page, bool write)
140	134	{
@@ -164,15 +158,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 23 unmodified lines
164	158	{
165	159	return NULL;
166	160	}
167		−
168		−static inline vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
169		− unsigned long addr,
170		− swp_entry_t entry,
171		− unsigned int flags,
172		− pmd_t *pmdp)
173		−{
174		− return VM_FAULT_SIGBUS;
175		−}
176	161	#endif /* CONFIG_DEVICE_PRIVATE */
177	162
178	163	#ifdef CONFIG_MIGRATION

kernel/memremap.c+118−76

@@ -11,41 +11,39 @@ · 10 unmodified lines
11	11	#include <linux/types.h>
12	12	#include <linux/wait_bit.h>
13	13	#include <linux/xarray.h>
14		−#include <linux/hmm.h>
15	14
16	15	static DEFINE_XARRAY(pgmap_array);
17	16	#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
18	17	#define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
19	18
20		−#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
21		−vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
22		− unsigned long addr,
23		− swp_entry_t entry,
24		− unsigned int flags,
25		− pmd_t *pmdp)
	19	+#ifdef CONFIG_DEV_PAGEMAP_OPS
	20	+DEFINE_STATIC_KEY_FALSE(devmap_managed_key);
	21	+EXPORT_SYMBOL(devmap_managed_key);
	22	+static atomic_t devmap_managed_enable;
	23	+
	24	+static void devmap_managed_enable_put(void *data)
26	25	{
27		− struct page *page = device_private_entry_to_page(entry);
28		− struct hmm_devmem *devmem;
	26	+ if (atomic_dec_and_test(&devmap_managed_enable))
	27	+ static_branch_disable(&devmap_managed_key);
	28	+}
29	29
30		− devmem = container_of(page->pgmap, typeof(*devmem), pagemap);
	30	+static int devmap_managed_enable_get(struct device dev, struct dev_pagemap pgmap)
	31	+{
	32	+ if (!pgmap->ops \|\| !pgmap->ops->page_free) {
	33	+ WARN(1, "Missing page_free method\n");
	34	+ return -EINVAL;
	35	+ }
31	36
32		− /*
33		− * The page_fault() callback must migrate page back to system memory
34		− * so that CPU can access it. This might fail for various reasons
35		− * (device issue, device was unsafely unplugged, ...). When such
36		− * error conditions happen, the callback must return VM_FAULT_SIGBUS.
37		− *
38		− * Note that because memory cgroup charges are accounted to the device
39		− * memory, this should never fail because of memory restrictions (but
40		− * allocation of regular system page might still fail because we are
41		− * out of memory).
42		− *
43		− * There is a more in-depth description of what that callback can and
44		− * cannot do, in include/linux/memremap.h
45		− */
46		− return devmem->page_fault(vma, addr, page, flags, pmdp);
	37	+ if (atomic_inc_return(&devmap_managed_enable) == 1)
	38	+ static_branch_enable(&devmap_managed_key);
	39	+ return devm_add_action_or_reset(dev, devmap_managed_enable_put, NULL);
47	40	}
48		−#endif /* CONFIG_DEVICE_PRIVATE */
	41	+#else
	42	+static int devmap_managed_enable_get(struct device dev, struct dev_pagemap pgmap)
	43	+{
	44	+ return -EINVAL;
	45	+}
	46	+#endif /* CONFIG_DEV_PAGEMAP_OPS */
49	47
50	48	static void pgmap_array_delete(struct resource *res)
51	49	{
@@ -56,14 +54,8 @@ static void pgmap_array_delete(struct resource *res) · 4 unmodified lines
56	54
57	55	static unsigned long pfn_first(struct dev_pagemap *pgmap)
58	56	{
59		− const struct resource *res = &pgmap->res;
60		− struct vmem_altmap *altmap = &pgmap->altmap;
61		− unsigned long pfn;
62		−
63		− pfn = res->start >> PAGE_SHIFT;
64		− if (pgmap->altmap_valid)
65		− pfn += vmem_altmap_offset(altmap);
66		− return pfn;
	57	+ return (pgmap->res.start >> PAGE_SHIFT) +
	58	+ vmem_altmap_offset(pgmap_altmap(pgmap));
67	59	}
68	60
69	61	static unsigned long pfn_end(struct dev_pagemap *pgmap)
@@ -83,6 +75,24 @@ static unsigned long pfn_next(unsigned long pfn) · 13 unmodified lines
83	75	#define for_each_device_pfn(pfn, map) \
84	76	for (pfn = pfn_first(map); pfn < pfn_end(map); pfn = pfn_next(pfn))
85	77
	78	+static void dev_pagemap_kill(struct dev_pagemap *pgmap)
	79	+{
	80	+ if (pgmap->ops && pgmap->ops->kill)
	81	+ pgmap->ops->kill(pgmap);
	82	+ else
	83	+ percpu_ref_kill(pgmap->ref);
	84	+}
	85	+
	86	+static void dev_pagemap_cleanup(struct dev_pagemap *pgmap)
	87	+{
	88	+ if (pgmap->ops && pgmap->ops->cleanup) {
	89	+ pgmap->ops->cleanup(pgmap);
	90	+ } else {
	91	+ wait_for_completion(&pgmap->done);
	92	+ percpu_ref_exit(pgmap->ref);
	93	+ }
	94	+}
	95	+
86	96	static void devm_memremap_pages_release(void *data)
87	97	{
88	98	struct dev_pagemap *pgmap = data;
@@ -92,10 +102,10 @@ static void devm_memremap_pages_release(void *data) · 3 unmodified lines
92	102	unsigned long pfn;
93	103	int nid;
94	104
95		− pgmap->kill(pgmap->ref);
	105	+ dev_pagemap_kill(pgmap);
96	106	for_each_device_pfn(pfn, pgmap)
97	107	put_page(pfn_to_page(pfn));
98		− pgmap->cleanup(pgmap->ref);
	108	+ dev_pagemap_cleanup(pgmap);
99	109
100	110	/* pages are dead and unused, undo the arch mapping */
101	111	align_start = res->start & ~(SECTION_SIZE - 1);
@@ -111,7 +121,7 @@ static void devm_memremap_pages_release(void *data) · 9 unmodified lines
111	121	align_size >> PAGE_SHIFT, NULL);
112	122	} else {
113	123	arch_remove_memory(nid, align_start, align_size,
114		− pgmap->altmap_valid ? &pgmap->altmap : NULL);
	124	+ pgmap_altmap(pgmap));
115	125	kasan_remove_zero_shadow(__va(align_start), align_size);
116	126	}
117	127	mem_hotplug_done();
@@ -122,20 +132,29 @@ static void devm_memremap_pages_release(void *data) · 4 unmodified lines
122	132	"%s: failed to free all reserved pages\n", __func__);
123	133	}
124	134
	135	+static void dev_pagemap_percpu_release(struct percpu_ref *ref)
	136	+{
	137	+ struct dev_pagemap *pgmap =
	138	+ container_of(ref, struct dev_pagemap, internal_ref);
	139	+
	140	+ complete(&pgmap->done);
	141	+}
	142	+
125	143	/**
126	144	* devm_memremap_pages - remap and provide memmap backing for the given resource
127	145	* @dev: hosting device for @res
128	146	* @pgmap: pointer to a struct dev_pagemap
129	147	*
130	148	* Notes:
131		− * 1/ At a minimum the res, ref and type members of @pgmap must be initialized
	149	+ * 1/ At a minimum the res and type members of @pgmap must be initialized
132	150	* by the caller before passing it to this function
133	151	*
134		− * 2/ The altmap field may optionally be initialized, in which case altmap_valid
135		− * must be set to true
	152	+ * 2/ The altmap field may optionally be initialized, in which case
	153	+ * PGMAP_ALTMAP_VALID must be set in pgmap->flags.
136	154	*
137		− * 3/ pgmap->ref must be 'live' on entry and will be killed and reaped
138		− * at devm_memremap_pages_release() time, or if this routine fails.
	155	+ * 3/ The ref field may optionally be provided, in which pgmap->ref must be
	156	+ * 'live' on entry and will be killed and reaped at
	157	+ * devm_memremap_pages_release() time, or if this routine fails.
139	158	*
140	159	* 4/ res is expected to be a host memory range that could feasibly be
141	160	* treated as a "System RAM" range, i.e. not a device mmio range, but
@@ -144,22 +163,66 @@ static void devm_memremap_pages_release(void *data) · 2 unmodified lines
144	163	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap)
145	164	{
146	165	resource_size_t align_start, align_size, align_end;
147		− struct vmem_altmap *altmap = pgmap->altmap_valid ?
148		− &pgmap->altmap : NULL;
149	166	struct resource *res = &pgmap->res;
150	167	struct dev_pagemap *conflict_pgmap;
151	168	struct mhp_restrictions restrictions = {
152	169	/*
153	170	* We do not want any optional features only our own memmap
154	171	*/
155		− .altmap = altmap,
	172	+ .altmap = pgmap_altmap(pgmap),
156	173	};
157	174	pgprot_t pgprot = PAGE_KERNEL;
158	175	int error, nid, is_ram;
	176	+ bool need_devmap_managed = true;
	177	+
	178	+ switch (pgmap->type) {
	179	+ case MEMORY_DEVICE_PRIVATE:
	180	+ if (!IS_ENABLED(CONFIG_DEVICE_PRIVATE)) {
	181	+ WARN(1, "Device private memory not supported\n");
	182	+ return ERR_PTR(-EINVAL);
	183	+ }
	184	+ if (!pgmap->ops \|\| !pgmap->ops->migrate_to_ram) {
	185	+ WARN(1, "Missing migrate_to_ram method\n");
	186	+ return ERR_PTR(-EINVAL);
	187	+ }
	188	+ break;
	189	+ case MEMORY_DEVICE_FS_DAX:
	190	+ if (!IS_ENABLED(CONFIG_ZONE_DEVICE) \|\|
	191	+ IS_ENABLED(CONFIG_FS_DAX_LIMITED)) {
	192	+ WARN(1, "File system DAX not supported\n");
	193	+ return ERR_PTR(-EINVAL);
	194	+ }
	195	+ break;
	196	+ case MEMORY_DEVICE_DEVDAX:
	197	+ case MEMORY_DEVICE_PCI_P2PDMA:
	198	+ need_devmap_managed = false;
	199	+ break;
	200	+ default:
	201	+ WARN(1, "Invalid pgmap type %d\n", pgmap->type);
	202	+ break;
	203	+ }
	204	+
	205	+ if (!pgmap->ref) {
	206	+ if (pgmap->ops && (pgmap->ops->kill \|\| pgmap->ops->cleanup))
	207	+ return ERR_PTR(-EINVAL);
	208	+
	209	+ init_completion(&pgmap->done);
	210	+ error = percpu_ref_init(&pgmap->internal_ref,
	211	+ dev_pagemap_percpu_release, 0, GFP_KERNEL);
	212	+ if (error)
	213	+ return ERR_PTR(error);
	214	+ pgmap->ref = &pgmap->internal_ref;
	215	+ } else {
	216	+ if (!pgmap->ops \|\| !pgmap->ops->kill \|\| !pgmap->ops->cleanup) {
	217	+ WARN(1, "Missing reference count teardown definition\n");
	218	+ return ERR_PTR(-EINVAL);
	219	+ }
	220	+ }
159	221
160		− if (!pgmap->ref \|\| !pgmap->kill \|\| !pgmap->cleanup) {
161		− WARN(1, "Missing reference count teardown definition\n");
162		− return ERR_PTR(-EINVAL);
	222	+ if (need_devmap_managed) {
	223	+ error = devmap_managed_enable_get(dev, pgmap);
	224	+ if (error)
	225	+ return ERR_PTR(error);
163	226	}
164	227
165	228	align_start = res->start & ~(SECTION_SIZE - 1);
@@ -241,7 +304,7 @@ void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap) · 75 unmodified lines
241	304
242	305	zone = &NODE_DATA(nid)->node_zones[ZONE_DEVICE];
243	306	move_pfn_range_to_zone(zone, align_start >> PAGE_SHIFT,
244		− align_size >> PAGE_SHIFT, altmap);
	307	+ align_size >> PAGE_SHIFT, pgmap_altmap(pgmap));
245	308	}
246	309
247	310	mem_hotplug_done();
@@ -271,9 +334,8 @@ void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap) · 23 unmodified lines
271	334	err_pfn_remap:
272	335	pgmap_array_delete(res);
273	336	err_array:
274		− pgmap->kill(pgmap->ref);
275		− pgmap->cleanup(pgmap->ref);
276		−
	337	+ dev_pagemap_kill(pgmap);
	338	+ dev_pagemap_cleanup(pgmap);
277	339	return ERR_PTR(error);
278	340	}
279	341	EXPORT_SYMBOL_GPL(devm_memremap_pages);
@@ -287,7 +349,9 @@ EXPORT_SYMBOL_GPL(devm_memunmap_pages); · 7 unmodified lines
287	349	unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
288	350	{
289	351	/* number of pfns from base where pfn_to_page() is valid */
290		− return altmap->reserve + altmap->free;
	352	+ if (altmap)
	353	+ return altmap->reserve + altmap->free;
	354	+ return 0;
291	355	}
292	356
293	357	void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns)
@@ -329,28 +393,6 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, · 35 unmodified lines
329	393	EXPORT_SYMBOL_GPL(get_dev_pagemap);
330	394
331	395	#ifdef CONFIG_DEV_PAGEMAP_OPS
332		−DEFINE_STATIC_KEY_FALSE(devmap_managed_key);
333		−EXPORT_SYMBOL(devmap_managed_key);
334		−static atomic_t devmap_enable;
335		−
336		−/*
337		− * Toggle the static key for ->page_free() callbacks when dev_pagemap
338		− * pages go idle.
339		− */
340		−void dev_pagemap_get_ops(void)
341		−{
342		− if (atomic_inc_return(&devmap_enable) == 1)
343		− static_branch_enable(&devmap_managed_key);
344		−}
345		−EXPORT_SYMBOL_GPL(dev_pagemap_get_ops);
346		−
347		−void dev_pagemap_put_ops(void)
348		−{
349		− if (atomic_dec_and_test(&devmap_enable))
350		− static_branch_disable(&devmap_managed_key);
351		−}
352		−EXPORT_SYMBOL_GPL(dev_pagemap_put_ops);
353		−
354	396	void __put_devmap_managed_page(struct page *page)
355	397	{
356	398	int count = page_ref_dec_return(page);
@@ -366,7 +408,7 @@ void __put_devmap_managed_page(struct page *page) · 9 unmodified lines
366	408
367	409	mem_cgroup_uncharge(page);
368	410
369		− page->pgmap->page_free(page, page->pgmap->data);
	411	+ page->pgmap->ops->page_free(page);
370	412	} else if (!count)
371	413	__put_page(page);
372	414	}

kernel/resource.c+39−0

@@ -1628,6 +1628,45 @@ void resource_list_free(struct list_head *head) · 1627 unmodified lines
1628	1628	}
1629	1629	EXPORT_SYMBOL(resource_list_free);
1630	1630
	1631	+#ifdef CONFIG_DEVICE_PRIVATE
	1632	+/**
	1633	+ * devm_request_free_mem_region - find free region for device private memory
	1634	+ *
	1635	+ * @dev: device struct to bind the resource to
	1636	+ * @size: size in bytes of the device memory to add
	1637	+ * @base: resource tree to look in
	1638	+ *
	1639	+ * This function tries to find an empty range of physical address big enough to
	1640	+ * contain the new resource, so that it can later be hotplugged as ZONE_DEVICE
	1641	+ * memory, which in turn allocates struct pages.
	1642	+ */
	1643	+struct resource devm_request_free_mem_region(struct device dev,
	1644	+ struct resource *base, unsigned long size)
	1645	+{
	1646	+ resource_size_t end, addr;
	1647	+ struct resource *res;
	1648	+
	1649	+ size = ALIGN(size, 1UL << PA_SECTION_SHIFT);
	1650	+ end = min_t(unsigned long, base->end, (1UL << MAX_PHYSMEM_BITS) - 1);
	1651	+ addr = end - size + 1UL;
	1652	+
	1653	+ for (; addr > size && addr >= base->start; addr -= size) {
	1654	+ if (region_intersects(addr, size, 0, IORES_DESC_NONE) !=
	1655	+ REGION_DISJOINT)
	1656	+ continue;
	1657	+
	1658	+ res = devm_request_mem_region(dev, addr, size, dev_name(dev));
	1659	+ if (!res)
	1660	+ return ERR_PTR(-ENOMEM);
	1661	+ res->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
	1662	+ return res;
	1663	+ }
	1664	+
	1665	+ return ERR_PTR(-ERANGE);
	1666	+}
	1667	+EXPORT_SYMBOL_GPL(devm_request_free_mem_region);
	1668	+#endif /* CONFIG_DEVICE_PRIVATE */
	1669	+
1631	1670	static int __init strict_iomem(char *str)
1632	1671	{
1633	1672	if (strstr(str, "relaxed"))

mm/Kconfig+4−46

@@ -669,47 +669,17 @@ config ZONE_DEVICE · 668 unmodified lines
669	669
670	670	If FS_DAX is enabled, then say Y.
671	671
672		−config ARCH_HAS_HMM_MIRROR
673		− bool
674		− default y
675		− depends on (X86_64 \|\| PPC64)
676		− depends on MMU && 64BIT
677		−
678		−config ARCH_HAS_HMM_DEVICE
679		− bool
680		− default y
681		− depends on (X86_64 \|\| PPC64)
682		− depends on MEMORY_HOTPLUG
683		− depends on MEMORY_HOTREMOVE
684		− depends on SPARSEMEM_VMEMMAP
685		− depends on ARCH_HAS_ZONE_DEVICE
686		− select XARRAY_MULTI
687		−
688		−config ARCH_HAS_HMM
689		− bool
690		− default y
691		− depends on (X86_64 \|\| PPC64)
692		− depends on ZONE_DEVICE
693		− depends on MMU && 64BIT
694		− depends on MEMORY_HOTPLUG
695		− depends on MEMORY_HOTREMOVE
696		− depends on SPARSEMEM_VMEMMAP
697		−
698	672	config MIGRATE_VMA_HELPER
699	673	bool
700	674
701	675	config DEV_PAGEMAP_OPS
702	676	bool
703	677
704		−config HMM
705		− bool
706		− select MMU_NOTIFIER
707		− select MIGRATE_VMA_HELPER
708		−
709	678	config HMM_MIRROR
710	679	bool "HMM mirror CPU page table into a device page table"
711		− depends on ARCH_HAS_HMM
712		− select HMM
	680	+ depends on (X86_64 \|\| PPC64)
	681	+ depends on MMU && 64BIT
	682	+ select MMU_NOTIFIER
713	683	help
714	684	Select HMM_MIRROR if you want to mirror range of the CPU page table of a
715	685	process into a device page table. Here, mirror means "keep synchronized".
@@ -719,8 +689,7 @@ config HMM_MIRROR · 3 unmodified lines
719	689
720	690	config DEVICE_PRIVATE
721	691	bool "Unaddressable device memory (GPU memory, ...)"
722		− depends on ARCH_HAS_HMM
723		− select HMM
	692	+ depends on ZONE_DEVICE
724	693	select DEV_PAGEMAP_OPS
725	694
726	695	help
@@ -728,17 +697,6 @@ config DEVICE_PRIVATE · 1 unmodified line
728	697	memory; i.e., memory that is only accessible from the device (or
729	698	group of devices). You likely also want to select HMM_MIRROR.
730	699
731		−config DEVICE_PUBLIC
732		− bool "Addressable device memory (like GPU memory)"
733		− depends on ARCH_HAS_HMM
734		− select HMM
735		− select DEV_PAGEMAP_OPS
736		−
737		− help
738		− Allows creation of struct pages to represent addressable device
739		− memory; i.e., memory that is accessible from both the device and
740		− the CPU
741		−
742	700	config FRAME_VECTOR
743	701	bool
744	702

mm/Makefile+1−1

@@ -102,5 +102,5 @@ obj-$(CONFIG_FRAME_VECTOR) += frame_vector.o · 101 unmodified lines
102	102	obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o
103	103	obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o
104	104	obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o
105		−obj-$(CONFIG_HMM) += hmm.o
	105	+obj-$(CONFIG_HMM_MIRROR) += hmm.o
106	106	obj-$(CONFIG_MEMFD_CREATE) += memfd.o

mm/gup.c+0−7

@@ -605,13 +605,6 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address, · 604 unmodified lines
605	605	if ((gup_flags & FOLL_DUMP) \|\| !is_zero_pfn(pte_pfn(*pte)))
606	606	goto unmap;
607	607	page = pte_page(pte);
608		−
609		− /*
610		− * This should never happen (a device public page in the gate
611		− * area).
612		− */
613		− if (is_device_public_page(*page))
614		− goto unmap;
615	608	}
616	609	if (unlikely(!try_get_page(*page))) {
617	610	ret = -ENOMEM;

mm/hmm.c+0−284

@@ -26,9 +26,6 @@ · 25 unmodified lines
26	26	#include <linux/mmu_notifier.h>
27	27	#include <linux/memory_hotplug.h>
28	28
29		−#define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)
30		−
31		−#if IS_ENABLED(CONFIG_HMM_MIRROR)
32	29	static const struct mmu_notifier_ops hmm_mmu_notifier_ops;
33	30
34	31	/**
@@ -1287,284 +1284,3 @@ long hmm_range_dma_unmap(struct hmm_range *range, · 1252 unmodified lines
1287	1284	return cpages;
1288	1285	}
1289	1286	EXPORT_SYMBOL(hmm_range_dma_unmap);
1290		−#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
1291		−
1292		−
1293		−#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) \|\| IS_ENABLED(CONFIG_DEVICE_PUBLIC)
1294		−struct page hmm_vma_alloc_locked_page(struct vm_area_struct vma,
1295		− unsigned long addr)
1296		−{
1297		− struct page *page;
1298		−
1299		− page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
1300		− if (!page)
1301		− return NULL;
1302		− lock_page(page);
1303		− return page;
1304		−}
1305		−EXPORT_SYMBOL(hmm_vma_alloc_locked_page);
1306		−
1307		−
1308		−static void hmm_devmem_ref_release(struct percpu_ref *ref)
1309		−{
1310		− struct hmm_devmem *devmem;
1311		−
1312		− devmem = container_of(ref, struct hmm_devmem, ref);
1313		− complete(&devmem->completion);
1314		−}
1315		−
1316		−static void hmm_devmem_ref_exit(struct percpu_ref *ref)
1317		−{
1318		− struct hmm_devmem *devmem;
1319		−
1320		− devmem = container_of(ref, struct hmm_devmem, ref);
1321		− wait_for_completion(&devmem->completion);
1322		− percpu_ref_exit(ref);
1323		−}
1324		−
1325		−static void hmm_devmem_ref_kill(struct percpu_ref *ref)
1326		−{
1327		− percpu_ref_kill(ref);
1328		−}
1329		−
1330		−static vm_fault_t hmm_devmem_fault(struct vm_area_struct *vma,
1331		− unsigned long addr,
1332		− const struct page *page,
1333		− unsigned int flags,
1334		− pmd_t *pmdp)
1335		−{
1336		− struct hmm_devmem *devmem = page->pgmap->data;
1337		−
1338		− return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp);
1339		−}
1340		−
1341		−static void hmm_devmem_free(struct page page, void data)
1342		−{
1343		− struct hmm_devmem *devmem = data;
1344		−
1345		− page->mapping = NULL;
1346		−
1347		− devmem->ops->free(devmem, page);
1348		−}
1349		−
1350		−/*
1351		− * hmm_devmem_add() - hotplug ZONE_DEVICE memory for device memory
1352		− *
1353		− * @ops: memory event device driver callback (see struct hmm_devmem_ops)
1354		− * @device: device struct to bind the resource too
1355		− * @size: size in bytes of the device memory to add
1356		− * Return: pointer to new hmm_devmem struct ERR_PTR otherwise
1357		− *
1358		− * This function first finds an empty range of physical address big enough to
1359		− * contain the new resource, and then hotplugs it as ZONE_DEVICE memory, which
1360		− * in turn allocates struct pages. It does not do anything beyond that; all
1361		− * events affecting the memory will go through the various callbacks provided
1362		− * by hmm_devmem_ops struct.
1363		− *
1364		− * Device driver should call this function during device initialization and
1365		− * is then responsible of memory management. HMM only provides helpers.
1366		− */
1367		−struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops,
1368		− struct device *device,
1369		− unsigned long size)
1370		−{
1371		− struct hmm_devmem *devmem;
1372		− resource_size_t addr;
1373		− void *result;
1374		− int ret;
1375		−
1376		− dev_pagemap_get_ops();
1377		−
1378		− devmem = devm_kzalloc(device, sizeof(*devmem), GFP_KERNEL);
1379		− if (!devmem)
1380		− return ERR_PTR(-ENOMEM);
1381		−
1382		− init_completion(&devmem->completion);
1383		− devmem->pfn_first = -1UL;
1384		− devmem->pfn_last = -1UL;
1385		− devmem->resource = NULL;
1386		− devmem->device = device;
1387		− devmem->ops = ops;
1388		−
1389		− ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1390		− 0, GFP_KERNEL);
1391		− if (ret)
1392		− return ERR_PTR(ret);
1393		−
1394		− size = ALIGN(size, PA_SECTION_SIZE);
1395		− addr = min((unsigned long)iomem_resource.end,
1396		− (1UL << MAX_PHYSMEM_BITS) - 1);
1397		− addr = addr - size + 1UL;
1398		−
1399		− /*
1400		− * FIXME add a new helper to quickly walk resource tree and find free
1401		− * range
1402		− *
1403		− * FIXME what about ioport_resource resource ?
1404		− */
1405		− for (; addr > size && addr >= iomem_resource.start; addr -= size) {
1406		− ret = region_intersects(addr, size, 0, IORES_DESC_NONE);
1407		− if (ret != REGION_DISJOINT)
1408		− continue;
1409		−
1410		− devmem->resource = devm_request_mem_region(device, addr, size,
1411		− dev_name(device));
1412		− if (!devmem->resource)
1413		− return ERR_PTR(-ENOMEM);
1414		− break;
1415		− }
1416		− if (!devmem->resource)
1417		− return ERR_PTR(-ERANGE);
1418		−
1419		− devmem->resource->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
1420		− devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1421		− devmem->pfn_last = devmem->pfn_first +
1422		− (resource_size(devmem->resource) >> PAGE_SHIFT);
1423		− devmem->page_fault = hmm_devmem_fault;
1424		−
1425		− devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
1426		− devmem->pagemap.res = *devmem->resource;
1427		− devmem->pagemap.page_free = hmm_devmem_free;
1428		− devmem->pagemap.altmap_valid = false;
1429		− devmem->pagemap.ref = &devmem->ref;
1430		− devmem->pagemap.data = devmem;
1431		− devmem->pagemap.kill = hmm_devmem_ref_kill;
1432		− devmem->pagemap.cleanup = hmm_devmem_ref_exit;
1433		−
1434		− result = devm_memremap_pages(devmem->device, &devmem->pagemap);
1435		− if (IS_ERR(result))
1436		− return result;
1437		− return devmem;
1438		−}
1439		−EXPORT_SYMBOL_GPL(hmm_devmem_add);
1440		−
1441		−struct hmm_devmem hmm_devmem_add_resource(const struct hmm_devmem_ops ops,
1442		− struct device *device,
1443		− struct resource *res)
1444		−{
1445		− struct hmm_devmem *devmem;
1446		− void *result;
1447		− int ret;
1448		−
1449		− if (res->desc != IORES_DESC_DEVICE_PUBLIC_MEMORY)
1450		− return ERR_PTR(-EINVAL);
1451		−
1452		− dev_pagemap_get_ops();
1453		−
1454		− devmem = devm_kzalloc(device, sizeof(*devmem), GFP_KERNEL);
1455		− if (!devmem)
1456		− return ERR_PTR(-ENOMEM);
1457		−
1458		− init_completion(&devmem->completion);
1459		− devmem->pfn_first = -1UL;
1460		− devmem->pfn_last = -1UL;
1461		− devmem->resource = res;
1462		− devmem->device = device;
1463		− devmem->ops = ops;
1464		−
1465		− ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1466		− 0, GFP_KERNEL);
1467		− if (ret)
1468		− return ERR_PTR(ret);
1469		−
1470		− devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1471		− devmem->pfn_last = devmem->pfn_first +
1472		− (resource_size(devmem->resource) >> PAGE_SHIFT);
1473		− devmem->page_fault = hmm_devmem_fault;
1474		−
1475		− devmem->pagemap.type = MEMORY_DEVICE_PUBLIC;
1476		− devmem->pagemap.res = *devmem->resource;
1477		− devmem->pagemap.page_free = hmm_devmem_free;
1478		− devmem->pagemap.altmap_valid = false;
1479		− devmem->pagemap.ref = &devmem->ref;
1480		− devmem->pagemap.data = devmem;
1481		− devmem->pagemap.kill = hmm_devmem_ref_kill;
1482		− devmem->pagemap.cleanup = hmm_devmem_ref_exit;
1483		−
1484		− result = devm_memremap_pages(devmem->device, &devmem->pagemap);
1485		− if (IS_ERR(result))
1486		− return result;
1487		− return devmem;
1488		−}
1489		−EXPORT_SYMBOL_GPL(hmm_devmem_add_resource);
1490		−
1491		−/*
1492		− * A device driver that wants to handle multiple devices memory through a
1493		− * single fake device can use hmm_device to do so. This is purely a helper
1494		− * and it is not needed to make use of any HMM functionality.
1495		− */
1496		−#define HMM_DEVICE_MAX 256
1497		−
1498		−static DECLARE_BITMAP(hmm_device_mask, HMM_DEVICE_MAX);
1499		−static DEFINE_SPINLOCK(hmm_device_lock);
1500		−static struct class *hmm_device_class;
1501		−static dev_t hmm_device_devt;
1502		−
1503		−static void hmm_device_release(struct device *device)
1504		−{
1505		− struct hmm_device *hmm_device;
1506		−
1507		− hmm_device = container_of(device, struct hmm_device, device);
1508		− spin_lock(&hmm_device_lock);
1509		− clear_bit(hmm_device->minor, hmm_device_mask);
1510		− spin_unlock(&hmm_device_lock);
1511		−
1512		− kfree(hmm_device);
1513		−}
1514		−
1515		−struct hmm_device hmm_device_new(void drvdata)
1516		−{
1517		− struct hmm_device *hmm_device;
1518		−
1519		− hmm_device = kzalloc(sizeof(*hmm_device), GFP_KERNEL);
1520		− if (!hmm_device)
1521		− return ERR_PTR(-ENOMEM);
1522		−
1523		− spin_lock(&hmm_device_lock);
1524		− hmm_device->minor = find_first_zero_bit(hmm_device_mask, HMM_DEVICE_MAX);
1525		− if (hmm_device->minor >= HMM_DEVICE_MAX) {
1526		− spin_unlock(&hmm_device_lock);
1527		− kfree(hmm_device);
1528		− return ERR_PTR(-EBUSY);
1529		− }
1530		− set_bit(hmm_device->minor, hmm_device_mask);
1531		− spin_unlock(&hmm_device_lock);
1532		−
1533		− dev_set_name(&hmm_device->device, "hmm_device%d", hmm_device->minor);
1534		− hmm_device->device.devt = MKDEV(MAJOR(hmm_device_devt),
1535		− hmm_device->minor);
1536		− hmm_device->device.release = hmm_device_release;
1537		− dev_set_drvdata(&hmm_device->device, drvdata);
1538		− hmm_device->device.class = hmm_device_class;
1539		− device_initialize(&hmm_device->device);
1540		−
1541		− return hmm_device;
1542		−}
1543		−EXPORT_SYMBOL(hmm_device_new);
1544		−
1545		−void hmm_device_put(struct hmm_device *hmm_device)
1546		−{
1547		− put_device(&hmm_device->device);
1548		−}
1549		−EXPORT_SYMBOL(hmm_device_put);
1550		−
1551		−static int __init hmm_init(void)
1552		−{
1553		− int ret;
1554		−
1555		− ret = alloc_chrdev_region(&hmm_device_devt, 0,
1556		− HMM_DEVICE_MAX,
1557		− "hmm_device");
1558		− if (ret)
1559		− return ret;
1560		−
1561		− hmm_device_class = class_create(THIS_MODULE, "hmm_device");
1562		− if (IS_ERR(hmm_device_class)) {
1563		− unregister_chrdev_region(hmm_device_devt, HMM_DEVICE_MAX);
1564		− return PTR_ERR(hmm_device_class);
1565		− }
1566		− return 0;
1567		−}
1568		−
1569		−device_initcall(hmm_init);
1570		−#endif /* CONFIG_DEVICE_PRIVATE \|\| CONFIG_DEVICE_PUBLIC */

mm/madvise.c+1−1

@@ -354,7 +354,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, · 353 unmodified lines
354	354	continue;
355	355	}
356	356
357		− page = _vm_normal_page(vma, addr, ptent, true);
	357	+ page = vm_normal_page(vma, addr, ptent);
358	358	if (!page)
359	359	continue;
360	360

mm/memcontrol.c+6−7

@@ -4793,7 +4793,7 @@ enum mc_target_type { · 4792 unmodified lines
4793	4793	static struct page mc_handle_present_pte(struct vm_area_struct vma,
4794	4794	unsigned long addr, pte_t ptent)
4795	4795	{
4796		− struct page *page = _vm_normal_page(vma, addr, ptent, true);
	4796	+ struct page *page = vm_normal_page(vma, addr, ptent);
4797	4797
4798	4798	if (!page \|\| !page_mapped(page))
4799	4799	return NULL;
@@ -4994,8 +4994,8 @@ out: · 194 unmodified lines
4994	4994	* 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4995	4995	* target for charge migration. if @target is not NULL, the entry is stored
4996	4996	* in target->ent.
4997		− * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PUBLIC
4998		− * or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru).
	4997	+ * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE
	4998	+ * (so ZONE_DEVICE page and thus not on the lru).
4999	4999	* For now we such page is charge like a regular page would be as for all
5000	5000	* intent and purposes it is just special memory taking the place of a
5001	5001	* regular page.
@@ -5029,8 +5029,7 @@ static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, · 27 unmodified lines
5029	5029	*/
5030	5030	if (page->mem_cgroup == mc.from) {
5031	5031	ret = MC_TARGET_PAGE;
5032		− if (is_device_private_page(page) \|\|
5033		− is_device_public_page(page))
	5032	+ if (is_device_private_page(page))
5034	5033	ret = MC_TARGET_DEVICE;
5035	5034	if (target)
5036	5035	target->page = page;
@@ -5101,8 +5100,8 @@ static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, · 64 unmodified lines
5101	5100	if (ptl) {
5102	5101	/*
5103	5102	* Note their can not be MC_TARGET_DEVICE for now as we do not
5104		− * support transparent huge page with MEMORY_DEVICE_PUBLIC or
5105		− * MEMORY_DEVICE_PRIVATE but this might change.
	5103	+ * support transparent huge page with MEMORY_DEVICE_PRIVATE but
	5104	+ * this might change.
5106	5105	*/
5107	5106	if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
5108	5107	mc.precharge += HPAGE_PMD_NR;

mm/memory-failure.c+1−5

@@ -1177,16 +1177,12 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags, · 1176 unmodified lines
1177	1177	goto unlock;
1178	1178	}
1179	1179
1180		− switch (pgmap->type) {
1181		− case MEMORY_DEVICE_PRIVATE:
1182		− case MEMORY_DEVICE_PUBLIC:
	1180	+ if (pgmap->type == MEMORY_DEVICE_PRIVATE) {
1183	1181	/*
1184	1182	* TODO: Handle HMM pages which may need coordination
1185	1183	* with device-side memory.
1186	1184	*/
1187	1185	goto unlock;
1188		− default:
1189		− break;
1190	1186	}
1191	1187
1192	1188	/*

mm/memory.c+5−44

@@ -571,8 +571,8 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, · 570 unmodified lines
571	571	* PFNMAP mappings in order to support COWable mappings.
572	572	*
573	573	*/
574		−struct page _vm_normal_page(struct vm_area_struct vma, unsigned long addr,
575		− pte_t pte, bool with_public_device)
	574	+struct page vm_normal_page(struct vm_area_struct vma, unsigned long addr,
	575	+ pte_t pte)
576	576	{
577	577	unsigned long pfn = pte_pfn(pte);
578	578
@@ -585,29 +585,6 @@ struct page _vm_normal_page(struct vm_area_struct vma, unsigned long addr, · 6 unmodified lines
585	585	return NULL;
586	586	if (is_zero_pfn(pfn))
587	587	return NULL;
588		−
589		− /*
590		− * Device public pages are special pages (they are ZONE_DEVICE
591		− * pages but different from persistent memory). They behave
592		− * allmost like normal pages. The difference is that they are
593		− * not on the lru and thus should never be involve with any-
594		− * thing that involve lru manipulation (mlock, numa balancing,
595		− * ...).
596		− *
597		− * This is why we still want to return NULL for such page from
598		− * vm_normal_page() so that we do not have to special case all
599		− * call site of vm_normal_page().
600		− */
601		− if (likely(pfn <= highest_memmap_pfn)) {
602		− struct page *page = pfn_to_page(pfn);
603		−
604		− if (is_device_public_page(page)) {
605		− if (with_public_device)
606		− return page;
607		− return NULL;
608		− }
609		− }
610		−
611	588	if (pte_devmap(pte))
612	589	return NULL;
613	590
@@ -797,17 +774,6 @@ copy_one_pte(struct mm_struct dst_mm, struct mm_struct src_mm, · 183 unmodified lines
797	774	rss[mm_counter(page)]++;
798	775	} else if (pte_devmap(pte)) {
799	776	page = pte_page(pte);
800		−
801		− /*
802		− * Cache coherent device memory behave like regular page and
803		− * not like persistent memory page. For more informations see
804		− * MEMORY_DEVICE_CACHE_COHERENT in memory_hotplug.h
805		− */
806		− if (is_device_public_page(page)) {
807		− get_page(page);
808		− page_dup_rmap(page, false);
809		− rss[mm_counter(page)]++;
810		− }
811	777	}
812	778
813	779	out_set_pte:
@@ -1063,7 +1029,7 @@ again: · 249 unmodified lines
1063	1029	if (pte_present(ptent)) {
1064	1030	struct page *page;
1065	1031
1066		− page = _vm_normal_page(vma, addr, ptent, true);
	1032	+ page = vm_normal_page(vma, addr, ptent);
1067	1033	if (unlikely(details) && page) {
1068	1034	/*
1069	1035	* unmap_shared_mapping_pages() wants to
@@ -2782,13 +2748,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) · 1712 unmodified lines
2782	2748	migration_entry_wait(vma->vm_mm, vmf->pmd,
2783	2749	vmf->address);
2784	2750	} else if (is_device_private_entry(entry)) {
2785		− /*
2786		− * For un-addressable device memory we call the pgmap
2787		− * fault handler callback. The callback must migrate
2788		− * the page back to some CPU accessible page.
2789		− */
2790		− ret = device_private_entry_fault(vma, vmf->address, entry,
2791		− vmf->flags, vmf->pmd);
	2751	+ vmf->page = device_private_entry_to_page(entry);
	2752	+ ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
2792	2753	} else if (is_hwpoison_entry(entry)) {
2793	2754	ret = VM_FAULT_HWPOISON;
2794	2755	} else {

mm/memory_hotplug.c+2−4

@@ -557,10 +557,8 @@ void __remove_pages(struct zone *zone, unsigned long phys_start_pfn, · 556 unmodified lines
557	557	int sections_to_remove;
558	558
559	559	/* In the ZONE_DEVICE case device driver owns the memory region */
560		− if (is_dev_zone(zone)) {
561		− if (altmap)
562		− map_offset = vmem_altmap_offset(altmap);
563		− }
	560	+ if (is_dev_zone(zone))
	561	+ map_offset = vmem_altmap_offset(altmap);
564	562
565	563	clear_zone_contiguous(zone);
566	564

mm/mempolicy.c+1−0

@@ -2098,6 +2098,7 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma, · 2097 unmodified lines
2098	2098	out:
2099	2099	return page;
2100	2100	}
	2101	+EXPORT_SYMBOL(alloc_pages_vma);
2101	2102
2102	2103	/**
2103	2104	* alloc_pages_current - Allocate pages.

mm/migrate.c+4−24

@@ -246,8 +246,6 @@ static bool remove_migration_pte(struct page page, struct vm_area_struct vma, · 245 unmodified lines
246	246	if (is_device_private_page(new)) {
247	247	entry = make_device_private_entry(new, pte_write(pte));
248	248	pte = swp_entry_to_pte(entry);
249		− } else if (is_device_public_page(new)) {
250		− pte = pte_mkdevmap(pte);
251	249	}
252	250	}
253	251
@@ -381,7 +379,6 @@ static int expected_page_refs(struct address_space mapping, struct page page) · 127 unmodified lines
381	379	* ZONE_DEVICE pages.
382	380	*/
383	381	expected_count += is_device_private_page(page);
384		− expected_count += is_device_public_page(page);
385	382	if (mapping)
386	383	expected_count += hpage_nr_pages(page) + page_has_private(page);
387	384
@@ -994,10 +991,7 @@ static int move_to_new_page(struct page newpage, struct page page, · 606 unmodified lines
994	991	if (!PageMappingFlags(page))
995	992	page->mapping = NULL;
996	993
997		− if (unlikely(is_zone_device_page(newpage))) {
998		− if (is_device_public_page(newpage))
999		− flush_dcache_page(newpage);
1000		− } else
	994	+ if (likely(!is_zone_device_page(newpage)))
1001	995	flush_dcache_page(newpage);
1002	996
1003	997	}
@@ -2265,7 +2259,7 @@ again: · 1261 unmodified lines
2265	2259	pfn = 0;
2266	2260	goto next;
2267	2261	}
2268		− page = _vm_normal_page(migrate->vma, addr, pte, true);
	2262	+ page = vm_normal_page(migrate->vma, addr, pte);
2269	2263	mpfn = migrate_pfn(pfn) \| MIGRATE_PFN_MIGRATE;
2270	2264	mpfn \|= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
2271	2265	}
@@ -2406,16 +2400,7 @@ static bool migrate_vma_check_page(struct page *page) · 134 unmodified lines
2406	2400	* FIXME proper solution is to rework migration_entry_wait() so
2407	2401	* it does not need to take a reference on page.
2408	2402	*/
2409		− if (is_device_private_page(page))
2410		− return true;
2411		−
2412		− /*
2413		− * Only allow device public page to be migrated and account for
2414		− * the extra reference count imply by ZONE_DEVICE pages.
2415		− */
2416		− if (!is_device_public_page(page))
2417		− return false;
2418		− extra++;
	2403	+ return is_device_private_page(page);
2419	2404	}
2420	2405
2421	2406	/* For file back page */
@@ -2665,11 +2650,6 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, · 243 unmodified lines
2665	2650
2666	2651	swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
2667	2652	entry = swp_entry_to_pte(swp_entry);
2668		− } else if (is_device_public_page(page)) {
2669		− entry = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
2670		− if (vma->vm_flags & VM_WRITE)
2671		− entry = pte_mkwrite(pte_mkdirty(entry));
2672		− entry = pte_mkdevmap(entry);
2673	2653	}
2674	2654	} else {
2675	2655	entry = mk_pte(page, vma->vm_page_prot);
@@ -2789,7 +2769,7 @@ static void migrate_vma_pages(struct migrate_vma *migrate) · 113 unmodified lines
2789	2769	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2790	2770	continue;
2791	2771	}
2792		− } else if (!is_device_public_page(newpage)) {
	2772	+ } else {
2793	2773	/*
2794	2774	* Other types of ZONE_DEVICE page are not
2795	2775	* supported.

mm/page_alloc.c+6−7

@@ -5853,6 +5853,7 @@ void __ref memmap_init_zone_device(struct zone *zone, · 5852 unmodified lines
5853	5853	{
5854	5854	unsigned long pfn, end_pfn = start_pfn + size;
5855	5855	struct pglist_data *pgdat = zone->zone_pgdat;
	5856	+ struct vmem_altmap *altmap = pgmap_altmap(pgmap);
5856	5857	unsigned long zone_idx = zone_idx(zone);
5857	5858	unsigned long start = jiffies;
5858	5859	int nid = pgdat->node_id;
@@ -5865,9 +5866,7 @@ void __ref memmap_init_zone_device(struct zone *zone, · 6 unmodified lines
5865	5866	* of the pages reserved for the memmap, so we can just jump to
5866	5867	* the end of that region and start processing the device pages.
5867	5868	*/
5868		− if (pgmap->altmap_valid) {
5869		− struct vmem_altmap *altmap = &pgmap->altmap;
5870		−
	5869	+ if (altmap) {
5871	5870	start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
5872	5871	size = end_pfn - start_pfn;
5873	5872	}
@@ -5887,12 +5886,12 @@ void __ref memmap_init_zone_device(struct zone *zone, · 13 unmodified lines
5887	5886	__SetPageReserved(page);
5888	5887
5889	5888	/*
5890		− * ZONE_DEVICE pages union ->lru with a ->pgmap back
5891		− * pointer and hmm_data. It is a bug if a ZONE_DEVICE
5892		− * page is ever freed or placed on a driver-private list.
	5889	+ * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
	5890	+ * and zone_device_data. It is a bug if a ZONE_DEVICE page is
	5891	+ * ever freed or placed on a driver-private list.
5893	5892	*/
5894	5893	page->pgmap = pgmap;
5895		− page->hmm_data = 0;
	5894	+ page->zone_device_data = NULL;
5896	5895
5897	5896	/*
5898	5897	* Mark the block movable so that blocks are reserved for

tools/testing/nvdimm/test/iomap.c+46−11

@@ -100,25 +100,60 @@ static void nfit_test_kill(void *_pgmap) · 99 unmodified lines
100	100	{
101	101	struct dev_pagemap *pgmap = _pgmap;
102	102
103		− WARN_ON(!pgmap \|\| !pgmap->ref \|\| !pgmap->kill \|\| !pgmap->cleanup);
104		− pgmap->kill(pgmap->ref);
105		− pgmap->cleanup(pgmap->ref);
	103	+ WARN_ON(!pgmap \|\| !pgmap->ref);
	104	+
	105	+ if (pgmap->ops && pgmap->ops->kill)
	106	+ pgmap->ops->kill(pgmap);
	107	+ else
	108	+ percpu_ref_kill(pgmap->ref);
	109	+
	110	+ if (pgmap->ops && pgmap->ops->cleanup) {
	111	+ pgmap->ops->cleanup(pgmap);
	112	+ } else {
	113	+ wait_for_completion(&pgmap->done);
	114	+ percpu_ref_exit(pgmap->ref);
	115	+ }
	116	+}
	117	+
	118	+static void dev_pagemap_percpu_release(struct percpu_ref *ref)
	119	+{
	120	+ struct dev_pagemap *pgmap =
	121	+ container_of(ref, struct dev_pagemap, internal_ref);
	122	+
	123	+ complete(&pgmap->done);
106	124	}
107	125
108	126	void __wrap_devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap)
109	127	{
	128	+ int error;
110	129	resource_size_t offset = pgmap->res.start;
111	130	struct nfit_test_resource *nfit_res = get_nfit_res(offset);
112	131
113		− if (nfit_res) {
114		− int rc;
115		−
116		− rc = devm_add_action_or_reset(dev, nfit_test_kill, pgmap);
117		− if (rc)
118		− return ERR_PTR(rc);
119		− return nfit_res->buf + offset - nfit_res->res.start;
	132	+ if (!nfit_res)
	133	+ return devm_memremap_pages(dev, pgmap);
	134	+
	135	+ pgmap->dev = dev;
	136	+ if (!pgmap->ref) {
	137	+ if (pgmap->ops && (pgmap->ops->kill \|\| pgmap->ops->cleanup))
	138	+ return ERR_PTR(-EINVAL);
	139	+
	140	+ init_completion(&pgmap->done);
	141	+ error = percpu_ref_init(&pgmap->internal_ref,
	142	+ dev_pagemap_percpu_release, 0, GFP_KERNEL);
	143	+ if (error)
	144	+ return ERR_PTR(error);
	145	+ pgmap->ref = &pgmap->internal_ref;
	146	+ } else {
	147	+ if (!pgmap->ops \|\| !pgmap->ops->kill \|\| !pgmap->ops->cleanup) {
	148	+ WARN(1, "Missing reference count teardown definition\n");
	149	+ return ERR_PTR(-EINVAL);
	150	+ }
120	151	}
121		− return devm_memremap_pages(dev, pgmap);
	152	+
	153	+ error = devm_add_action_or_reset(dev, nfit_test_kill, pgmap);
	154	+ if (error)
	155	+ return ERR_PTR(error);
	156	+ return nfit_res->buf + offset - nfit_res->res.start;
122	157	}
123	158	EXPORT_SYMBOL_GPL(__wrap_devm_memremap_pages);
124	159

Merge branch 'hmm-devmem-cleanup.4' into rdma.git hmm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code