lael-labs/linux

Documentation/vm/hmm.rst+0−27

@@ -336,33 +336,6 @@ directly using struct page for device memory which left most kernel code paths · 335 unmodified lines
336	unaware of the difference. We only need to make sure that no one ever tries to	336	unaware of the difference. We only need to make sure that no one ever tries to
337	map those pages from the CPU side.	337	map those pages from the CPU side.
338		338
339	HMM provides a set of helpers to register and hotplug device memory as a new
340	region needing a struct page. This is offered through a very simple API::
341
342	struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops,
343	struct device *device,
344	unsigned long size);
345	void hmm_devmem_remove(struct hmm_devmem *devmem);
346
347	The hmm_devmem_ops is where most of the important things are::
348
349	struct hmm_devmem_ops {
350	void (free)(struct hmm_devmem devmem, struct page *page);
351	vm_fault_t (fault)(struct hmm_devmem devmem,
352	struct vm_area_struct *vma,
353	unsigned long addr,
354	struct page *page,
355	unsigned flags,
356	pmd_t *pmdp);
357	};
358
359	The first callback (free()) happens when the last reference on a device page is
360	dropped. This means the device page is now free and no longer used by anyone.
361	The second callback happens whenever the CPU tries to access a device page
362	which it cannot do. This second callback must trigger a migration back to
363	system memory.
364
365
366	Migration to and from device memory	339	Migration to and from device memory
367	===================================	340	===================================
368		341

arch/powerpc/mm/mem.c+1−9

 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
-	struct page *page;
+	struct page *page = pfn_to_page(start_pfn) + vmem_altmap_offset(altmap);
 	int ret;
-	/*
-	 * If we have an altmap then we need to skip over any reserved PFNs
-	 * when querying the zone.
-	 */
-	page = pfn_to_page(start_pfn);
-	if (altmap)
-		page += vmem_altmap_offset(altmap);
 	__remove_pages(page_zone(page), start_pfn, nr_pages, altmap);
 	/* Remove htab bolted mappings for this section of memory */

arch/x86/mm/init_64.c+2−6

 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
-	struct page *page = pfn_to_page(start_pfn);
+	struct page *page = pfn_to_page(start_pfn) + vmem_altmap_offset(altmap);
-	struct zone *zone;
+	struct zone *zone = page_zone(page);
-	/* With altmap the first mapped page is offset from @start */
-	if (altmap)
-		page += vmem_altmap_offset(altmap);
-	zone = page_zone(page);
 	__remove_pages(zone, start_pfn, nr_pages, altmap);
 	kernel_physical_mapping_remove(start, start + size);
 }

drivers/dax/dax-private.h+0−4

  * @target_node: effective numa node if dev_dax memory range is onlined
  * @dev - device core
  * @pgmap - pgmap for memmap setup / lifetime (driver owned)
- * @ref: pgmap reference count (driver owned)
- * @cmp: @ref final put completion (driver owned)
  */
 struct dev_dax {
 	struct dax_region *region;
 	int target_node;
 	struct device dev;
 	struct dev_pagemap pgmap;
-	struct percpu_ref ref;
-	struct completion cmp;
 };
 static inline struct dev_dax *to_dev_dax(struct device *dev)

drivers/dax/device.c+1−40

@@ -14,37 +14,6 @@ · 13 unmodified lines
14	#include "dax-private.h"	14	#include "dax-private.h"
15	#include "bus.h"	15	#include "bus.h"
16		16
17	static struct dev_dax ref_to_dev_dax(struct percpu_ref ref)
18	{
19	return container_of(ref, struct dev_dax, ref);
20	}
21
22	static void dev_dax_percpu_release(struct percpu_ref *ref)
23	{
24	struct dev_dax *dev_dax = ref_to_dev_dax(ref);
25
26	dev_dbg(&dev_dax->dev, "%s\n", __func__);
27	complete(&dev_dax->cmp);
28	}
29
30	static void dev_dax_percpu_exit(struct percpu_ref *ref)
31	{
32	struct dev_dax *dev_dax = ref_to_dev_dax(ref);
33
34	dev_dbg(&dev_dax->dev, "%s\n", __func__);
35	wait_for_completion(&dev_dax->cmp);
36	percpu_ref_exit(ref);
37	}
38
39	static void dev_dax_percpu_kill(struct percpu_ref *data)
40	{
41	struct percpu_ref *ref = data;
42	struct dev_dax *dev_dax = ref_to_dev_dax(ref);
43
44	dev_dbg(&dev_dax->dev, "%s\n", __func__);
45	percpu_ref_kill(ref);
46	}
47
48	static int check_vma(struct dev_dax dev_dax, struct vm_area_struct vma,	17	static int check_vma(struct dev_dax dev_dax, struct vm_area_struct vma,
49	const char *func)	18	const char *func)
50	{	19	{
@@ -459,15 +428,7 @@ int dev_dax_probe(struct device *dev) · 408 unmodified lines
459	return -EBUSY;	428	return -EBUSY;
460	}	429	}
461		430
462	init_completion(&dev_dax->cmp);	431	dev_dax->pgmap.type = MEMORY_DEVICE_DEVDAX;
463	rc = percpu_ref_init(&dev_dax->ref, dev_dax_percpu_release, 0,
464	GFP_KERNEL);
465	if (rc)
466	return rc;
467
468	dev_dax->pgmap.ref = &dev_dax->ref;
469	dev_dax->pgmap.kill = dev_dax_percpu_kill;
470	dev_dax->pgmap.cleanup = dev_dax_percpu_exit;
471	addr = devm_memremap_pages(dev, &dev_dax->pgmap);	432	addr = devm_memremap_pages(dev, &dev_dax->pgmap);
472	if (IS_ERR(addr))	433	if (IS_ERR(addr))
473	return PTR_ERR(addr);	434	return PTR_ERR(addr);

drivers/dax/pmem/core.c+1−1

@@ -16,7 +16,7 @@ struct dev_dax __dax_pmem_probe(struct device dev, enum dev_dax_subsys subsys) · 15 unmodified lines
16	struct dev_dax *dev_dax;	16	struct dev_dax *dev_dax;
17	struct nd_namespace_io *nsio;	17	struct nd_namespace_io *nsio;
18	struct dax_region *dax_region;	18	struct dax_region *dax_region;
19	struct dev_pagemap pgmap = { 0 };	19	struct dev_pagemap pgmap = { };
20	struct nd_namespace_common *ndns;	20	struct nd_namespace_common *ndns;
21	struct nd_dax *nd_dax = to_nd_dax(dev);	21	struct nd_dax *nd_dax = to_nd_dax(dev);
22	struct nd_pfn *nd_pfn = &nd_dax->nd_pfn;	22	struct nd_pfn *nd_pfn = &nd_dax->nd_pfn;

drivers/gpu/drm/nouveau/Kconfig+3−3

 config DRM_NOUVEAU_SVM
 	bool "(EXPERIMENTAL) Enable SVM (Shared Virtual Memory) support"
-	depends on ARCH_HAS_HMM
+	depends on DEVICE_PRIVATE
 	depends on DRM_NOUVEAU
+	depends on HMM_MIRROR
 	depends on STAGING
-	select HMM_MIRROR
+	select MIGRATE_VMA_HELPER
-	select DEVICE_PRIVATE
 	default n
 	help
 	  Say Y here if you want to enable experimental support for

drivers/gpu/drm/nouveau/nouveau_dmem.c+47−56

@@ -72,7 +72,8 @@ struct nouveau_dmem_migrate { · 71 unmodified lines
72	};	72	};
73		73
74	struct nouveau_dmem {	74	struct nouveau_dmem {
75	struct hmm_devmem *devmem;	75	struct nouveau_drm *drm;
		76	struct dev_pagemap pagemap;
76	struct nouveau_dmem_migrate migrate;	77	struct nouveau_dmem_migrate migrate;
77	struct list_head chunk_free;	78	struct list_head chunk_free;
78	struct list_head chunk_full;	79	struct list_head chunk_full;
@@ -80,6 +81,11 @@ struct nouveau_dmem { · 1 unmodified line
80	struct mutex mutex;	81	struct mutex mutex;
81	};	82	};
82		83
		84	static inline struct nouveau_dmem page_to_dmem(struct page page)
		85	{
		86	return container_of(page->pgmap, struct nouveau_dmem, pagemap);
		87	}
		88
83	struct nouveau_dmem_fault {	89	struct nouveau_dmem_fault {
84	struct nouveau_drm *drm;	90	struct nouveau_drm *drm;
85	struct nouveau_fence *fence;	91	struct nouveau_fence *fence;
@@ -96,14 +102,10 @@ struct nouveau_migrate { · 10 unmodified lines
96	unsigned long dma_nr;	102	unsigned long dma_nr;
97	};	103	};
98		104
99	static void	105	static void nouveau_dmem_page_free(struct page *page)
100	nouveau_dmem_free(struct hmm_devmem devmem, struct page page)
101	{	106	{
102	struct nouveau_dmem_chunk *chunk;	107	struct nouveau_dmem_chunk *chunk = page->zone_device_data;
103	unsigned long idx;	108	unsigned long idx = page_to_pfn(page) - chunk->pfn_first;
104
105	chunk = (void *)hmm_devmem_page_get_drvdata(page);
106	idx = page_to_pfn(page) - chunk->pfn_first;
107		109
108	/*	110	/*
109	* FIXME:	111	* FIXME:
@@ -148,11 +150,12 @@ nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, · 38 unmodified lines
148	if (!spage \|\| !(src_pfns[i] & MIGRATE_PFN_MIGRATE))	150	if (!spage \|\| !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
149	continue;	151	continue;
150		152
151	dpage = hmm_vma_alloc_locked_page(vma, addr);	153	dpage = alloc_page_vma(GFP_HIGHUSER, vma, addr);
152	if (!dpage) {	154	if (!dpage) {
153	dst_pfns[i] = MIGRATE_PFN_ERROR;	155	dst_pfns[i] = MIGRATE_PFN_ERROR;
154	continue;	156	continue;
155	}	157	}
		158	lock_page(dpage);
156		159
157	dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) \|	160	dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) \|
158	MIGRATE_PFN_LOCKED;	161	MIGRATE_PFN_LOCKED;
@@ -194,7 +197,7 @@ nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, · 35 unmodified lines
194		197
195	dst_addr = fault->dma[fault->npages++];	198	dst_addr = fault->dma[fault->npages++];
196		199
197	chunk = (void *)hmm_devmem_page_get_drvdata(spage);	200	chunk = spage->zone_device_data;
198	src_addr = page_to_pfn(spage) - chunk->pfn_first;	201	src_addr = page_to_pfn(spage) - chunk->pfn_first;
199	src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;	202	src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
200		203
@@ -259,29 +262,21 @@ static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = { · 58 unmodified lines
259	.finalize_and_map = nouveau_dmem_fault_finalize_and_map,	262	.finalize_and_map = nouveau_dmem_fault_finalize_and_map,
260	};	263	};
261		264
262	static vm_fault_t	265	static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
263	nouveau_dmem_fault(struct hmm_devmem *devmem,
264	struct vm_area_struct *vma,
265	unsigned long addr,
266	const struct page *page,
267	unsigned int flags,
268	pmd_t *pmdp)
269	{	266	{
270	struct drm_device *drm_dev = dev_get_drvdata(devmem->device);	267	struct nouveau_dmem *dmem = page_to_dmem(vmf->page);
271	unsigned long src[1] = {0}, dst[1] = {0};	268	unsigned long src[1] = {0}, dst[1] = {0};
272	struct nouveau_dmem_fault fault = {0};	269	struct nouveau_dmem_fault fault = { .drm = dmem->drm };
273	int ret;	270	int ret;
274		271
275
276
277	/*	272	/*
278	* FIXME what we really want is to find some heuristic to migrate more	273	* FIXME what we really want is to find some heuristic to migrate more
279	* than just one page on CPU fault. When such fault happens it is very	274	* than just one page on CPU fault. When such fault happens it is very
280	* likely that more surrounding page will CPU fault too.	275	* likely that more surrounding page will CPU fault too.
281	*/	276	*/
282	fault.drm = nouveau_drm(drm_dev);	277	ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vmf->vma,
283	ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,	278	vmf->address, vmf->address + PAGE_SIZE,
284	addr + PAGE_SIZE, src, dst, &fault);	279	src, dst, &fault);
285	if (ret)	280	if (ret)
286	return VM_FAULT_SIGBUS;	281	return VM_FAULT_SIGBUS;
287		282
@@ -291,10 +286,9 @@ nouveau_dmem_fault(struct hmm_devmem *devmem, · 3 unmodified lines
291	return 0;	286	return 0;
292	}	287	}
293		288
294	static const struct hmm_devmem_ops	289	static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = {
295	nouveau_dmem_devmem_ops = {	290	.page_free = nouveau_dmem_page_free,
296	.free = nouveau_dmem_free,	291	.migrate_to_ram = nouveau_dmem_migrate_to_ram,
297	.fault = nouveau_dmem_fault,
298	};	292	};
299		293
300	static int	294	static int
@@ -580,7 +574,8 @@ void · 279 unmodified lines
580	nouveau_dmem_init(struct nouveau_drm *drm)	574	nouveau_dmem_init(struct nouveau_drm *drm)
581	{	575	{
582	struct device *device = drm->dev->dev;	576	struct device *device = drm->dev->dev;
583	unsigned long i, size;	577	struct resource *res;
		578	unsigned long i, size, pfn_first;
584	int ret;	579	int ret;
585		580
586	/* This only make sense on PASCAL or newer */	581	/* This only make sense on PASCAL or newer */
@@ -590,6 +585,7 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 3 unmodified lines
590	if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))	585	if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
591	return;	586	return;
592		587
		588	drm->dmem->drm = drm;
593	mutex_init(&drm->dmem->mutex);	589	mutex_init(&drm->dmem->mutex);
594	INIT_LIST_HEAD(&drm->dmem->chunk_free);	590	INIT_LIST_HEAD(&drm->dmem->chunk_free);
595	INIT_LIST_HEAD(&drm->dmem->chunk_full);	591	INIT_LIST_HEAD(&drm->dmem->chunk_full);
@@ -599,11 +595,8 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 3 unmodified lines
599		595
600	/* Initialize migration dma helpers before registering memory */	596	/* Initialize migration dma helpers before registering memory */
601	ret = nouveau_dmem_migrate_init(drm);	597	ret = nouveau_dmem_migrate_init(drm);
602	if (ret) {	598	if (ret)
603	kfree(drm->dmem);	599	goto out_free;
604	drm->dmem = NULL;
605	return;
606	}
607		600
608	/*	601	/*
609	* FIXME we need some kind of policy to decide how much VRAM we	602	* FIXME we need some kind of policy to decide how much VRAM we
@@ -611,14 +604,16 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 1 unmodified line
611	* and latter if we want to do thing like over commit then we	604	* and latter if we want to do thing like over commit then we
612	* could revisit this.	605	* could revisit this.
613	*/	606	*/
614	drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,	607	res = devm_request_free_mem_region(device, &iomem_resource, size);
615	device, size);	608	if (IS_ERR(res))
616	if (IS_ERR(drm->dmem->devmem)) {	609	goto out_free;
617	kfree(drm->dmem);	610	drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
618	drm->dmem = NULL;	611	drm->dmem->pagemap.res = *res;
619	return;	612	drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops;
620	}	613	if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap)))
621		614	goto out_free;
		615
		616	pfn_first = res->start >> PAGE_SHIFT;
622	for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {	617	for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
623	struct nouveau_dmem_chunk *chunk;	618	struct nouveau_dmem_chunk *chunk;
624	struct page *page;	619	struct page *page;
@@ -631,17 +626,19 @@ nouveau_dmem_init(struct nouveau_drm *drm) · 6 unmodified lines
631	}	626	}
632		627
633	chunk->drm = drm;	628	chunk->drm = drm;
634	chunk->pfn_first = drm->dmem->devmem->pfn_first;	629	chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES);
635	chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
636	list_add_tail(&chunk->list, &drm->dmem->chunk_empty);	630	list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
637		631
638	page = pfn_to_page(chunk->pfn_first);	632	page = pfn_to_page(chunk->pfn_first);
639	for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {	633	for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page)
640	hmm_devmem_page_set_drvdata(page, (long)chunk);	634	page->zone_device_data = chunk;
641	}
642	}	635	}
643		636
644	NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);	637	NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
		638	return;
		639	out_free:
		640	kfree(drm->dmem);
		641	drm->dmem = NULL;
645	}	642	}
646		643
647	static void	644	static void
@@ -697,7 +694,7 @@ nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma, · 49 unmodified lines
697	if (!dpage \|\| dst_pfns[i] == MIGRATE_PFN_ERROR)	694	if (!dpage \|\| dst_pfns[i] == MIGRATE_PFN_ERROR)
698	continue;	695	continue;
699		696
700	chunk = (void *)hmm_devmem_page_get_drvdata(dpage);	697	chunk = dpage->zone_device_data;
701	dst_addr = page_to_pfn(dpage) - chunk->pfn_first;	698	dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
702	dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;	699	dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
703		700
@@ -832,13 +829,7 @@ out: · 128 unmodified lines
832	static inline bool	829	static inline bool
833	nouveau_dmem_page(struct nouveau_drm drm, struct page page)	830	nouveau_dmem_page(struct nouveau_drm drm, struct page page)
834	{	831	{
835	if (!is_device_private_page(page))	832	return is_device_private_page(page) && drm->dmem == page_to_dmem(page);
836	return false;
837
838	if (drm->dmem->devmem != page->pgmap->data)
839	return false;
840
841	return true;
842	}	833	}
843		834
844	void	835	void
@@ -867,7 +858,7 @@ nouveau_dmem_convert_pfn(struct nouveau_drm *drm, · 22 unmodified lines
867	continue;	858	continue;
868	}	859	}
869		860
870	chunk = (void *)hmm_devmem_page_get_drvdata(page);	861	chunk = page->zone_device_data;
871	addr = page_to_pfn(page) - chunk->pfn_first;	862	addr = page_to_pfn(page) - chunk->pfn_first;
872	addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;	863	addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
873		864

drivers/nvdimm/pfn_devs.c+1−2

 		if (offset < reserve)
 			return -EINVAL;
 		nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns);
-		pgmap->altmap_valid = false;
 	} else if (nd_pfn->mode == PFN_MODE_PMEM) {
 		nd_pfn->npfns = PFN_SECTION_ALIGN_UP((resource_size(res)
 					- offset) / PAGE_SIZE);
 		memcpy(altmap, &__altmap, sizeof(*altmap));
 		altmap->free = PHYS_PFN(offset - reserve);
 		altmap->alloc = 0;
-		pgmap->altmap_valid = true;
+		pgmap->flags |= PGMAP_ALTMAP_VALID;
 	} else
 		return -ENXIO;

drivers/nvdimm/pmem.c+19−32

 	NULL,
 };
-static void __pmem_release_queue(struct percpu_ref *ref)
+static void pmem_pagemap_cleanup(struct dev_pagemap *pgmap)
 {
-	struct request_queue *q;
+	struct request_queue *q =
+		container_of(pgmap->ref, struct request_queue, q_usage_counter);
-	q = container_of(ref, typeof(*q), q_usage_counter);
 	blk_cleanup_queue(q);
 }
-static void pmem_release_queue(void *ref)
+static void pmem_release_queue(void *pgmap)
 {
-	__pmem_release_queue(ref);
+	pmem_pagemap_cleanup(pgmap);
 }
-static void pmem_freeze_queue(struct percpu_ref *ref)
+static void pmem_pagemap_kill(struct dev_pagemap *pgmap)
 {
-	struct request_queue *q;
+	struct request_queue *q =
+		container_of(pgmap->ref, struct request_queue, q_usage_counter);
-	q = container_of(ref, typeof(*q), q_usage_counter);
 	blk_freeze_queue_start(q);
 }
 	put_disk(pmem->disk);
 }
-static void pmem_release_pgmap_ops(void *__pgmap)
+static void pmem_pagemap_page_free(struct page *page)
-{
-	dev_pagemap_put_ops();
-}
-static void fsdax_pagefree(struct page *page, void *data)
 {
 	wake_up_var(&page->_refcount);
 }
-static int setup_pagemap_fsdax(struct device *dev, struct dev_pagemap *pgmap)
+static const struct dev_pagemap_ops fsdax_pagemap_ops = {
-{
+	.page_free		= pmem_pagemap_page_free,
-	dev_pagemap_get_ops();
+	.kill			= pmem_pagemap_kill,
-	if (devm_add_action_or_reset(dev, pmem_release_pgmap_ops, pgmap))
+	.cleanup		= pmem_pagemap_cleanup,
-		return -ENOMEM;
+};
-	pgmap->type = MEMORY_DEVICE_FS_DAX;
-	pgmap->page_free = fsdax_pagefree;
-	return 0;
-}
 static int pmem_attach_disk(struct device *dev,
 		struct nd_namespace_common *ndns)
 	pmem->pfn_flags = PFN_DEV;
 	pmem->pgmap.ref = &q->q_usage_counter;
-	pmem->pgmap.kill = pmem_freeze_queue;
-	pmem->pgmap.cleanup = __pmem_release_queue;
 	if (is_nd_pfn(dev)) {
-		if (setup_pagemap_fsdax(dev, &pmem->pgmap))
+		pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;
-			return -ENOMEM;
+		pmem->pgmap.ops = &fsdax_pagemap_ops;
 		addr = devm_memremap_pages(dev, &pmem->pgmap);
 		pfn_sb = nd_pfn->pfn_sb;
 		pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);
 		bb_res.start += pmem->data_offset;
 	} else if (pmem_should_map_pages(dev)) {
 		memcpy(&pmem->pgmap.res, &nsio->res, sizeof(pmem->pgmap.res));
-		pmem->pgmap.altmap_valid = false;
+		pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;
-		if (setup_pagemap_fsdax(dev, &pmem->pgmap))
+		pmem->pgmap.ops = &fsdax_pagemap_ops;
-			return -ENOMEM;
 		addr = devm_memremap_pages(dev, &pmem->pgmap);
 		pmem->pfn_flags |= PFN_MAP;
 		memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));
 	} else {
 		if (devm_add_action_or_reset(dev, pmem_release_queue,
-					&q->q_usage_counter))
+					&pmem->pgmap))
 			return -ENOMEM;
 		addr = devm_memremap(dev, pmem->phys_addr,
 				pmem->size, ARCH_MEMREMAP_PMEM);

drivers/pci/p2pdma.c+4−48

 	bool p2pmem_published;
 };
-struct p2pdma_pagemap {
-	struct dev_pagemap pgmap;
-	struct percpu_ref ref;
-	struct completion ref_done;
-};
 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
 			 char *buf)
 {
 	.name = "p2pmem",
 };
-static struct p2pdma_pagemap *to_p2p_pgmap(struct percpu_ref *ref)
-{
-	return container_of(ref, struct p2pdma_pagemap, ref);
-}
-static void pci_p2pdma_percpu_release(struct percpu_ref *ref)
-{
-	struct p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(ref);
-	complete(&p2p_pgmap->ref_done);
-}
-static void pci_p2pdma_percpu_kill(struct percpu_ref *ref)
-{
-	percpu_ref_kill(ref);
-}
-static void pci_p2pdma_percpu_cleanup(struct percpu_ref *ref)
-{
-	struct p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(ref);
-	wait_for_completion(&p2p_pgmap->ref_done);
-	percpu_ref_exit(&p2p_pgmap->ref);
-}
 static void pci_p2pdma_release(void *data)
 {
 	struct pci_dev *pdev = data;
 int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
 			    u64 offset)
 {
-	struct p2pdma_pagemap *p2p_pgmap;
 	struct dev_pagemap *pgmap;
 	void *addr;
 	int error;
 			return error;
 	}
-	p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL);
+	pgmap = devm_kzalloc(&pdev->dev, sizeof(*pgmap), GFP_KERNEL);
-	if (!p2p_pgmap)
+	if (!pgmap)
 		return -ENOMEM;
-	init_completion(&p2p_pgmap->ref_done);
-	error = percpu_ref_init(&p2p_pgmap->ref,
-			pci_p2pdma_percpu_release, 0, GFP_KERNEL);
-	if (error)
-		goto pgmap_free;
-	pgmap = &p2p_pgmap->pgmap;
 	pgmap->res.start = pci_resource_start(pdev, bar) + offset;
 	pgmap->res.end = pgmap->res.start + size - 1;
 	pgmap->res.flags = pci_resource_flags(pdev, bar);
-	pgmap->ref = &p2p_pgmap->ref;
 	pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;
 	pgmap->pci_p2pdma_bus_offset = pci_bus_address(pdev, bar) -
 		pci_resource_start(pdev, bar);
-	pgmap->kill = pci_p2pdma_percpu_kill;
-	pgmap->cleanup = pci_p2pdma_percpu_cleanup;
 	addr = devm_memremap_pages(&pdev->dev, pgmap);
 	if (IS_ERR(addr)) {
 	error = gen_pool_add_owner(pdev->p2pdma->pool, (unsigned long)addr,
 			pci_bus_address(pdev, bar) + offset,
 			resource_size(&pgmap->res), dev_to_node(&pdev->dev),
-			&p2p_pgmap->ref);
+			pgmap->ref);
 	if (error)
 		goto pages_free;
 pages_free:
 	devm_memunmap_pages(&pdev->dev, pgmap);
 pgmap_free:
-	devm_kfree(&pdev->dev, p2p_pgmap);
+	devm_kfree(&pdev->dev, pgmap);
 	return error;
 }
 EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource);

fs/proc/task_mmu.c+1−1

@@ -1279,7 +1279,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, · 1278 unmodified lines
1279	if (pm->show_pfn)	1279	if (pm->show_pfn)
1280	frame = pte_pfn(pte);	1280	frame = pte_pfn(pte);
1281	flags \|= PM_PRESENT;	1281	flags \|= PM_PRESENT;
1282	page = _vm_normal_page(vma, addr, pte, true);	1282	page = vm_normal_page(vma, addr, pte);
1283	if (pte_soft_dirty(pte))	1283	if (pte_soft_dirty(pte))
1284	flags \|= PM_SOFT_DIRTY;	1284	flags \|= PM_SOFT_DIRTY;
1285	} else if (is_swap_pte(pte)) {	1285	} else if (is_swap_pte(pte)) {

include/linux/hmm.h+1−197

@@ -62,7 +62,7 @@ · 61 unmodified lines
62	#include <linux/kconfig.h>	62	#include <linux/kconfig.h>
63	#include <asm/pgtable.h>	63	#include <asm/pgtable.h>
64		64
65	#if IS_ENABLED(CONFIG_HMM)	65	#ifdef CONFIG_HMM_MIRROR
66		66
67	#include <linux/device.h>	67	#include <linux/device.h>
68	#include <linux/migrate.h>	68	#include <linux/migrate.h>
@@ -324,9 +324,6 @@ static inline uint64_t hmm_pfn_from_pfn(const struct hmm_range *range, · 255 unmodified lines
324	return hmm_device_entry_from_pfn(range, pfn);	324	return hmm_device_entry_from_pfn(range, pfn);
325	}	325	}
326		326
327
328
329	#if IS_ENABLED(CONFIG_HMM_MIRROR)
330	/*	327	/*
331	* Mirroring: how to synchronize device page table with CPU page table.	328	* Mirroring: how to synchronize device page table with CPU page table.
332	*	329	*
@@ -550,197 +547,4 @@ static inline void hmm_mm_init(struct mm_struct *mm) · 217 unmodified lines
550	static inline void hmm_mm_init(struct mm_struct *mm) {}	547	static inline void hmm_mm_init(struct mm_struct *mm) {}
551	#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */	548	#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
552		549
553	#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) \|\| IS_ENABLED(CONFIG_DEVICE_PUBLIC)
554	struct hmm_devmem;
555
556	struct page hmm_vma_alloc_locked_page(struct vm_area_struct vma,
557	unsigned long addr);
558
559	/*
560	* struct hmm_devmem_ops - callback for ZONE_DEVICE memory events
561	*
562	* @free: call when refcount on page reach 1 and thus is no longer use
563	* @fault: call when there is a page fault to unaddressable memory
564	*
565	* Both callback happens from page_free() and page_fault() callback of struct
566	* dev_pagemap respectively. See include/linux/memremap.h for more details on
567	* those.
568	*
569	* The hmm_devmem_ops callback are just here to provide a coherent and
570	* uniq API to device driver and device driver should not register their
571	* own page_free() or page_fault() but rely on the hmm_devmem_ops call-
572	* back.
573	*/
574	struct hmm_devmem_ops {
575	/*
576	* free() - free a device page
577	* @devmem: device memory structure (see struct hmm_devmem)
578	* @page: pointer to struct page being freed
579	*
580	* Call back occurs whenever a device page refcount reach 1 which
581	* means that no one is holding any reference on the page anymore
582	* (ZONE_DEVICE page have an elevated refcount of 1 as default so
583	* that they are not release to the general page allocator).
584	*
585	* Note that callback has exclusive ownership of the page (as no
586	* one is holding any reference).
587	*/
588	void (free)(struct hmm_devmem devmem, struct page *page);
589	/*
590	* fault() - CPU page fault or get user page (GUP)
591	* @devmem: device memory structure (see struct hmm_devmem)
592	* @vma: virtual memory area containing the virtual address
593	* @addr: virtual address that faulted or for which there is a GUP
594	* @page: pointer to struct page backing virtual address (unreliable)
595	* @flags: FAULT_FLAG_* (see include/linux/mm.h)
596	* @pmdp: page middle directory
597	* Return: VM_FAULT_MINOR/MAJOR on success or one of VM_FAULT_ERROR
598	* on error
599	*
600	* The callback occurs whenever there is a CPU page fault or GUP on a
601	* virtual address. This means that the device driver must migrate the
602	* page back to regular memory (CPU accessible).
603	*
604	* The device driver is free to migrate more than one page from the
605	* fault() callback as an optimization. However if the device decides
606	* to migrate more than one page it must always priotirize the faulting
607	* address over the others.
608	*
609	* The struct page pointer is only given as a hint to allow quick
610	* lookup of internal device driver data. A concurrent migration
611	* might have already freed that page and the virtual address might
612	* no longer be backed by it. So it should not be modified by the
613	* callback.
614	*
615	* Note that mmap semaphore is held in read mode at least when this
616	* callback occurs, hence the vma is valid upon callback entry.
617	*/
618	vm_fault_t (fault)(struct hmm_devmem devmem,
619	struct vm_area_struct *vma,
620	unsigned long addr,
621	const struct page *page,
622	unsigned int flags,
623	pmd_t *pmdp);
624	};
625
626	/*
627	* struct hmm_devmem - track device memory
628	*
629	* @completion: completion object for device memory
630	* @pfn_first: first pfn for this resource (set by hmm_devmem_add())
631	* @pfn_last: last pfn for this resource (set by hmm_devmem_add())
632	* @resource: IO resource reserved for this chunk of memory
633	* @pagemap: device page map for that chunk
634	* @device: device to bind resource to
635	* @ops: memory operations callback
636	* @ref: per CPU refcount
637	* @page_fault: callback when CPU fault on an unaddressable device page
638	*
639	* This is a helper structure for device drivers that do not wish to implement
640	* the gory details related to hotplugging new memoy and allocating struct
641	* pages.
642	*
643	* Device drivers can directly use ZONE_DEVICE memory on their own if they
644	* wish to do so.
645	*
646	* The page_fault() callback must migrate page back, from device memory to
647	* system memory, so that the CPU can access it. This might fail for various
648	* reasons (device issues, device have been unplugged, ...). When such error
649	* conditions happen, the page_fault() callback must return VM_FAULT_SIGBUS and
650	* set the CPU page table entry to "poisoned".
651	*
652	* Note that because memory cgroup charges are transferred to the device memory,
653	* this should never fail due to memory restrictions. However, allocation
654	* of a regular system page might still fail because we are out of memory. If
655	* that happens, the page_fault() callback must return VM_FAULT_OOM.
656	*
657	* The page_fault() callback can also try to migrate back multiple pages in one
658	* chunk, as an optimization. It must, however, prioritize the faulting address
659	* over all the others.
660	*/
661	typedef vm_fault_t (dev_page_fault_t)(struct vm_area_struct vma,
662	unsigned long addr,
663	const struct page *page,
664	unsigned int flags,
665	pmd_t *pmdp);
666
667	struct hmm_devmem {
668	struct completion completion;
669	unsigned long pfn_first;
670	unsigned long pfn_last;
671	struct resource *resource;
672	struct device *device;
673	struct dev_pagemap pagemap;
674	const struct hmm_devmem_ops *ops;
675	struct percpu_ref ref;
676	dev_page_fault_t page_fault;
677	};
678
679	/*
680	* To add (hotplug) device memory, HMM assumes that there is no real resource
681	* that reserves a range in the physical address space (this is intended to be
682	* use by unaddressable device memory). It will reserve a physical range big
683	* enough and allocate struct page for it.
684	*
685	* The device driver can wrap the hmm_devmem struct inside a private device
686	* driver struct.
687	*/
688	struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops,
689	struct device *device,
690	unsigned long size);
691	struct hmm_devmem hmm_devmem_add_resource(const struct hmm_devmem_ops ops,
692	struct device *device,
693	struct resource *res);
694
695	/*
696	* hmm_devmem_page_set_drvdata - set per-page driver data field
697	*
698	* @page: pointer to struct page
699	* @data: driver data value to set
700	*
701	* Because page can not be on lru we have an unsigned long that driver can use
702	* to store a per page field. This just a simple helper to do that.
703	*/
704	static inline void hmm_devmem_page_set_drvdata(struct page *page,
705	unsigned long data)
706	{
707	page->hmm_data = data;
708	}
709
710	/*
711	* hmm_devmem_page_get_drvdata - get per page driver data field
712	*
713	* @page: pointer to struct page
714	* Return: driver data value
715	*/
716	static inline unsigned long hmm_devmem_page_get_drvdata(const struct page *page)
717	{
718	return page->hmm_data;
719	}
720
721
722	/*
723	* struct hmm_device - fake device to hang device memory onto
724	*
725	* @device: device struct
726	* @minor: device minor number
727	*/
728	struct hmm_device {
729	struct device device;
730	unsigned int minor;
731	};
732
733	/*
734	* A device driver that wants to handle multiple devices memory through a
735	* single fake device can use hmm_device to do so. This is purely a helper and
736	* it is not strictly needed, in order to make use of any HMM functionality.
737	*/
738	struct hmm_device hmm_device_new(void drvdata);
739	void hmm_device_put(struct hmm_device *hmm_device);
740	#endif /* CONFIG_DEVICE_PRIVATE \|\| CONFIG_DEVICE_PUBLIC */
741	#else /* IS_ENABLED(CONFIG_HMM) */
742	static inline void hmm_mm_destroy(struct mm_struct *mm) {}
743	static inline void hmm_mm_init(struct mm_struct *mm) {}
744	#endif /* IS_ENABLED(CONFIG_HMM) */
745
746	#endif /* LINUX_HMM_H */	550	#endif /* LINUX_HMM_H */

include/linux/ioport.h+2−1

 	IORES_DESC_PERSISTENT_MEMORY		= 4,
 	IORES_DESC_PERSISTENT_MEMORY_LEGACY	= 5,
 	IORES_DESC_DEVICE_PRIVATE_MEMORY	= 6,
-	IORES_DESC_DEVICE_PUBLIC_MEMORY		= 7,
 };
 /* helpers to define resources */
        return (r1->start <= r2->end && r1->end >= r2->start);
 }
+struct resource *devm_request_free_mem_region(struct device *dev,
+		struct resource *base, unsigned long size);
 #endif /* __ASSEMBLY__ */
 #endif	/* _LINUX_IOPORT_H */

include/linux/memremap.h+49−26

@@ -37,13 +37,6 @@ struct vmem_altmap { · 36 unmodified lines
37	* A more complete discussion of unaddressable memory may be found in	37	* A more complete discussion of unaddressable memory may be found in
38	* include/linux/hmm.h and Documentation/vm/hmm.rst.	38	* include/linux/hmm.h and Documentation/vm/hmm.rst.
39	*	39	*
40	* MEMORY_DEVICE_PUBLIC:
41	* Device memory that is cache coherent from device and CPU point of view. This
42	* is use on platform that have an advance system bus (like CAPI or CCIX). A
43	* driver can hotplug the device memory using ZONE_DEVICE and with that memory
44	* type. Any page of a process can be migrated to such memory. However no one
45	* should be allow to pin such memory so that it can always be evicted.
46	*
47	* MEMORY_DEVICE_FS_DAX:	40	* MEMORY_DEVICE_FS_DAX:
48	* Host memory that has similar access semantics as System RAM i.e. DMA	41	* Host memory that has similar access semantics as System RAM i.e. DMA
49	* coherent and supports page pinning. In support of coordinating page	42	* coherent and supports page pinning. In support of coordinating page
@@ -52,54 +45,84 @@ struct vmem_altmap { · 2 unmodified lines
52	* wakeup is used to coordinate physical address space management (ex:	45	* wakeup is used to coordinate physical address space management (ex:
53	* fs truncate/hole punch) vs pinned pages (ex: device dma).	46	* fs truncate/hole punch) vs pinned pages (ex: device dma).
54	*	47	*
		48	* MEMORY_DEVICE_DEVDAX:
		49	* Host memory that has similar access semantics as System RAM i.e. DMA
		50	* coherent and supports page pinning. In contrast to
		51	* MEMORY_DEVICE_FS_DAX, this memory is access via a device-dax
		52	* character device.
		53	*
55	* MEMORY_DEVICE_PCI_P2PDMA:	54	* MEMORY_DEVICE_PCI_P2PDMA:
56	* Device memory residing in a PCI BAR intended for use with Peer-to-Peer	55	* Device memory residing in a PCI BAR intended for use with Peer-to-Peer
57	* transactions.	56	* transactions.
58	*/	57	*/
59	enum memory_type {	58	enum memory_type {
		59	/* 0 is reserved to catch uninitialized type fields */
60	MEMORY_DEVICE_PRIVATE = 1,	60	MEMORY_DEVICE_PRIVATE = 1,
61	MEMORY_DEVICE_PUBLIC,
62	MEMORY_DEVICE_FS_DAX,	61	MEMORY_DEVICE_FS_DAX,
		62	MEMORY_DEVICE_DEVDAX,
63	MEMORY_DEVICE_PCI_P2PDMA,	63	MEMORY_DEVICE_PCI_P2PDMA,
64	};	64	};
65		65
66	/*	66	struct dev_pagemap_ops {
67	* Additional notes about MEMORY_DEVICE_PRIVATE may be found in	67	/*
68	* include/linux/hmm.h and Documentation/vm/hmm.rst. There is also a brief	68	* Called once the page refcount reaches 1. (ZONE_DEVICE pages never
69	* explanation in include/linux/memory_hotplug.h.	69	* reach 0 refcount unless there is a refcount bug. This allows the
70	*	70	* device driver to implement its own memory management.)
71	* The page_free() callback is called once the page refcount reaches 1	71	*/
72	* (ZONE_DEVICE pages never reach 0 refcount unless there is a refcount bug.	72	void (page_free)(struct page page);
73	* This allows the device driver to implement its own memory management.)	73
74	*/	74	/*
75	typedef void (dev_page_free_t)(struct page page, void *data);	75	* Transition the refcount in struct dev_pagemap to the dead state.
		76	*/
		77	void (kill)(struct dev_pagemap pgmap);
		78
		79	/*
		80	* Wait for refcount in struct dev_pagemap to be idle and reap it.
		81	*/
		82	void (cleanup)(struct dev_pagemap pgmap);
		83
		84	/*
		85	* Used for private (un-addressable) device memory only. Must migrate
		86	* the page back to a CPU accessible page.
		87	*/
		88	vm_fault_t (migrate_to_ram)(struct vm_fault vmf);
		89	};
		90
		91	#define PGMAP_ALTMAP_VALID (1 << 0)
76		92
77	/**	93	/**
78	* struct dev_pagemap - metadata for ZONE_DEVICE mappings	94	* struct dev_pagemap - metadata for ZONE_DEVICE mappings
79	* @page_free: free page callback when page refcount reaches 1
80	* @altmap: pre-allocated/reserved memory for vmemmap allocations	95	* @altmap: pre-allocated/reserved memory for vmemmap allocations
81	* @res: physical address range covered by @ref	96	* @res: physical address range covered by @ref
82	* @ref: reference count that pins the devm_memremap_pages() mapping	97	* @ref: reference count that pins the devm_memremap_pages() mapping
83	* @kill: callback to transition @ref to the dead state	98	* @internal_ref: internal reference if @ref is not provided by the caller
84	* @cleanup: callback to wait for @ref to be idle and reap it	99	* @done: completion for @internal_ref
85	* @dev: host device of the mapping for debug	100	* @dev: host device of the mapping for debug
86	* @data: private data pointer for page_free()	101	* @data: private data pointer for page_free()
87	* @type: memory type: see MEMORY_* in memory_hotplug.h	102	* @type: memory type: see MEMORY_* in memory_hotplug.h
		103	* @flags: PGMAP_* flags to specify defailed behavior
		104	* @ops: method table
88	*/	105	*/
89	struct dev_pagemap {	106	struct dev_pagemap {
90	dev_page_free_t page_free;
91	struct vmem_altmap altmap;	107	struct vmem_altmap altmap;
92	bool altmap_valid;
93	struct resource res;	108	struct resource res;
94	struct percpu_ref *ref;	109	struct percpu_ref *ref;
95	void (kill)(struct percpu_ref ref);	110	struct percpu_ref internal_ref;
96	void (cleanup)(struct percpu_ref ref);	111	struct completion done;
97	struct device *dev;	112	struct device *dev;
98	void *data;
99	enum memory_type type;	113	enum memory_type type;
		114	unsigned int flags;
100	u64 pci_p2pdma_bus_offset;	115	u64 pci_p2pdma_bus_offset;
		116	const struct dev_pagemap_ops *ops;
101	};	117	};
102		118
		119	static inline struct vmem_altmap pgmap_altmap(struct dev_pagemap pgmap)
		120	{
		121	if (pgmap->flags & PGMAP_ALTMAP_VALID)
		122	return &pgmap->altmap;
		123	return NULL;
		124	}
		125
103	#ifdef CONFIG_ZONE_DEVICE	126	#ifdef CONFIG_ZONE_DEVICE
104	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap);	127	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap);
105	void devm_memunmap_pages(struct device dev, struct dev_pagemap pgmap);	128	void devm_memunmap_pages(struct device dev, struct dev_pagemap pgmap);

include/linux/mm.h+2−26

 #endif
 #ifdef CONFIG_DEV_PAGEMAP_OPS
-void dev_pagemap_get_ops(void);
-void dev_pagemap_put_ops(void);
 void __put_devmap_managed_page(struct page *page);
 DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
 static inline bool put_devmap_managed_page(struct page *page)
 		return false;
 	switch (page->pgmap->type) {
 	case MEMORY_DEVICE_PRIVATE:
-	case MEMORY_DEVICE_PUBLIC:
 	case MEMORY_DEVICE_FS_DAX:
 		__put_devmap_managed_page(page);
 		return true;
 		page->pgmap->type == MEMORY_DEVICE_PRIVATE;
 }
-static inline bool is_device_public_page(const struct page *page)
-{
-	return is_zone_device_page(page) &&
-		page->pgmap->type == MEMORY_DEVICE_PUBLIC;
-}
 #ifdef CONFIG_PCI_P2PDMA
 static inline bool is_pci_p2pdma_page(const struct page *page)
 {
 #endif /* CONFIG_PCI_P2PDMA */
 #else /* CONFIG_DEV_PAGEMAP_OPS */
-static inline void dev_pagemap_get_ops(void)
-{
-}
-static inline void dev_pagemap_put_ops(void)
-{
-}
 static inline bool put_devmap_managed_page(struct page *page)
 {
 	return false;
 	return false;
 }
-static inline bool is_device_public_page(const struct page *page)
-{
-	return false;
-}
 static inline bool is_pci_p2pdma_page(const struct page *page)
 {
 	return false;
 	pgoff_t last_index;			/* Highest page->index to unmap */
 };
-struct page *_vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
+struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte, bool with_public_device);
+			     pte_t pte);
-#define vm_normal_page(vma, addr, pte) _vm_normal_page(vma, addr, pte, false)
 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
 				pmd_t pmd);

include/linux/mm_types.h+2−2

@@ -158,7 +158,7 @@ struct page { · 157 unmodified lines
158	struct { /* ZONE_DEVICE pages */	158	struct { /* ZONE_DEVICE pages */
159	/** @pgmap: Points to the hosting device page map. */	159	/** @pgmap: Points to the hosting device page map. */
160	struct dev_pagemap *pgmap;	160	struct dev_pagemap *pgmap;
161	unsigned long hmm_data;	161	void *zone_device_data;
162	unsigned long _zd_pad_1; /* uses mapping */	162	unsigned long _zd_pad_1; /* uses mapping */
163	};	163	};
164		164
@@ -501,7 +501,7 @@ struct mm_struct { · 336 unmodified lines
501	#endif	501	#endif
502	struct work_struct async_put_work;	502	struct work_struct async_put_work;
503		503
504	#if IS_ENABLED(CONFIG_HMM)	504	#ifdef CONFIG_HMM_MIRROR
505	/* HMM needs to track a few things per mm */	505	/* HMM needs to track a few things per mm */
506	struct hmm *hmm;	506	struct hmm *hmm;
507	#endif	507	#endif

include/linux/swapops.h+0−15

@@ -129,12 +129,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 128 unmodified lines
129	{	129	{
130	return pfn_to_page(swp_offset(entry));	130	return pfn_to_page(swp_offset(entry));
131	}	131	}
132
133	vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
134	unsigned long addr,
135	swp_entry_t entry,
136	unsigned int flags,
137	pmd_t *pmdp);
138	#else /* CONFIG_DEVICE_PRIVATE */	132	#else /* CONFIG_DEVICE_PRIVATE */
139	static inline swp_entry_t make_device_private_entry(struct page *page, bool write)	133	static inline swp_entry_t make_device_private_entry(struct page *page, bool write)
140	{	134	{
@@ -164,15 +158,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 23 unmodified lines
164	{	158	{
165	return NULL;	159	return NULL;
166	}	160	}
167
168	static inline vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
169	unsigned long addr,
170	swp_entry_t entry,
171	unsigned int flags,
172	pmd_t *pmdp)
173	{
174	return VM_FAULT_SIGBUS;
175	}
176	#endif /* CONFIG_DEVICE_PRIVATE */	161	#endif /* CONFIG_DEVICE_PRIVATE */
177		162
178	#ifdef CONFIG_MIGRATION	163	#ifdef CONFIG_MIGRATION

kernel/memremap.c+118−76

@@ -11,41 +11,39 @@ · 10 unmodified lines
11	#include <linux/types.h>	11	#include <linux/types.h>
12	#include <linux/wait_bit.h>	12	#include <linux/wait_bit.h>
13	#include <linux/xarray.h>	13	#include <linux/xarray.h>
14	#include <linux/hmm.h>
15		14
16	static DEFINE_XARRAY(pgmap_array);	15	static DEFINE_XARRAY(pgmap_array);
17	#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)	16	#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
18	#define SECTION_SIZE (1UL << PA_SECTION_SHIFT)	17	#define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
19		18
20	#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)	19	#ifdef CONFIG_DEV_PAGEMAP_OPS
21	vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,	20	DEFINE_STATIC_KEY_FALSE(devmap_managed_key);
22	unsigned long addr,	21	EXPORT_SYMBOL(devmap_managed_key);
23	swp_entry_t entry,	22	static atomic_t devmap_managed_enable;
24	unsigned int flags,	23
25	pmd_t *pmdp)	24	static void devmap_managed_enable_put(void *data)
26	{	25	{
27	struct page *page = device_private_entry_to_page(entry);	26	if (atomic_dec_and_test(&devmap_managed_enable))
28	struct hmm_devmem *devmem;	27	static_branch_disable(&devmap_managed_key);
		28	}
29		29
30	devmem = container_of(page->pgmap, typeof(*devmem), pagemap);	30	static int devmap_managed_enable_get(struct device dev, struct dev_pagemap pgmap)
		31	{
		32	if (!pgmap->ops \|\| !pgmap->ops->page_free) {
		33	WARN(1, "Missing page_free method\n");
		34	return -EINVAL;
		35	}
31		36
32	/*	37	if (atomic_inc_return(&devmap_managed_enable) == 1)
33	* The page_fault() callback must migrate page back to system memory	38	static_branch_enable(&devmap_managed_key);
34	* so that CPU can access it. This might fail for various reasons	39	return devm_add_action_or_reset(dev, devmap_managed_enable_put, NULL);
35	* (device issue, device was unsafely unplugged, ...). When such
36	* error conditions happen, the callback must return VM_FAULT_SIGBUS.
37	*
38	* Note that because memory cgroup charges are accounted to the device
39	* memory, this should never fail because of memory restrictions (but
40	* allocation of regular system page might still fail because we are
41	* out of memory).
42	*
43	* There is a more in-depth description of what that callback can and
44	* cannot do, in include/linux/memremap.h
45	*/
46	return devmem->page_fault(vma, addr, page, flags, pmdp);
47	}	40	}
48	#endif /* CONFIG_DEVICE_PRIVATE */	41	#else
		42	static int devmap_managed_enable_get(struct device dev, struct dev_pagemap pgmap)
		43	{
		44	return -EINVAL;
		45	}
		46	#endif /* CONFIG_DEV_PAGEMAP_OPS */
49		47
50	static void pgmap_array_delete(struct resource *res)	48	static void pgmap_array_delete(struct resource *res)
51	{	49	{
@@ -56,14 +54,8 @@ static void pgmap_array_delete(struct resource *res) · 4 unmodified lines
56		54
57	static unsigned long pfn_first(struct dev_pagemap *pgmap)	55	static unsigned long pfn_first(struct dev_pagemap *pgmap)
58	{	56	{
59	const struct resource *res = &pgmap->res;	57	return (pgmap->res.start >> PAGE_SHIFT) +
60	struct vmem_altmap *altmap = &pgmap->altmap;	58	vmem_altmap_offset(pgmap_altmap(pgmap));
61	unsigned long pfn;
62
63	pfn = res->start >> PAGE_SHIFT;
64	if (pgmap->altmap_valid)
65	pfn += vmem_altmap_offset(altmap);
66	return pfn;
67	}	59	}
68		60
69	static unsigned long pfn_end(struct dev_pagemap *pgmap)	61	static unsigned long pfn_end(struct dev_pagemap *pgmap)
@@ -83,6 +75,24 @@ static unsigned long pfn_next(unsigned long pfn) · 13 unmodified lines
83	#define for_each_device_pfn(pfn, map) \	75	#define for_each_device_pfn(pfn, map) \
84	for (pfn = pfn_first(map); pfn < pfn_end(map); pfn = pfn_next(pfn))	76	for (pfn = pfn_first(map); pfn < pfn_end(map); pfn = pfn_next(pfn))
85		77
		78	static void dev_pagemap_kill(struct dev_pagemap *pgmap)
		79	{
		80	if (pgmap->ops && pgmap->ops->kill)
		81	pgmap->ops->kill(pgmap);
		82	else
		83	percpu_ref_kill(pgmap->ref);
		84	}
		85
		86	static void dev_pagemap_cleanup(struct dev_pagemap *pgmap)
		87	{
		88	if (pgmap->ops && pgmap->ops->cleanup) {
		89	pgmap->ops->cleanup(pgmap);
		90	} else {
		91	wait_for_completion(&pgmap->done);
		92	percpu_ref_exit(pgmap->ref);
		93	}
		94	}
		95
86	static void devm_memremap_pages_release(void *data)	96	static void devm_memremap_pages_release(void *data)
87	{	97	{
88	struct dev_pagemap *pgmap = data;	98	struct dev_pagemap *pgmap = data;
@@ -92,10 +102,10 @@ static void devm_memremap_pages_release(void *data) · 3 unmodified lines
92	unsigned long pfn;	102	unsigned long pfn;
93	int nid;	103	int nid;
94		104
95	pgmap->kill(pgmap->ref);	105	dev_pagemap_kill(pgmap);
96	for_each_device_pfn(pfn, pgmap)	106	for_each_device_pfn(pfn, pgmap)
97	put_page(pfn_to_page(pfn));	107	put_page(pfn_to_page(pfn));
98	pgmap->cleanup(pgmap->ref);	108	dev_pagemap_cleanup(pgmap);
99		109
100	/* pages are dead and unused, undo the arch mapping */	110	/* pages are dead and unused, undo the arch mapping */
101	align_start = res->start & ~(SECTION_SIZE - 1);	111	align_start = res->start & ~(SECTION_SIZE - 1);
@@ -111,7 +121,7 @@ static void devm_memremap_pages_release(void *data) · 9 unmodified lines
111	align_size >> PAGE_SHIFT, NULL);	121	align_size >> PAGE_SHIFT, NULL);
112	} else {	122	} else {
113	arch_remove_memory(nid, align_start, align_size,	123	arch_remove_memory(nid, align_start, align_size,
114	pgmap->altmap_valid ? &pgmap->altmap : NULL);	124	pgmap_altmap(pgmap));
115	kasan_remove_zero_shadow(__va(align_start), align_size);	125	kasan_remove_zero_shadow(__va(align_start), align_size);
116	}	126	}
117	mem_hotplug_done();	127	mem_hotplug_done();
@@ -122,20 +132,29 @@ static void devm_memremap_pages_release(void *data) · 4 unmodified lines
122	"%s: failed to free all reserved pages\n", __func__);	132	"%s: failed to free all reserved pages\n", __func__);
123	}	133	}
124		134
		135	static void dev_pagemap_percpu_release(struct percpu_ref *ref)
		136	{
		137	struct dev_pagemap *pgmap =
		138	container_of(ref, struct dev_pagemap, internal_ref);
		139
		140	complete(&pgmap->done);
		141	}
		142
125	/**	143	/**
126	* devm_memremap_pages - remap and provide memmap backing for the given resource	144	* devm_memremap_pages - remap and provide memmap backing for the given resource
127	* @dev: hosting device for @res	145	* @dev: hosting device for @res
128	* @pgmap: pointer to a struct dev_pagemap	146	* @pgmap: pointer to a struct dev_pagemap
129	*	147	*
130	* Notes:	148	* Notes:
131	* 1/ At a minimum the res, ref and type members of @pgmap must be initialized	149	* 1/ At a minimum the res and type members of @pgmap must be initialized
132	* by the caller before passing it to this function	150	* by the caller before passing it to this function
133	*	151	*
134	* 2/ The altmap field may optionally be initialized, in which case altmap_valid	152	* 2/ The altmap field may optionally be initialized, in which case
135	* must be set to true	153	* PGMAP_ALTMAP_VALID must be set in pgmap->flags.
136	*	154	*
137	* 3/ pgmap->ref must be 'live' on entry and will be killed and reaped	155	* 3/ The ref field may optionally be provided, in which pgmap->ref must be
138	* at devm_memremap_pages_release() time, or if this routine fails.	156	* 'live' on entry and will be killed and reaped at
		157	* devm_memremap_pages_release() time, or if this routine fails.
139	*	158	*
140	* 4/ res is expected to be a host memory range that could feasibly be	159	* 4/ res is expected to be a host memory range that could feasibly be
141	* treated as a "System RAM" range, i.e. not a device mmio range, but	160	* treated as a "System RAM" range, i.e. not a device mmio range, but
@@ -144,22 +163,66 @@ static void devm_memremap_pages_release(void *data) · 2 unmodified lines
144	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap)	163	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap)
145	{	164	{
146	resource_size_t align_start, align_size, align_end;	165	resource_size_t align_start, align_size, align_end;
147	struct vmem_altmap *altmap = pgmap->altmap_valid ?
148	&pgmap->altmap : NULL;
149	struct resource *res = &pgmap->res;	166	struct resource *res = &pgmap->res;
150	struct dev_pagemap *conflict_pgmap;	167	struct dev_pagemap *conflict_pgmap;
151	struct mhp_restrictions restrictions = {	168	struct mhp_restrictions restrictions = {
152	/*	169	/*
153	* We do not want any optional features only our own memmap	170	* We do not want any optional features only our own memmap
154	*/	171	*/
155	.altmap = altmap,	172	.altmap = pgmap_altmap(pgmap),
156	};	173	};
157	pgprot_t pgprot = PAGE_KERNEL;	174	pgprot_t pgprot = PAGE_KERNEL;
158	int error, nid, is_ram;	175	int error, nid, is_ram;
		176	bool need_devmap_managed = true;
		177
		178	switch (pgmap->type) {
		179	case MEMORY_DEVICE_PRIVATE:
		180	if (!IS_ENABLED(CONFIG_DEVICE_PRIVATE)) {
		181	WARN(1, "Device private memory not supported\n");
		182	return ERR_PTR(-EINVAL);
		183	}
		184	if (!pgmap->ops \|\| !pgmap->ops->migrate_to_ram) {
		185	WARN(1, "Missing migrate_to_ram method\n");
		186	return ERR_PTR(-EINVAL);
		187	}
		188	break;
		189	case MEMORY_DEVICE_FS_DAX:
		190	if (!IS_ENABLED(CONFIG_ZONE_DEVICE) \|\|
		191	IS_ENABLED(CONFIG_FS_DAX_LIMITED)) {
		192	WARN(1, "File system DAX not supported\n");
		193	return ERR_PTR(-EINVAL);
		194	}
		195	break;
		196	case MEMORY_DEVICE_DEVDAX:
		197	case MEMORY_DEVICE_PCI_P2PDMA:
		198	need_devmap_managed = false;
		199	break;
		200	default:
		201	WARN(1, "Invalid pgmap type %d\n", pgmap->type);
		202	break;
		203	}
		204
		205	if (!pgmap->ref) {
		206	if (pgmap->ops && (pgmap->ops->kill \|\| pgmap->ops->cleanup))
		207	return ERR_PTR(-EINVAL);
		208
		209	init_completion(&pgmap->done);
		210	error = percpu_ref_init(&pgmap->internal_ref,
		211	dev_pagemap_percpu_release, 0, GFP_KERNEL);
		212	if (error)
		213	return ERR_PTR(error);
		214	pgmap->ref = &pgmap->internal_ref;
		215	} else {
		216	if (!pgmap->ops \|\| !pgmap->ops->kill \|\| !pgmap->ops->cleanup) {
		217	WARN(1, "Missing reference count teardown definition\n");
		218	return ERR_PTR(-EINVAL);
		219	}
		220	}
159		221
160	if (!pgmap->ref \|\| !pgmap->kill \|\| !pgmap->cleanup) {	222	if (need_devmap_managed) {
161	WARN(1, "Missing reference count teardown definition\n");	223	error = devmap_managed_enable_get(dev, pgmap);
162	return ERR_PTR(-EINVAL);	224	if (error)
		225	return ERR_PTR(error);
163	}	226	}
164		227
165	align_start = res->start & ~(SECTION_SIZE - 1);	228	align_start = res->start & ~(SECTION_SIZE - 1);
@@ -241,7 +304,7 @@ void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap) · 75 unmodified lines
241		304
242	zone = &NODE_DATA(nid)->node_zones[ZONE_DEVICE];	305	zone = &NODE_DATA(nid)->node_zones[ZONE_DEVICE];
243	move_pfn_range_to_zone(zone, align_start >> PAGE_SHIFT,	306	move_pfn_range_to_zone(zone, align_start >> PAGE_SHIFT,
244	align_size >> PAGE_SHIFT, altmap);	307	align_size >> PAGE_SHIFT, pgmap_altmap(pgmap));
245	}	308	}
246		309
247	mem_hotplug_done();	310	mem_hotplug_done();
@@ -271,9 +334,8 @@ void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap) · 23 unmodified lines
271	err_pfn_remap:	334	err_pfn_remap:
272	pgmap_array_delete(res);	335	pgmap_array_delete(res);
273	err_array:	336	err_array:
274	pgmap->kill(pgmap->ref);	337	dev_pagemap_kill(pgmap);
275	pgmap->cleanup(pgmap->ref);	338	dev_pagemap_cleanup(pgmap);
276
277	return ERR_PTR(error);	339	return ERR_PTR(error);
278	}	340	}
279	EXPORT_SYMBOL_GPL(devm_memremap_pages);	341	EXPORT_SYMBOL_GPL(devm_memremap_pages);
@@ -287,7 +349,9 @@ EXPORT_SYMBOL_GPL(devm_memunmap_pages); · 7 unmodified lines
287	unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)	349	unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
288	{	350	{
289	/* number of pfns from base where pfn_to_page() is valid */	351	/* number of pfns from base where pfn_to_page() is valid */
290	return altmap->reserve + altmap->free;	352	if (altmap)
		353	return altmap->reserve + altmap->free;
		354	return 0;
291	}	355	}
292		356
293	void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns)	357	void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns)
@@ -329,28 +393,6 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, · 35 unmodified lines
329	EXPORT_SYMBOL_GPL(get_dev_pagemap);	393	EXPORT_SYMBOL_GPL(get_dev_pagemap);
330		394
331	#ifdef CONFIG_DEV_PAGEMAP_OPS	395	#ifdef CONFIG_DEV_PAGEMAP_OPS
332	DEFINE_STATIC_KEY_FALSE(devmap_managed_key);
333	EXPORT_SYMBOL(devmap_managed_key);
334	static atomic_t devmap_enable;
335
336	/*
337	* Toggle the static key for ->page_free() callbacks when dev_pagemap
338	* pages go idle.
339	*/
340	void dev_pagemap_get_ops(void)
341	{
342	if (atomic_inc_return(&devmap_enable) == 1)
343	static_branch_enable(&devmap_managed_key);
344	}
345	EXPORT_SYMBOL_GPL(dev_pagemap_get_ops);
346
347	void dev_pagemap_put_ops(void)
348	{
349	if (atomic_dec_and_test(&devmap_enable))
350	static_branch_disable(&devmap_managed_key);
351	}
352	EXPORT_SYMBOL_GPL(dev_pagemap_put_ops);
353
354	void __put_devmap_managed_page(struct page *page)	396	void __put_devmap_managed_page(struct page *page)
355	{	397	{
356	int count = page_ref_dec_return(page);	398	int count = page_ref_dec_return(page);
@@ -366,7 +408,7 @@ void __put_devmap_managed_page(struct page *page) · 9 unmodified lines
366		408
367	mem_cgroup_uncharge(page);	409	mem_cgroup_uncharge(page);
368		410
369	page->pgmap->page_free(page, page->pgmap->data);	411	page->pgmap->ops->page_free(page);
370	} else if (!count)	412	} else if (!count)
371	__put_page(page);	413	__put_page(page);
372	}	414	}

kernel/resource.c+39−0

@@ -1628,6 +1628,45 @@ void resource_list_free(struct list_head *head) · 1627 unmodified lines
1628	}	1628	}
1629	EXPORT_SYMBOL(resource_list_free);	1629	EXPORT_SYMBOL(resource_list_free);
1630		1630
		1631	#ifdef CONFIG_DEVICE_PRIVATE
		1632	/**
		1633	* devm_request_free_mem_region - find free region for device private memory
		1634	*
		1635	* @dev: device struct to bind the resource to
		1636	* @size: size in bytes of the device memory to add
		1637	* @base: resource tree to look in
		1638	*
		1639	* This function tries to find an empty range of physical address big enough to
		1640	* contain the new resource, so that it can later be hotplugged as ZONE_DEVICE
		1641	* memory, which in turn allocates struct pages.
		1642	*/
		1643	struct resource devm_request_free_mem_region(struct device dev,
		1644	struct resource *base, unsigned long size)
		1645	{
		1646	resource_size_t end, addr;
		1647	struct resource *res;
		1648
		1649	size = ALIGN(size, 1UL << PA_SECTION_SHIFT);
		1650	end = min_t(unsigned long, base->end, (1UL << MAX_PHYSMEM_BITS) - 1);
		1651	addr = end - size + 1UL;
		1652
		1653	for (; addr > size && addr >= base->start; addr -= size) {
		1654	if (region_intersects(addr, size, 0, IORES_DESC_NONE) !=
		1655	REGION_DISJOINT)
		1656	continue;
		1657
		1658	res = devm_request_mem_region(dev, addr, size, dev_name(dev));
		1659	if (!res)
		1660	return ERR_PTR(-ENOMEM);
		1661	res->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
		1662	return res;
		1663	}
		1664
		1665	return ERR_PTR(-ERANGE);
		1666	}
		1667	EXPORT_SYMBOL_GPL(devm_request_free_mem_region);
		1668	#endif /* CONFIG_DEVICE_PRIVATE */
		1669
1631	static int __init strict_iomem(char *str)	1670	static int __init strict_iomem(char *str)
1632	{	1671	{
1633	if (strstr(str, "relaxed"))	1672	if (strstr(str, "relaxed"))

mm/Kconfig+4−46

@@ -669,47 +669,17 @@ config ZONE_DEVICE · 668 unmodified lines
669		669
670	If FS_DAX is enabled, then say Y.	670	If FS_DAX is enabled, then say Y.
671		671
672	config ARCH_HAS_HMM_MIRROR
673	bool
674	default y
675	depends on (X86_64 \|\| PPC64)
676	depends on MMU && 64BIT
677
678	config ARCH_HAS_HMM_DEVICE
679	bool
680	default y
681	depends on (X86_64 \|\| PPC64)
682	depends on MEMORY_HOTPLUG
683	depends on MEMORY_HOTREMOVE
684	depends on SPARSEMEM_VMEMMAP
685	depends on ARCH_HAS_ZONE_DEVICE
686	select XARRAY_MULTI
687
688	config ARCH_HAS_HMM
689	bool
690	default y
691	depends on (X86_64 \|\| PPC64)
692	depends on ZONE_DEVICE
693	depends on MMU && 64BIT
694	depends on MEMORY_HOTPLUG
695	depends on MEMORY_HOTREMOVE
696	depends on SPARSEMEM_VMEMMAP
697
698	config MIGRATE_VMA_HELPER	672	config MIGRATE_VMA_HELPER
699	bool	673	bool
700		674
701	config DEV_PAGEMAP_OPS	675	config DEV_PAGEMAP_OPS
702	bool	676	bool
703		677
704	config HMM
705	bool
706	select MMU_NOTIFIER
707	select MIGRATE_VMA_HELPER
708
709	config HMM_MIRROR	678	config HMM_MIRROR
710	bool "HMM mirror CPU page table into a device page table"	679	bool "HMM mirror CPU page table into a device page table"
711	depends on ARCH_HAS_HMM	680	depends on (X86_64 \|\| PPC64)
712	select HMM	681	depends on MMU && 64BIT
		682	select MMU_NOTIFIER
713	help	683	help
714	Select HMM_MIRROR if you want to mirror range of the CPU page table of a	684	Select HMM_MIRROR if you want to mirror range of the CPU page table of a
715	process into a device page table. Here, mirror means "keep synchronized".	685	process into a device page table. Here, mirror means "keep synchronized".
@@ -719,8 +689,7 @@ config HMM_MIRROR · 3 unmodified lines
719		689
720	config DEVICE_PRIVATE	690	config DEVICE_PRIVATE
721	bool "Unaddressable device memory (GPU memory, ...)"	691	bool "Unaddressable device memory (GPU memory, ...)"
722	depends on ARCH_HAS_HMM	692	depends on ZONE_DEVICE
723	select HMM
724	select DEV_PAGEMAP_OPS	693	select DEV_PAGEMAP_OPS
725		694
726	help	695	help
@@ -728,17 +697,6 @@ config DEVICE_PRIVATE · 1 unmodified line
728	memory; i.e., memory that is only accessible from the device (or	697	memory; i.e., memory that is only accessible from the device (or
729	group of devices). You likely also want to select HMM_MIRROR.	698	group of devices). You likely also want to select HMM_MIRROR.
730		699
731	config DEVICE_PUBLIC
732	bool "Addressable device memory (like GPU memory)"
733	depends on ARCH_HAS_HMM
734	select HMM
735	select DEV_PAGEMAP_OPS
736
737	help
738	Allows creation of struct pages to represent addressable device
739	memory; i.e., memory that is accessible from both the device and
740	the CPU
741
742	config FRAME_VECTOR	700	config FRAME_VECTOR
743	bool	701	bool
744		702

mm/Makefile+1−1

@@ -102,5 +102,5 @@ obj-$(CONFIG_FRAME_VECTOR) += frame_vector.o · 101 unmodified lines
102	obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o	102	obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o
103	obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o	103	obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o
104	obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o	104	obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o
105	obj-$(CONFIG_HMM) += hmm.o	105	obj-$(CONFIG_HMM_MIRROR) += hmm.o
106	obj-$(CONFIG_MEMFD_CREATE) += memfd.o	106	obj-$(CONFIG_MEMFD_CREATE) += memfd.o

mm/gup.c+0−7

@@ -605,13 +605,6 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address, · 604 unmodified lines
605	if ((gup_flags & FOLL_DUMP) \|\| !is_zero_pfn(pte_pfn(*pte)))	605	if ((gup_flags & FOLL_DUMP) \|\| !is_zero_pfn(pte_pfn(*pte)))
606	goto unmap;	606	goto unmap;
607	page = pte_page(pte);	607	page = pte_page(pte);
608
609	/*
610	* This should never happen (a device public page in the gate
611	* area).
612	*/
613	if (is_device_public_page(*page))
614	goto unmap;
615	}	608	}
616	if (unlikely(!try_get_page(*page))) {	609	if (unlikely(!try_get_page(*page))) {
617	ret = -ENOMEM;	610	ret = -ENOMEM;

mm/hmm.c+0−284

@@ -26,9 +26,6 @@ · 25 unmodified lines
26	#include <linux/mmu_notifier.h>	26	#include <linux/mmu_notifier.h>
27	#include <linux/memory_hotplug.h>	27	#include <linux/memory_hotplug.h>
28		28
29	#define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)
30
31	#if IS_ENABLED(CONFIG_HMM_MIRROR)
32	static const struct mmu_notifier_ops hmm_mmu_notifier_ops;	29	static const struct mmu_notifier_ops hmm_mmu_notifier_ops;
33		30
34	/**	31	/**
@@ -1287,284 +1284,3 @@ long hmm_range_dma_unmap(struct hmm_range *range, · 1252 unmodified lines
1287	return cpages;	1284	return cpages;
1288	}	1285	}
1289	EXPORT_SYMBOL(hmm_range_dma_unmap);	1286	EXPORT_SYMBOL(hmm_range_dma_unmap);
1290	#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
1291
1292
1293	#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) \|\| IS_ENABLED(CONFIG_DEVICE_PUBLIC)
1294	struct page hmm_vma_alloc_locked_page(struct vm_area_struct vma,
1295	unsigned long addr)
1296	{
1297	struct page *page;
1298
1299	page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
1300	if (!page)
1301	return NULL;
1302	lock_page(page);
1303	return page;
1304	}
1305	EXPORT_SYMBOL(hmm_vma_alloc_locked_page);
1306
1307
1308	static void hmm_devmem_ref_release(struct percpu_ref *ref)
1309	{
1310	struct hmm_devmem *devmem;
1311
1312	devmem = container_of(ref, struct hmm_devmem, ref);
1313	complete(&devmem->completion);
1314	}
1315
1316	static void hmm_devmem_ref_exit(struct percpu_ref *ref)
1317	{
1318	struct hmm_devmem *devmem;
1319
1320	devmem = container_of(ref, struct hmm_devmem, ref);
1321	wait_for_completion(&devmem->completion);
1322	percpu_ref_exit(ref);
1323	}
1324
1325	static void hmm_devmem_ref_kill(struct percpu_ref *ref)
1326	{
1327	percpu_ref_kill(ref);
1328	}
1329
1330	static vm_fault_t hmm_devmem_fault(struct vm_area_struct *vma,
1331	unsigned long addr,
1332	const struct page *page,
1333	unsigned int flags,
1334	pmd_t *pmdp)
1335	{
1336	struct hmm_devmem *devmem = page->pgmap->data;
1337
1338	return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp);
1339	}
1340
1341	static void hmm_devmem_free(struct page page, void data)
1342	{
1343	struct hmm_devmem *devmem = data;
1344
1345	page->mapping = NULL;
1346
1347	devmem->ops->free(devmem, page);
1348	}
1349
1350	/*
1351	* hmm_devmem_add() - hotplug ZONE_DEVICE memory for device memory
1352	*
1353	* @ops: memory event device driver callback (see struct hmm_devmem_ops)
1354	* @device: device struct to bind the resource too
1355	* @size: size in bytes of the device memory to add
1356	* Return: pointer to new hmm_devmem struct ERR_PTR otherwise
1357	*
1358	* This function first finds an empty range of physical address big enough to
1359	* contain the new resource, and then hotplugs it as ZONE_DEVICE memory, which
1360	* in turn allocates struct pages. It does not do anything beyond that; all
1361	* events affecting the memory will go through the various callbacks provided
1362	* by hmm_devmem_ops struct.
1363	*
1364	* Device driver should call this function during device initialization and
1365	* is then responsible of memory management. HMM only provides helpers.
1366	*/
1367	struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops,
1368	struct device *device,
1369	unsigned long size)
1370	{
1371	struct hmm_devmem *devmem;
1372	resource_size_t addr;
1373	void *result;
1374	int ret;
1375
1376	dev_pagemap_get_ops();
1377
1378	devmem = devm_kzalloc(device, sizeof(*devmem), GFP_KERNEL);
1379	if (!devmem)
1380	return ERR_PTR(-ENOMEM);
1381
1382	init_completion(&devmem->completion);
1383	devmem->pfn_first = -1UL;
1384	devmem->pfn_last = -1UL;
1385	devmem->resource = NULL;
1386	devmem->device = device;
1387	devmem->ops = ops;
1388
1389	ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1390	0, GFP_KERNEL);
1391	if (ret)
1392	return ERR_PTR(ret);
1393
1394	size = ALIGN(size, PA_SECTION_SIZE);
1395	addr = min((unsigned long)iomem_resource.end,
1396	(1UL << MAX_PHYSMEM_BITS) - 1);
1397	addr = addr - size + 1UL;
1398
1399	/*
1400	* FIXME add a new helper to quickly walk resource tree and find free
1401	* range
1402	*
1403	* FIXME what about ioport_resource resource ?
1404	*/
1405	for (; addr > size && addr >= iomem_resource.start; addr -= size) {
1406	ret = region_intersects(addr, size, 0, IORES_DESC_NONE);
1407	if (ret != REGION_DISJOINT)
1408	continue;
1409
1410	devmem->resource = devm_request_mem_region(device, addr, size,
1411	dev_name(device));
1412	if (!devmem->resource)
1413	return ERR_PTR(-ENOMEM);
1414	break;
1415	}
1416	if (!devmem->resource)
1417	return ERR_PTR(-ERANGE);
1418
1419	devmem->resource->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
1420	devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1421	devmem->pfn_last = devmem->pfn_first +
1422	(resource_size(devmem->resource) >> PAGE_SHIFT);
1423	devmem->page_fault = hmm_devmem_fault;
1424
1425	devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
1426	devmem->pagemap.res = *devmem->resource;
1427	devmem->pagemap.page_free = hmm_devmem_free;
1428	devmem->pagemap.altmap_valid = false;
1429	devmem->pagemap.ref = &devmem->ref;
1430	devmem->pagemap.data = devmem;
1431	devmem->pagemap.kill = hmm_devmem_ref_kill;
1432	devmem->pagemap.cleanup = hmm_devmem_ref_exit;
1433
1434	result = devm_memremap_pages(devmem->device, &devmem->pagemap);
1435	if (IS_ERR(result))
1436	return result;
1437	return devmem;
1438	}
1439	EXPORT_SYMBOL_GPL(hmm_devmem_add);
1440
1441	struct hmm_devmem hmm_devmem_add_resource(const struct hmm_devmem_ops ops,
1442	struct device *device,
1443	struct resource *res)
1444	{
1445	struct hmm_devmem *devmem;
1446	void *result;
1447	int ret;
1448
1449	if (res->desc != IORES_DESC_DEVICE_PUBLIC_MEMORY)
1450	return ERR_PTR(-EINVAL);
1451
1452	dev_pagemap_get_ops();
1453
1454	devmem = devm_kzalloc(device, sizeof(*devmem), GFP_KERNEL);
1455	if (!devmem)
1456	return ERR_PTR(-ENOMEM);
1457
1458	init_completion(&devmem->completion);
1459	devmem->pfn_first = -1UL;
1460	devmem->pfn_last = -1UL;
1461	devmem->resource = res;
1462	devmem->device = device;
1463	devmem->ops = ops;
1464
1465	ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1466	0, GFP_KERNEL);
1467	if (ret)
1468	return ERR_PTR(ret);
1469
1470	devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1471	devmem->pfn_last = devmem->pfn_first +
1472	(resource_size(devmem->resource) >> PAGE_SHIFT);
1473	devmem->page_fault = hmm_devmem_fault;
1474
1475	devmem->pagemap.type = MEMORY_DEVICE_PUBLIC;
1476	devmem->pagemap.res = *devmem->resource;
1477	devmem->pagemap.page_free = hmm_devmem_free;
1478	devmem->pagemap.altmap_valid = false;
1479	devmem->pagemap.ref = &devmem->ref;
1480	devmem->pagemap.data = devmem;
1481	devmem->pagemap.kill = hmm_devmem_ref_kill;
1482	devmem->pagemap.cleanup = hmm_devmem_ref_exit;
1483
1484	result = devm_memremap_pages(devmem->device, &devmem->pagemap);
1485	if (IS_ERR(result))
1486	return result;
1487	return devmem;
1488	}
1489	EXPORT_SYMBOL_GPL(hmm_devmem_add_resource);
1490
1491	/*
1492	* A device driver that wants to handle multiple devices memory through a
1493	* single fake device can use hmm_device to do so. This is purely a helper
1494	* and it is not needed to make use of any HMM functionality.
1495	*/
1496	#define HMM_DEVICE_MAX 256
1497
1498	static DECLARE_BITMAP(hmm_device_mask, HMM_DEVICE_MAX);
1499	static DEFINE_SPINLOCK(hmm_device_lock);
1500	static struct class *hmm_device_class;
1501	static dev_t hmm_device_devt;
1502
1503	static void hmm_device_release(struct device *device)
1504	{
1505	struct hmm_device *hmm_device;
1506
1507	hmm_device = container_of(device, struct hmm_device, device);
1508	spin_lock(&hmm_device_lock);
1509	clear_bit(hmm_device->minor, hmm_device_mask);
1510	spin_unlock(&hmm_device_lock);
1511
1512	kfree(hmm_device);
1513	}
1514
1515	struct hmm_device hmm_device_new(void drvdata)
1516	{
1517	struct hmm_device *hmm_device;
1518
1519	hmm_device = kzalloc(sizeof(*hmm_device), GFP_KERNEL);
1520	if (!hmm_device)
1521	return ERR_PTR(-ENOMEM);
1522
1523	spin_lock(&hmm_device_lock);
1524	hmm_device->minor = find_first_zero_bit(hmm_device_mask, HMM_DEVICE_MAX);
1525	if (hmm_device->minor >= HMM_DEVICE_MAX) {
1526	spin_unlock(&hmm_device_lock);
1527	kfree(hmm_device);
1528	return ERR_PTR(-EBUSY);
1529	}
1530	set_bit(hmm_device->minor, hmm_device_mask);
1531	spin_unlock(&hmm_device_lock);
1532
1533	dev_set_name(&hmm_device->device, "hmm_device%d", hmm_device->minor);
1534	hmm_device->device.devt = MKDEV(MAJOR(hmm_device_devt),
1535	hmm_device->minor);
1536	hmm_device->device.release = hmm_device_release;
1537	dev_set_drvdata(&hmm_device->device, drvdata);
1538	hmm_device->device.class = hmm_device_class;
1539	device_initialize(&hmm_device->device);
1540
1541	return hmm_device;
1542	}
1543	EXPORT_SYMBOL(hmm_device_new);
1544
1545	void hmm_device_put(struct hmm_device *hmm_device)
1546	{
1547	put_device(&hmm_device->device);
1548	}
1549	EXPORT_SYMBOL(hmm_device_put);
1550
1551	static int __init hmm_init(void)
1552	{
1553	int ret;
1554
1555	ret = alloc_chrdev_region(&hmm_device_devt, 0,
1556	HMM_DEVICE_MAX,
1557	"hmm_device");
1558	if (ret)
1559	return ret;
1560
1561	hmm_device_class = class_create(THIS_MODULE, "hmm_device");
1562	if (IS_ERR(hmm_device_class)) {
1563	unregister_chrdev_region(hmm_device_devt, HMM_DEVICE_MAX);
1564	return PTR_ERR(hmm_device_class);
1565	}
1566	return 0;
1567	}
1568
1569	device_initcall(hmm_init);
1570	#endif /* CONFIG_DEVICE_PRIVATE \|\| CONFIG_DEVICE_PUBLIC */

mm/madvise.c+1−1

@@ -354,7 +354,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, · 353 unmodified lines
354	continue;	354	continue;
355	}	355	}
356		356
357	page = _vm_normal_page(vma, addr, ptent, true);	357	page = vm_normal_page(vma, addr, ptent);
358	if (!page)	358	if (!page)
359	continue;	359	continue;
360		360

mm/memcontrol.c+6−7

 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
 						unsigned long addr, pte_t ptent)
 {
-	struct page *page = _vm_normal_page(vma, addr, ptent, true);
+	struct page *page = vm_normal_page(vma, addr, ptent);
 	if (!page || !page_mapped(page))
 		return NULL;
  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
  *     target for charge migration. if @target is not NULL, the entry is stored
  *     in target->ent.
- *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PUBLIC
+ *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PRIVATE
- *     or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru).
+ *     (so ZONE_DEVICE page and thus not on the lru).
  *     For now we such page is charge like a regular page would be as for all
  *     intent and purposes it is just special memory taking the place of a
  *     regular page.
 		 */
 		if (page->mem_cgroup == mc.from) {
 			ret = MC_TARGET_PAGE;
-			if (is_device_private_page(page) ||
+			if (is_device_private_page(page))
-			    is_device_public_page(page))
 				ret = MC_TARGET_DEVICE;
 			if (target)
 				target->page = page;
 	if (ptl) {
 		/*
 		 * Note their can not be MC_TARGET_DEVICE for now as we do not
-		 * support transparent huge page with MEMORY_DEVICE_PUBLIC or
+		 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
-		 * MEMORY_DEVICE_PRIVATE but this might change.
+		 * this might change.
 		 */
 		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
 			mc.precharge += HPAGE_PMD_NR;

mm/memory-failure.c+1−5

 		goto unlock;
 	}
-	switch (pgmap->type) {
+	if (pgmap->type == MEMORY_DEVICE_PRIVATE) {
-	case MEMORY_DEVICE_PRIVATE:
-	case MEMORY_DEVICE_PUBLIC:
 		/*
 		 * TODO: Handle HMM pages which may need coordination
 		 * with device-side memory.
 		 */
 		goto unlock;
-	default:
-		break;
 	}
 	/*

mm/memory.c+5−44

@@ -571,8 +571,8 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, · 570 unmodified lines
571	* PFNMAP mappings in order to support COWable mappings.	571	* PFNMAP mappings in order to support COWable mappings.
572	*	572	*
573	*/	573	*/
574	struct page _vm_normal_page(struct vm_area_struct vma, unsigned long addr,	574	struct page vm_normal_page(struct vm_area_struct vma, unsigned long addr,
575	pte_t pte, bool with_public_device)	575	pte_t pte)
576	{	576	{
577	unsigned long pfn = pte_pfn(pte);	577	unsigned long pfn = pte_pfn(pte);
578		578
@@ -585,29 +585,6 @@ struct page _vm_normal_page(struct vm_area_struct vma, unsigned long addr, · 6 unmodified lines
585	return NULL;	585	return NULL;
586	if (is_zero_pfn(pfn))	586	if (is_zero_pfn(pfn))
587	return NULL;	587	return NULL;
588
589	/*
590	* Device public pages are special pages (they are ZONE_DEVICE
591	* pages but different from persistent memory). They behave
592	* allmost like normal pages. The difference is that they are
593	* not on the lru and thus should never be involve with any-
594	* thing that involve lru manipulation (mlock, numa balancing,
595	* ...).
596	*
597	* This is why we still want to return NULL for such page from
598	* vm_normal_page() so that we do not have to special case all
599	* call site of vm_normal_page().
600	*/
601	if (likely(pfn <= highest_memmap_pfn)) {
602	struct page *page = pfn_to_page(pfn);
603
604	if (is_device_public_page(page)) {
605	if (with_public_device)
606	return page;
607	return NULL;
608	}
609	}
610
611	if (pte_devmap(pte))	588	if (pte_devmap(pte))
612	return NULL;	589	return NULL;
613		590
@@ -797,17 +774,6 @@ copy_one_pte(struct mm_struct dst_mm, struct mm_struct src_mm, · 183 unmodified lines
797	rss[mm_counter(page)]++;	774	rss[mm_counter(page)]++;
798	} else if (pte_devmap(pte)) {	775	} else if (pte_devmap(pte)) {
799	page = pte_page(pte);	776	page = pte_page(pte);
800
801	/*
802	* Cache coherent device memory behave like regular page and
803	* not like persistent memory page. For more informations see
804	* MEMORY_DEVICE_CACHE_COHERENT in memory_hotplug.h
805	*/
806	if (is_device_public_page(page)) {
807	get_page(page);
808	page_dup_rmap(page, false);
809	rss[mm_counter(page)]++;
810	}
811	}	777	}
812		778
813	out_set_pte:	779	out_set_pte:
@@ -1063,7 +1029,7 @@ again: · 249 unmodified lines
1063	if (pte_present(ptent)) {	1029	if (pte_present(ptent)) {
1064	struct page *page;	1030	struct page *page;
1065		1031
1066	page = _vm_normal_page(vma, addr, ptent, true);	1032	page = vm_normal_page(vma, addr, ptent);
1067	if (unlikely(details) && page) {	1033	if (unlikely(details) && page) {
1068	/*	1034	/*
1069	* unmap_shared_mapping_pages() wants to	1035	* unmap_shared_mapping_pages() wants to
@@ -2782,13 +2748,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) · 1712 unmodified lines
2782	migration_entry_wait(vma->vm_mm, vmf->pmd,	2748	migration_entry_wait(vma->vm_mm, vmf->pmd,
2783	vmf->address);	2749	vmf->address);
2784	} else if (is_device_private_entry(entry)) {	2750	} else if (is_device_private_entry(entry)) {
2785	/*	2751	vmf->page = device_private_entry_to_page(entry);
2786	* For un-addressable device memory we call the pgmap	2752	ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
2787	* fault handler callback. The callback must migrate
2788	* the page back to some CPU accessible page.
2789	*/
2790	ret = device_private_entry_fault(vma, vmf->address, entry,
2791	vmf->flags, vmf->pmd);
2792	} else if (is_hwpoison_entry(entry)) {	2753	} else if (is_hwpoison_entry(entry)) {
2793	ret = VM_FAULT_HWPOISON;	2754	ret = VM_FAULT_HWPOISON;
2794	} else {	2755	} else {

mm/memory_hotplug.c+2−4

@@ -557,10 +557,8 @@ void __remove_pages(struct zone *zone, unsigned long phys_start_pfn, · 556 unmodified lines
557	int sections_to_remove;	557	int sections_to_remove;
558		558
559	/* In the ZONE_DEVICE case device driver owns the memory region */	559	/* In the ZONE_DEVICE case device driver owns the memory region */
560	if (is_dev_zone(zone)) {	560	if (is_dev_zone(zone))
561	if (altmap)	561	map_offset = vmem_altmap_offset(altmap);
562	map_offset = vmem_altmap_offset(altmap);
563	}
564		562
565	clear_zone_contiguous(zone);	563	clear_zone_contiguous(zone);
566		564

mm/mempolicy.c+1−0

@@ -2098,6 +2098,7 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma, · 2097 unmodified lines
2098	out:	2098	out:
2099	return page;	2099	return page;
2100	}	2100	}
		2101	EXPORT_SYMBOL(alloc_pages_vma);
2101		2102
2102	/**	2103	/**
2103	* alloc_pages_current - Allocate pages.	2104	* alloc_pages_current - Allocate pages.

mm/migrate.c+4−24

 			if (is_device_private_page(new)) {
 				entry = make_device_private_entry(new, pte_write(pte));
 				pte = swp_entry_to_pte(entry);
-			} else if (is_device_public_page(new)) {
-				pte = pte_mkdevmap(pte);
 			}
 		}
 	 * ZONE_DEVICE pages.
 	 */
 	expected_count += is_device_private_page(page);
-	expected_count += is_device_public_page(page);
 	if (mapping)
 		expected_count += hpage_nr_pages(page) + page_has_private(page);
 		if (!PageMappingFlags(page))
 			page->mapping = NULL;
-		if (unlikely(is_zone_device_page(newpage))) {
+		if (likely(!is_zone_device_page(newpage)))
-			if (is_device_public_page(newpage))
-				flush_dcache_page(newpage);
-		} else
 			flush_dcache_page(newpage);
 	}
 				pfn = 0;
 				goto next;
 			}
-			page = _vm_normal_page(migrate->vma, addr, pte, true);
+			page = vm_normal_page(migrate->vma, addr, pte);
 			mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
 			mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
 		}
 		 * FIXME proper solution is to rework migration_entry_wait() so
 		 * it does not need to take a reference on page.
 		 */
-		if (is_device_private_page(page))
+		return is_device_private_page(page);
-			return true;
-		/*
-		 * Only allow device public page to be migrated and account for
-		 * the extra reference count imply by ZONE_DEVICE pages.
-		 */
-		if (!is_device_public_page(page))
-			return false;
-		extra++;
 	}
 	/* For file back page */
 			swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
 			entry = swp_entry_to_pte(swp_entry);
-		} else if (is_device_public_page(page)) {
-			entry = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
-			if (vma->vm_flags & VM_WRITE)
-				entry = pte_mkwrite(pte_mkdirty(entry));
-			entry = pte_mkdevmap(entry);
 		}
 	} else {
 		entry = mk_pte(page, vma->vm_page_prot);
 					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
 					continue;
 				}
-			} else if (!is_device_public_page(newpage)) {
+			} else {
 				/*
 				 * Other types of ZONE_DEVICE page are not
 				 * supported.

mm/page_alloc.c+6−7

 {
 	unsigned long pfn, end_pfn = start_pfn + size;
 	struct pglist_data *pgdat = zone->zone_pgdat;
+	struct vmem_altmap *altmap = pgmap_altmap(pgmap);
 	unsigned long zone_idx = zone_idx(zone);
 	unsigned long start = jiffies;
 	int nid = pgdat->node_id;
 	 * of the pages reserved for the memmap, so we can just jump to
 	 * the end of that region and start processing the device pages.
 	 */
-	if (pgmap->altmap_valid) {
+	if (altmap) {
-		struct vmem_altmap *altmap = &pgmap->altmap;
 		start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
 		size = end_pfn - start_pfn;
 	}
 		__SetPageReserved(page);
 		/*
-		 * ZONE_DEVICE pages union ->lru with a ->pgmap back
+		 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
-		 * pointer and hmm_data.  It is a bug if a ZONE_DEVICE
+		 * and zone_device_data.  It is a bug if a ZONE_DEVICE page is
-		 * page is ever freed or placed on a driver-private list.
+		 * ever freed or placed on a driver-private list.
 		 */
 		page->pgmap = pgmap;
-		page->hmm_data = 0;
+		page->zone_device_data = NULL;
 		/*
 		 * Mark the block movable so that blocks are reserved for

tools/testing/nvdimm/test/iomap.c+46−11

@@ -100,25 +100,60 @@ static void nfit_test_kill(void *_pgmap) · 99 unmodified lines
100	{	100	{
101	struct dev_pagemap *pgmap = _pgmap;	101	struct dev_pagemap *pgmap = _pgmap;
102		102
103	WARN_ON(!pgmap \|\| !pgmap->ref \|\| !pgmap->kill \|\| !pgmap->cleanup);	103	WARN_ON(!pgmap \|\| !pgmap->ref);
104	pgmap->kill(pgmap->ref);	104
105	pgmap->cleanup(pgmap->ref);	105	if (pgmap->ops && pgmap->ops->kill)
		106	pgmap->ops->kill(pgmap);
		107	else
		108	percpu_ref_kill(pgmap->ref);
		109
		110	if (pgmap->ops && pgmap->ops->cleanup) {
		111	pgmap->ops->cleanup(pgmap);
		112	} else {
		113	wait_for_completion(&pgmap->done);
		114	percpu_ref_exit(pgmap->ref);
		115	}
		116	}
		117
		118	static void dev_pagemap_percpu_release(struct percpu_ref *ref)
		119	{
		120	struct dev_pagemap *pgmap =
		121	container_of(ref, struct dev_pagemap, internal_ref);
		122
		123	complete(&pgmap->done);
106	}	124	}
107		125
108	void __wrap_devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap)	126	void __wrap_devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap)
109	{	127	{
		128	int error;
110	resource_size_t offset = pgmap->res.start;	129	resource_size_t offset = pgmap->res.start;
111	struct nfit_test_resource *nfit_res = get_nfit_res(offset);	130	struct nfit_test_resource *nfit_res = get_nfit_res(offset);
112		131
113	if (nfit_res) {	132	if (!nfit_res)
114	int rc;	133	return devm_memremap_pages(dev, pgmap);
115		134
116	rc = devm_add_action_or_reset(dev, nfit_test_kill, pgmap);	135	pgmap->dev = dev;
117	if (rc)	136	if (!pgmap->ref) {
118	return ERR_PTR(rc);	137	if (pgmap->ops && (pgmap->ops->kill \|\| pgmap->ops->cleanup))
119	return nfit_res->buf + offset - nfit_res->res.start;	138	return ERR_PTR(-EINVAL);
		139
		140	init_completion(&pgmap->done);
		141	error = percpu_ref_init(&pgmap->internal_ref,
		142	dev_pagemap_percpu_release, 0, GFP_KERNEL);
		143	if (error)
		144	return ERR_PTR(error);
		145	pgmap->ref = &pgmap->internal_ref;
		146	} else {
		147	if (!pgmap->ops \|\| !pgmap->ops->kill \|\| !pgmap->ops->cleanup) {
		148	WARN(1, "Missing reference count teardown definition\n");
		149	return ERR_PTR(-EINVAL);
		150	}
120	}	151	}
121	return devm_memremap_pages(dev, pgmap);	152
		153	error = devm_add_action_or_reset(dev, nfit_test_kill, pgmap);
		154	if (error)
		155	return ERR_PTR(error);
		156	return nfit_res->buf + offset - nfit_res->res.start;
122	}	157	}
123	EXPORT_SYMBOL_GPL(__wrap_devm_memremap_pages);	158	EXPORT_SYMBOL_GPL(__wrap_devm_memremap_pages);
124		159

Merge branch 'hmm-devmem-cleanup.4' into rdma.git hmm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code