memremap: add a migrate_to_ram method to struct dev_pagemap_ops

This replaces the hacky ->fault callback, which is currently directly called from common code through a hmm specific data structure as an exercise in layering violations.

Christoph Hellwig committed 7 years ago

Commit 897e6361 parent f6a55e16 files · +17 −67Browse files

Unified · Split

6 files

include/linux/hmm.h +0 −6
include/linux/memremap.h +6 −0
include/linux/swapops.h +0 −15
kernel/memremap.c +4 −31
mm/hmm.c +5 −8
mm/memory.c +2 −7

include/linux/hmm.h+0−6

@@ -692,11 +692,6 @@ struct hmm_devmem_ops { · 691 unmodified lines
692	692	* chunk, as an optimization. It must, however, prioritize the faulting address
693	693	* over all the others.
694	694	*/
695		−typedef vm_fault_t (dev_page_fault_t)(struct vm_area_struct vma,
696		− unsigned long addr,
697		− const struct page *page,
698		− unsigned int flags,
699		− pmd_t *pmdp);
700	695
701	696	struct hmm_devmem {
702	697	struct completion completion;
@@ -707,7 +702,6 @@ struct hmm_devmem { · 4 unmodified lines
707	702	struct dev_pagemap pagemap;
708	703	const struct hmm_devmem_ops *ops;
709	704	struct percpu_ref ref;
710		− dev_page_fault_t page_fault;
711	705	};
712	706
713	707	/*

include/linux/memremap.h+6−0

@@ -80,6 +80,12 @@ struct dev_pagemap_ops { · 79 unmodified lines
80	80	* Wait for refcount in struct dev_pagemap to be idle and reap it.
81	81	*/
82	82	void (cleanup)(struct dev_pagemap pgmap);
	83	+
	84	+ /*
	85	+ * Used for private (un-addressable) device memory only. Must migrate
	86	+ * the page back to a CPU accessible page.
	87	+ */
	88	+ vm_fault_t (migrate_to_ram)(struct vm_fault vmf);
83	89	};
84	90
85	91	/**

include/linux/swapops.h+0−15

@@ -129,12 +129,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 128 unmodified lines
129	129	{
130	130	return pfn_to_page(swp_offset(entry));
131	131	}
132		−
133		−vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
134		− unsigned long addr,
135		− swp_entry_t entry,
136		− unsigned int flags,
137		− pmd_t *pmdp);
138	132	#else /* CONFIG_DEVICE_PRIVATE */
139	133	static inline swp_entry_t make_device_private_entry(struct page *page, bool write)
140	134	{
@@ -164,15 +158,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 23 unmodified lines
164	158	{
165	159	return NULL;
166	160	}
167		−
168		−static inline vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
169		− unsigned long addr,
170		− swp_entry_t entry,
171		− unsigned int flags,
172		− pmd_t *pmdp)
173		−{
174		− return VM_FAULT_SIGBUS;
175		−}
176	161	#endif /* CONFIG_DEVICE_PRIVATE */
177	162
178	163	#ifdef CONFIG_MIGRATION

kernel/memremap.c+4−31

@@ -11,7 +11,6 @@ · 10 unmodified lines
11	11	#include <linux/types.h>
12	12	#include <linux/wait_bit.h>
13	13	#include <linux/xarray.h>
14		−#include <linux/hmm.h>
15	14
16	15	static DEFINE_XARRAY(pgmap_array);
17	16	#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
@@ -46,36 +45,6 @@ static int devmap_managed_enable_get(struct device dev, struct dev_pagemap pgm · 28 unmodified lines
46	45	}
47	46	#endif /* CONFIG_DEV_PAGEMAP_OPS */
48	47
49		−#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
50		−vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
51		− unsigned long addr,
52		− swp_entry_t entry,
53		− unsigned int flags,
54		− pmd_t *pmdp)
55		−{
56		− struct page *page = device_private_entry_to_page(entry);
57		− struct hmm_devmem *devmem;
58		−
59		− devmem = container_of(page->pgmap, typeof(*devmem), pagemap);
60		−
61		− /*
62		− * The page_fault() callback must migrate page back to system memory
63		− * so that CPU can access it. This might fail for various reasons
64		− * (device issue, device was unsafely unplugged, ...). When such
65		− * error conditions happen, the callback must return VM_FAULT_SIGBUS.
66		− *
67		− * Note that because memory cgroup charges are accounted to the device
68		− * memory, this should never fail because of memory restrictions (but
69		− * allocation of regular system page might still fail because we are
70		− * out of memory).
71		− *
72		− * There is a more in-depth description of what that callback can and
73		− * cannot do, in include/linux/memremap.h
74		− */
75		− return devmem->page_fault(vma, addr, page, flags, pmdp);
76		−}
77		−#endif /* CONFIG_DEVICE_PRIVATE */
78		−
79	48	static void pgmap_array_delete(struct resource *res)
80	49	{
81	50	xa_store_range(&pgmap_array, PHYS_PFN(res->start), PHYS_PFN(res->end),
@@ -193,6 +162,10 @@ void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap) · 111 unmodified lines
193	162	WARN(1, "Device private memory not supported\n");
194	163	return ERR_PTR(-EINVAL);
195	164	}
	165	+ if (!pgmap->ops \|\| !pgmap->ops->migrate_to_ram) {
	166	+ WARN(1, "Missing migrate_to_ram method\n");
	167	+ return ERR_PTR(-EINVAL);
	168	+ }
196	169	break;
197	170	case MEMORY_DEVICE_FS_DAX:
198	171	if (!IS_ENABLED(CONFIG_ZONE_DEVICE) \|\|

mm/hmm.c+5−8

@@ -1366,15 +1366,12 @@ static void hmm_devmem_ref_kill(struct dev_pagemap *pgmap) · 1365 unmodified lines
1366	1366	percpu_ref_kill(pgmap->ref);
1367	1367	}
1368	1368
1369		−static vm_fault_t hmm_devmem_fault(struct vm_area_struct *vma,
1370		− unsigned long addr,
1371		− const struct page *page,
1372		− unsigned int flags,
1373		− pmd_t *pmdp)
	1369	+static vm_fault_t hmm_devmem_migrate_to_ram(struct vm_fault *vmf)
1374	1370	{
1375		− struct hmm_devmem *devmem = page->pgmap->data;
	1371	+ struct hmm_devmem *devmem = vmf->page->pgmap->data;
1376	1372
1377		− return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp);
	1373	+ return devmem->ops->fault(devmem, vmf->vma, vmf->address, vmf->page,
	1374	+ vmf->flags, vmf->pmd);
1378	1375	}
1379	1376
1380	1377	static void hmm_devmem_free(struct page page, void data)
@@ -1388,6 +1385,7 @@ static const struct dev_pagemap_ops hmm_pagemap_ops = { · 7 unmodified lines
1388	1385	.page_free = hmm_devmem_free,
1389	1386	.kill = hmm_devmem_ref_kill,
1390	1387	.cleanup = hmm_devmem_ref_exit,
	1388	+ .migrate_to_ram = hmm_devmem_migrate_to_ram,
1391	1389	};
1392	1390
1393	1391	/*
@@ -1438,7 +1436,6 @@ struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops, · 44 unmodified lines
1438	1436	devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1439	1437	devmem->pfn_last = devmem->pfn_first +
1440	1438	(resource_size(devmem->resource) >> PAGE_SHIFT);
1441		− devmem->page_fault = hmm_devmem_fault;
1442	1439
1443	1440	devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
1444	1441	devmem->pagemap.res = *devmem->resource;

mm/memory.c+2−7

@@ -2748,13 +2748,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) · 2747 unmodified lines
2748	2748	migration_entry_wait(vma->vm_mm, vmf->pmd,
2749	2749	vmf->address);
2750	2750	} else if (is_device_private_entry(entry)) {
2751		− /*
2752		− * For un-addressable device memory we call the pgmap
2753		− * fault handler callback. The callback must migrate
2754		− * the page back to some CPU accessible page.
2755		− */
2756		− ret = device_private_entry_fault(vma, vmf->address, entry,
2757		− vmf->flags, vmf->pmd);
	2751	+ vmf->page = device_private_entry_to_page(entry);
	2752	+ ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
2758	2753	} else if (is_hwpoison_entry(entry)) {
2759	2754	ret = VM_FAULT_HWPOISON;
2760	2755	} else {

memremap: add a migrate_to_ram method to struct dev_pagemap_ops

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code