Merge branch 'hmm-devmem-cleanup.4' into rdma.git hmm

Christoph Hellwig says: ==================== Below is a series that cleans up the dev_pagemap interface so that it is more easily usable, which removes the need to wrap it in hmm and thus allowing to kill a lot of code Changes since v3: - pull in "mm/swap: Fix release_pages() when releasing devmap pages" and rebase the other patches on top of that - fold the hmm_devmem_add_resource into the DEVICE_PUBLIC memory removal patch - remove _vm_normal_page as it isn't needed without DEVICE_PUBLIC memory - pick up various ACKs Changes since v2: - fix nvdimm kunit build - add a new memory type for device dax - fix a few issues in intermediate patches that didn't show up in the end result - incorporate feedback from Michal Hocko, including killing of the DEVICE_PUBLIC memory type entirely Changes since v1: - rebase - also switch p2pdma to the internal refcount - add type checking for pgmap->type - rename the migrate method to migrate_to_ram - cleanup the altmap_valid flag - various tidbits from the reviews ==================== Conflicts resolved by: - Keeping Ira's version of the code in swap.c - Using the delete for the section in hmm.rst - Using the delete for the devmap code in hmm.c and .h * branch 'hmm-devmem-cleanup.4': (24 commits) mm: don't select MIGRATE_VMA_HELPER from HMM_MIRROR mm: remove the HMM config option mm: sort out the DEVICE_PRIVATE Kconfig mess mm: simplify ZONE_DEVICE page private data mm: remove hmm_devmem_add mm: remove hmm_vma_alloc_locked_page nouveau: use devm_memremap_pages directly nouveau: use alloc_page_vma directly PCI/P2PDMA: use the dev_pagemap internal refcount device-dax: use the dev_pagemap internal refcount memremap: provide an optional internal refcount in struct dev_pagemap memremap: replace the altmap_valid field with a PGMAP_ALTMAP_VALID flag memremap: remove the data field in struct dev_pagemap memremap: add a migrate_to_ram method to struct dev_pagemap_ops memremap: lift the devmap_enable manipulation into devm_memremap_pages memremap: pass a struct dev_pagemap to ->kill and ->cleanup memremap: move dev_pagemap callbacks into a separate structure memremap: validate the pagemap type passed to devm_memremap_pages mm: factor out a devm_request_free_mem_region helper mm: export alloc_pages_vma ...

Jason Gunthorpe committed 7 years ago

Commit cc5dfd52 parents 9ec3f4c b6b346a33 files · +370 −1013Browse files

Showing the changes to include/linux (6 of 33 files). Show all

Unified · Split

6 files

include/linux/hmm.h +1 −197
include/linux/ioport.h +2 −1
include/linux/memremap.h +49 −26
include/linux/mm.h +2 −26
include/linux/mm_types.h +2 −2
include/linux/swapops.h +0 −15

include/linux/hmm.h+1−197

@@ -62,7 +62,7 @@ · 61 unmodified lines
62	#include <linux/kconfig.h>	62	#include <linux/kconfig.h>
63	#include <asm/pgtable.h>	63	#include <asm/pgtable.h>
64		64
65	#if IS_ENABLED(CONFIG_HMM)	65	#ifdef CONFIG_HMM_MIRROR
66		66
67	#include <linux/device.h>	67	#include <linux/device.h>
68	#include <linux/migrate.h>	68	#include <linux/migrate.h>
@@ -324,9 +324,6 @@ static inline uint64_t hmm_pfn_from_pfn(const struct hmm_range *range, · 255 unmodified lines
324	return hmm_device_entry_from_pfn(range, pfn);	324	return hmm_device_entry_from_pfn(range, pfn);
325	}	325	}
326		326
327
328
329	#if IS_ENABLED(CONFIG_HMM_MIRROR)
330	/*	327	/*
331	* Mirroring: how to synchronize device page table with CPU page table.	328	* Mirroring: how to synchronize device page table with CPU page table.
332	*	329	*
@@ -550,197 +547,4 @@ static inline void hmm_mm_init(struct mm_struct *mm) · 217 unmodified lines
550	static inline void hmm_mm_init(struct mm_struct *mm) {}	547	static inline void hmm_mm_init(struct mm_struct *mm) {}
551	#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */	548	#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
552		549
553	#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) \|\| IS_ENABLED(CONFIG_DEVICE_PUBLIC)
554	struct hmm_devmem;
555
556	struct page hmm_vma_alloc_locked_page(struct vm_area_struct vma,
557	unsigned long addr);
558
559	/*
560	* struct hmm_devmem_ops - callback for ZONE_DEVICE memory events
561	*
562	* @free: call when refcount on page reach 1 and thus is no longer use
563	* @fault: call when there is a page fault to unaddressable memory
564	*
565	* Both callback happens from page_free() and page_fault() callback of struct
566	* dev_pagemap respectively. See include/linux/memremap.h for more details on
567	* those.
568	*
569	* The hmm_devmem_ops callback are just here to provide a coherent and
570	* uniq API to device driver and device driver should not register their
571	* own page_free() or page_fault() but rely on the hmm_devmem_ops call-
572	* back.
573	*/
574	struct hmm_devmem_ops {
575	/*
576	* free() - free a device page
577	* @devmem: device memory structure (see struct hmm_devmem)
578	* @page: pointer to struct page being freed
579	*
580	* Call back occurs whenever a device page refcount reach 1 which
581	* means that no one is holding any reference on the page anymore
582	* (ZONE_DEVICE page have an elevated refcount of 1 as default so
583	* that they are not release to the general page allocator).
584	*
585	* Note that callback has exclusive ownership of the page (as no
586	* one is holding any reference).
587	*/
588	void (free)(struct hmm_devmem devmem, struct page *page);
589	/*
590	* fault() - CPU page fault or get user page (GUP)
591	* @devmem: device memory structure (see struct hmm_devmem)
592	* @vma: virtual memory area containing the virtual address
593	* @addr: virtual address that faulted or for which there is a GUP
594	* @page: pointer to struct page backing virtual address (unreliable)
595	* @flags: FAULT_FLAG_* (see include/linux/mm.h)
596	* @pmdp: page middle directory
597	* Return: VM_FAULT_MINOR/MAJOR on success or one of VM_FAULT_ERROR
598	* on error
599	*
600	* The callback occurs whenever there is a CPU page fault or GUP on a
601	* virtual address. This means that the device driver must migrate the
602	* page back to regular memory (CPU accessible).
603	*
604	* The device driver is free to migrate more than one page from the
605	* fault() callback as an optimization. However if the device decides
606	* to migrate more than one page it must always priotirize the faulting
607	* address over the others.
608	*
609	* The struct page pointer is only given as a hint to allow quick
610	* lookup of internal device driver data. A concurrent migration
611	* might have already freed that page and the virtual address might
612	* no longer be backed by it. So it should not be modified by the
613	* callback.
614	*
615	* Note that mmap semaphore is held in read mode at least when this
616	* callback occurs, hence the vma is valid upon callback entry.
617	*/
618	vm_fault_t (fault)(struct hmm_devmem devmem,
619	struct vm_area_struct *vma,
620	unsigned long addr,
621	const struct page *page,
622	unsigned int flags,
623	pmd_t *pmdp);
624	};
625
626	/*
627	* struct hmm_devmem - track device memory
628	*
629	* @completion: completion object for device memory
630	* @pfn_first: first pfn for this resource (set by hmm_devmem_add())
631	* @pfn_last: last pfn for this resource (set by hmm_devmem_add())
632	* @resource: IO resource reserved for this chunk of memory
633	* @pagemap: device page map for that chunk
634	* @device: device to bind resource to
635	* @ops: memory operations callback
636	* @ref: per CPU refcount
637	* @page_fault: callback when CPU fault on an unaddressable device page
638	*
639	* This is a helper structure for device drivers that do not wish to implement
640	* the gory details related to hotplugging new memoy and allocating struct
641	* pages.
642	*
643	* Device drivers can directly use ZONE_DEVICE memory on their own if they
644	* wish to do so.
645	*
646	* The page_fault() callback must migrate page back, from device memory to
647	* system memory, so that the CPU can access it. This might fail for various
648	* reasons (device issues, device have been unplugged, ...). When such error
649	* conditions happen, the page_fault() callback must return VM_FAULT_SIGBUS and
650	* set the CPU page table entry to "poisoned".
651	*
652	* Note that because memory cgroup charges are transferred to the device memory,
653	* this should never fail due to memory restrictions. However, allocation
654	* of a regular system page might still fail because we are out of memory. If
655	* that happens, the page_fault() callback must return VM_FAULT_OOM.
656	*
657	* The page_fault() callback can also try to migrate back multiple pages in one
658	* chunk, as an optimization. It must, however, prioritize the faulting address
659	* over all the others.
660	*/
661	typedef vm_fault_t (dev_page_fault_t)(struct vm_area_struct vma,
662	unsigned long addr,
663	const struct page *page,
664	unsigned int flags,
665	pmd_t *pmdp);
666
667	struct hmm_devmem {
668	struct completion completion;
669	unsigned long pfn_first;
670	unsigned long pfn_last;
671	struct resource *resource;
672	struct device *device;
673	struct dev_pagemap pagemap;
674	const struct hmm_devmem_ops *ops;
675	struct percpu_ref ref;
676	dev_page_fault_t page_fault;
677	};
678
679	/*
680	* To add (hotplug) device memory, HMM assumes that there is no real resource
681	* that reserves a range in the physical address space (this is intended to be
682	* use by unaddressable device memory). It will reserve a physical range big
683	* enough and allocate struct page for it.
684	*
685	* The device driver can wrap the hmm_devmem struct inside a private device
686	* driver struct.
687	*/
688	struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops,
689	struct device *device,
690	unsigned long size);
691	struct hmm_devmem hmm_devmem_add_resource(const struct hmm_devmem_ops ops,
692	struct device *device,
693	struct resource *res);
694
695	/*
696	* hmm_devmem_page_set_drvdata - set per-page driver data field
697	*
698	* @page: pointer to struct page
699	* @data: driver data value to set
700	*
701	* Because page can not be on lru we have an unsigned long that driver can use
702	* to store a per page field. This just a simple helper to do that.
703	*/
704	static inline void hmm_devmem_page_set_drvdata(struct page *page,
705	unsigned long data)
706	{
707	page->hmm_data = data;
708	}
709
710	/*
711	* hmm_devmem_page_get_drvdata - get per page driver data field
712	*
713	* @page: pointer to struct page
714	* Return: driver data value
715	*/
716	static inline unsigned long hmm_devmem_page_get_drvdata(const struct page *page)
717	{
718	return page->hmm_data;
719	}
720
721
722	/*
723	* struct hmm_device - fake device to hang device memory onto
724	*
725	* @device: device struct
726	* @minor: device minor number
727	*/
728	struct hmm_device {
729	struct device device;
730	unsigned int minor;
731	};
732
733	/*
734	* A device driver that wants to handle multiple devices memory through a
735	* single fake device can use hmm_device to do so. This is purely a helper and
736	* it is not strictly needed, in order to make use of any HMM functionality.
737	*/
738	struct hmm_device hmm_device_new(void drvdata);
739	void hmm_device_put(struct hmm_device *hmm_device);
740	#endif /* CONFIG_DEVICE_PRIVATE \|\| CONFIG_DEVICE_PUBLIC */
741	#else /* IS_ENABLED(CONFIG_HMM) */
742	static inline void hmm_mm_destroy(struct mm_struct *mm) {}
743	static inline void hmm_mm_init(struct mm_struct *mm) {}
744	#endif /* IS_ENABLED(CONFIG_HMM) */
745
746	#endif /* LINUX_HMM_H */	550	#endif /* LINUX_HMM_H */

include/linux/ioport.h+2−1

 	IORES_DESC_PERSISTENT_MEMORY		= 4,
 	IORES_DESC_PERSISTENT_MEMORY_LEGACY	= 5,
 	IORES_DESC_DEVICE_PRIVATE_MEMORY	= 6,
-	IORES_DESC_DEVICE_PUBLIC_MEMORY		= 7,
 };
 /* helpers to define resources */
        return (r1->start <= r2->end && r1->end >= r2->start);
 }
+struct resource *devm_request_free_mem_region(struct device *dev,
+		struct resource *base, unsigned long size);
 #endif /* __ASSEMBLY__ */
 #endif	/* _LINUX_IOPORT_H */

include/linux/memremap.h+49−26

@@ -37,13 +37,6 @@ struct vmem_altmap { · 36 unmodified lines
37	* A more complete discussion of unaddressable memory may be found in	37	* A more complete discussion of unaddressable memory may be found in
38	* include/linux/hmm.h and Documentation/vm/hmm.rst.	38	* include/linux/hmm.h and Documentation/vm/hmm.rst.
39	*	39	*
40	* MEMORY_DEVICE_PUBLIC:
41	* Device memory that is cache coherent from device and CPU point of view. This
42	* is use on platform that have an advance system bus (like CAPI or CCIX). A
43	* driver can hotplug the device memory using ZONE_DEVICE and with that memory
44	* type. Any page of a process can be migrated to such memory. However no one
45	* should be allow to pin such memory so that it can always be evicted.
46	*
47	* MEMORY_DEVICE_FS_DAX:	40	* MEMORY_DEVICE_FS_DAX:
48	* Host memory that has similar access semantics as System RAM i.e. DMA	41	* Host memory that has similar access semantics as System RAM i.e. DMA
49	* coherent and supports page pinning. In support of coordinating page	42	* coherent and supports page pinning. In support of coordinating page
@@ -52,54 +45,84 @@ struct vmem_altmap { · 2 unmodified lines
52	* wakeup is used to coordinate physical address space management (ex:	45	* wakeup is used to coordinate physical address space management (ex:
53	* fs truncate/hole punch) vs pinned pages (ex: device dma).	46	* fs truncate/hole punch) vs pinned pages (ex: device dma).
54	*	47	*
		48	* MEMORY_DEVICE_DEVDAX:
		49	* Host memory that has similar access semantics as System RAM i.e. DMA
		50	* coherent and supports page pinning. In contrast to
		51	* MEMORY_DEVICE_FS_DAX, this memory is access via a device-dax
		52	* character device.
		53	*
55	* MEMORY_DEVICE_PCI_P2PDMA:	54	* MEMORY_DEVICE_PCI_P2PDMA:
56	* Device memory residing in a PCI BAR intended for use with Peer-to-Peer	55	* Device memory residing in a PCI BAR intended for use with Peer-to-Peer
57	* transactions.	56	* transactions.
58	*/	57	*/
59	enum memory_type {	58	enum memory_type {
		59	/* 0 is reserved to catch uninitialized type fields */
60	MEMORY_DEVICE_PRIVATE = 1,	60	MEMORY_DEVICE_PRIVATE = 1,
61	MEMORY_DEVICE_PUBLIC,
62	MEMORY_DEVICE_FS_DAX,	61	MEMORY_DEVICE_FS_DAX,
		62	MEMORY_DEVICE_DEVDAX,
63	MEMORY_DEVICE_PCI_P2PDMA,	63	MEMORY_DEVICE_PCI_P2PDMA,
64	};	64	};
65		65
66	/*	66	struct dev_pagemap_ops {
67	* Additional notes about MEMORY_DEVICE_PRIVATE may be found in	67	/*
68	* include/linux/hmm.h and Documentation/vm/hmm.rst. There is also a brief	68	* Called once the page refcount reaches 1. (ZONE_DEVICE pages never
69	* explanation in include/linux/memory_hotplug.h.	69	* reach 0 refcount unless there is a refcount bug. This allows the
70	*	70	* device driver to implement its own memory management.)
71	* The page_free() callback is called once the page refcount reaches 1	71	*/
72	* (ZONE_DEVICE pages never reach 0 refcount unless there is a refcount bug.	72	void (page_free)(struct page page);
73	* This allows the device driver to implement its own memory management.)	73
74	*/	74	/*
75	typedef void (dev_page_free_t)(struct page page, void *data);	75	* Transition the refcount in struct dev_pagemap to the dead state.
		76	*/
		77	void (kill)(struct dev_pagemap pgmap);
		78
		79	/*
		80	* Wait for refcount in struct dev_pagemap to be idle and reap it.
		81	*/
		82	void (cleanup)(struct dev_pagemap pgmap);
		83
		84	/*
		85	* Used for private (un-addressable) device memory only. Must migrate
		86	* the page back to a CPU accessible page.
		87	*/
		88	vm_fault_t (migrate_to_ram)(struct vm_fault vmf);
		89	};
		90
		91	#define PGMAP_ALTMAP_VALID (1 << 0)
76		92
77	/**	93	/**
78	* struct dev_pagemap - metadata for ZONE_DEVICE mappings	94	* struct dev_pagemap - metadata for ZONE_DEVICE mappings
79	* @page_free: free page callback when page refcount reaches 1
80	* @altmap: pre-allocated/reserved memory for vmemmap allocations	95	* @altmap: pre-allocated/reserved memory for vmemmap allocations
81	* @res: physical address range covered by @ref	96	* @res: physical address range covered by @ref
82	* @ref: reference count that pins the devm_memremap_pages() mapping	97	* @ref: reference count that pins the devm_memremap_pages() mapping
83	* @kill: callback to transition @ref to the dead state	98	* @internal_ref: internal reference if @ref is not provided by the caller
84	* @cleanup: callback to wait for @ref to be idle and reap it	99	* @done: completion for @internal_ref
85	* @dev: host device of the mapping for debug	100	* @dev: host device of the mapping for debug
86	* @data: private data pointer for page_free()	101	* @data: private data pointer for page_free()
87	* @type: memory type: see MEMORY_* in memory_hotplug.h	102	* @type: memory type: see MEMORY_* in memory_hotplug.h
		103	* @flags: PGMAP_* flags to specify defailed behavior
		104	* @ops: method table
88	*/	105	*/
89	struct dev_pagemap {	106	struct dev_pagemap {
90	dev_page_free_t page_free;
91	struct vmem_altmap altmap;	107	struct vmem_altmap altmap;
92	bool altmap_valid;
93	struct resource res;	108	struct resource res;
94	struct percpu_ref *ref;	109	struct percpu_ref *ref;
95	void (kill)(struct percpu_ref ref);	110	struct percpu_ref internal_ref;
96	void (cleanup)(struct percpu_ref ref);	111	struct completion done;
97	struct device *dev;	112	struct device *dev;
98	void *data;
99	enum memory_type type;	113	enum memory_type type;
		114	unsigned int flags;
100	u64 pci_p2pdma_bus_offset;	115	u64 pci_p2pdma_bus_offset;
		116	const struct dev_pagemap_ops *ops;
101	};	117	};
102		118
		119	static inline struct vmem_altmap pgmap_altmap(struct dev_pagemap pgmap)
		120	{
		121	if (pgmap->flags & PGMAP_ALTMAP_VALID)
		122	return &pgmap->altmap;
		123	return NULL;
		124	}
		125
103	#ifdef CONFIG_ZONE_DEVICE	126	#ifdef CONFIG_ZONE_DEVICE
104	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap);	127	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap);
105	void devm_memunmap_pages(struct device dev, struct dev_pagemap pgmap);	128	void devm_memunmap_pages(struct device dev, struct dev_pagemap pgmap);

include/linux/mm.h+2−26

 #endif
 #ifdef CONFIG_DEV_PAGEMAP_OPS
-void dev_pagemap_get_ops(void);
-void dev_pagemap_put_ops(void);
 void __put_devmap_managed_page(struct page *page);
 DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
 static inline bool put_devmap_managed_page(struct page *page)
 		return false;
 	switch (page->pgmap->type) {
 	case MEMORY_DEVICE_PRIVATE:
-	case MEMORY_DEVICE_PUBLIC:
 	case MEMORY_DEVICE_FS_DAX:
 		__put_devmap_managed_page(page);
 		return true;
 		page->pgmap->type == MEMORY_DEVICE_PRIVATE;
 }
-static inline bool is_device_public_page(const struct page *page)
-{
-	return is_zone_device_page(page) &&
-		page->pgmap->type == MEMORY_DEVICE_PUBLIC;
-}
 #ifdef CONFIG_PCI_P2PDMA
 static inline bool is_pci_p2pdma_page(const struct page *page)
 {
 #endif /* CONFIG_PCI_P2PDMA */
 #else /* CONFIG_DEV_PAGEMAP_OPS */
-static inline void dev_pagemap_get_ops(void)
-{
-}
-static inline void dev_pagemap_put_ops(void)
-{
-}
 static inline bool put_devmap_managed_page(struct page *page)
 {
 	return false;
 	return false;
 }
-static inline bool is_device_public_page(const struct page *page)
-{
-	return false;
-}
 static inline bool is_pci_p2pdma_page(const struct page *page)
 {
 	return false;
 	pgoff_t last_index;			/* Highest page->index to unmap */
 };
-struct page *_vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
+struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte, bool with_public_device);
+			     pte_t pte);
-#define vm_normal_page(vma, addr, pte) _vm_normal_page(vma, addr, pte, false)
 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
 				pmd_t pmd);

include/linux/mm_types.h+2−2

@@ -158,7 +158,7 @@ struct page { · 157 unmodified lines
158	struct { /* ZONE_DEVICE pages */	158	struct { /* ZONE_DEVICE pages */
159	/** @pgmap: Points to the hosting device page map. */	159	/** @pgmap: Points to the hosting device page map. */
160	struct dev_pagemap *pgmap;	160	struct dev_pagemap *pgmap;
161	unsigned long hmm_data;	161	void *zone_device_data;
162	unsigned long _zd_pad_1; /* uses mapping */	162	unsigned long _zd_pad_1; /* uses mapping */
163	};	163	};
164		164
@@ -501,7 +501,7 @@ struct mm_struct { · 336 unmodified lines
501	#endif	501	#endif
502	struct work_struct async_put_work;	502	struct work_struct async_put_work;
503		503
504	#if IS_ENABLED(CONFIG_HMM)	504	#ifdef CONFIG_HMM_MIRROR
505	/* HMM needs to track a few things per mm */	505	/* HMM needs to track a few things per mm */
506	struct hmm *hmm;	506	struct hmm *hmm;
507	#endif	507	#endif

include/linux/swapops.h+0−15

@@ -129,12 +129,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 128 unmodified lines
129	{	129	{
130	return pfn_to_page(swp_offset(entry));	130	return pfn_to_page(swp_offset(entry));
131	}	131	}
132
133	vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
134	unsigned long addr,
135	swp_entry_t entry,
136	unsigned int flags,
137	pmd_t *pmdp);
138	#else /* CONFIG_DEVICE_PRIVATE */	132	#else /* CONFIG_DEVICE_PRIVATE */
139	static inline swp_entry_t make_device_private_entry(struct page *page, bool write)	133	static inline swp_entry_t make_device_private_entry(struct page *page, bool write)
140	{	134	{
@@ -164,15 +158,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 23 unmodified lines
164	{	158	{
165	return NULL;	159	return NULL;
166	}	160	}
167
168	static inline vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
169	unsigned long addr,
170	swp_entry_t entry,
171	unsigned int flags,
172	pmd_t *pmdp)
173	{
174	return VM_FAULT_SIGBUS;
175	}
176	#endif /* CONFIG_DEVICE_PRIVATE */	161	#endif /* CONFIG_DEVICE_PRIVATE */
177		162
178	#ifdef CONFIG_MIGRATION	163	#ifdef CONFIG_MIGRATION

Merge branch 'hmm-devmem-cleanup.4' into rdma.git hmm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code