Merge branch 'hmm-devmem-cleanup.4' into rdma.git hmm

Christoph Hellwig says: ==================== Below is a series that cleans up the dev_pagemap interface so that it is more easily usable, which removes the need to wrap it in hmm and thus allowing to kill a lot of code Changes since v3: - pull in "mm/swap: Fix release_pages() when releasing devmap pages" and rebase the other patches on top of that - fold the hmm_devmem_add_resource into the DEVICE_PUBLIC memory removal patch - remove _vm_normal_page as it isn't needed without DEVICE_PUBLIC memory - pick up various ACKs Changes since v2: - fix nvdimm kunit build - add a new memory type for device dax - fix a few issues in intermediate patches that didn't show up in the end result - incorporate feedback from Michal Hocko, including killing of the DEVICE_PUBLIC memory type entirely Changes since v1: - rebase - also switch p2pdma to the internal refcount - add type checking for pgmap->type - rename the migrate method to migrate_to_ram - cleanup the altmap_valid flag - various tidbits from the reviews ==================== Conflicts resolved by: - Keeping Ira's version of the code in swap.c - Using the delete for the section in hmm.rst - Using the delete for the devmap code in hmm.c and .h * branch 'hmm-devmem-cleanup.4': (24 commits) mm: don't select MIGRATE_VMA_HELPER from HMM_MIRROR mm: remove the HMM config option mm: sort out the DEVICE_PRIVATE Kconfig mess mm: simplify ZONE_DEVICE page private data mm: remove hmm_devmem_add mm: remove hmm_vma_alloc_locked_page nouveau: use devm_memremap_pages directly nouveau: use alloc_page_vma directly PCI/P2PDMA: use the dev_pagemap internal refcount device-dax: use the dev_pagemap internal refcount memremap: provide an optional internal refcount in struct dev_pagemap memremap: replace the altmap_valid field with a PGMAP_ALTMAP_VALID flag memremap: remove the data field in struct dev_pagemap memremap: add a migrate_to_ram method to struct dev_pagemap_ops memremap: lift the devmap_enable manipulation into devm_memremap_pages memremap: pass a struct dev_pagemap to ->kill and ->cleanup memremap: move dev_pagemap callbacks into a separate structure memremap: validate the pagemap type passed to devm_memremap_pages mm: factor out a devm_request_free_mem_region helper mm: export alloc_pages_vma ...

Jason Gunthorpe committed 7 years ago

Commit cc5dfd52 parents 9ec3f4c b6b346a33 files · +370 −1013Browse files

Showing the changes to include/linux (6 of 33 files). Show all

Unified · Split

6 files

include/linux/hmm.h +1 −197
include/linux/ioport.h +2 −1
include/linux/memremap.h +49 −26
include/linux/mm.h +2 −26
include/linux/mm_types.h +2 −2
include/linux/swapops.h +0 −15

include/linux/hmm.h+1−197

@@ -62,7 +62,7 @@ · 61 unmodified lines
62	62	#include <linux/kconfig.h>
63	63	#include <asm/pgtable.h>
64	64
65		−#if IS_ENABLED(CONFIG_HMM)
	65	+#ifdef CONFIG_HMM_MIRROR
66	66
67	67	#include <linux/device.h>
68	68	#include <linux/migrate.h>
@@ -324,9 +324,6 @@ static inline uint64_t hmm_pfn_from_pfn(const struct hmm_range *range, · 255 unmodified lines
324	324	return hmm_device_entry_from_pfn(range, pfn);
325	325	}
326	326
327		−
328		−
329		−#if IS_ENABLED(CONFIG_HMM_MIRROR)
330	327	/*
331	328	* Mirroring: how to synchronize device page table with CPU page table.
332	329	*
@@ -550,197 +547,4 @@ static inline void hmm_mm_init(struct mm_struct *mm) · 217 unmodified lines
550	547	static inline void hmm_mm_init(struct mm_struct *mm) {}
551	548	#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
552	549
553		−#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) \|\| IS_ENABLED(CONFIG_DEVICE_PUBLIC)
554		−struct hmm_devmem;
555		−
556		−struct page hmm_vma_alloc_locked_page(struct vm_area_struct vma,
557		− unsigned long addr);
558		−
559		−/*
560		− * struct hmm_devmem_ops - callback for ZONE_DEVICE memory events
561		− *
562		− * @free: call when refcount on page reach 1 and thus is no longer use
563		− * @fault: call when there is a page fault to unaddressable memory
564		− *
565		− * Both callback happens from page_free() and page_fault() callback of struct
566		− * dev_pagemap respectively. See include/linux/memremap.h for more details on
567		− * those.
568		− *
569		− * The hmm_devmem_ops callback are just here to provide a coherent and
570		− * uniq API to device driver and device driver should not register their
571		− * own page_free() or page_fault() but rely on the hmm_devmem_ops call-
572		− * back.
573		− */
574		−struct hmm_devmem_ops {
575		− /*
576		− * free() - free a device page
577		− * @devmem: device memory structure (see struct hmm_devmem)
578		− * @page: pointer to struct page being freed
579		− *
580		− * Call back occurs whenever a device page refcount reach 1 which
581		− * means that no one is holding any reference on the page anymore
582		− * (ZONE_DEVICE page have an elevated refcount of 1 as default so
583		− * that they are not release to the general page allocator).
584		− *
585		− * Note that callback has exclusive ownership of the page (as no
586		− * one is holding any reference).
587		− */
588		− void (free)(struct hmm_devmem devmem, struct page *page);
589		− /*
590		− * fault() - CPU page fault or get user page (GUP)
591		− * @devmem: device memory structure (see struct hmm_devmem)
592		− * @vma: virtual memory area containing the virtual address
593		− * @addr: virtual address that faulted or for which there is a GUP
594		− * @page: pointer to struct page backing virtual address (unreliable)
595		− * @flags: FAULT_FLAG_* (see include/linux/mm.h)
596		− * @pmdp: page middle directory
597		− * Return: VM_FAULT_MINOR/MAJOR on success or one of VM_FAULT_ERROR
598		− * on error
599		− *
600		− * The callback occurs whenever there is a CPU page fault or GUP on a
601		− * virtual address. This means that the device driver must migrate the
602		− * page back to regular memory (CPU accessible).
603		− *
604		− * The device driver is free to migrate more than one page from the
605		− * fault() callback as an optimization. However if the device decides
606		− * to migrate more than one page it must always priotirize the faulting
607		− * address over the others.
608		− *
609		− * The struct page pointer is only given as a hint to allow quick
610		− * lookup of internal device driver data. A concurrent migration
611		− * might have already freed that page and the virtual address might
612		− * no longer be backed by it. So it should not be modified by the
613		− * callback.
614		− *
615		− * Note that mmap semaphore is held in read mode at least when this
616		− * callback occurs, hence the vma is valid upon callback entry.
617		− */
618		− vm_fault_t (fault)(struct hmm_devmem devmem,
619		− struct vm_area_struct *vma,
620		− unsigned long addr,
621		− const struct page *page,
622		− unsigned int flags,
623		− pmd_t *pmdp);
624		−};
625		−
626		−/*
627		− * struct hmm_devmem - track device memory
628		− *
629		− * @completion: completion object for device memory
630		− * @pfn_first: first pfn for this resource (set by hmm_devmem_add())
631		− * @pfn_last: last pfn for this resource (set by hmm_devmem_add())
632		− * @resource: IO resource reserved for this chunk of memory
633		− * @pagemap: device page map for that chunk
634		− * @device: device to bind resource to
635		− * @ops: memory operations callback
636		− * @ref: per CPU refcount
637		− * @page_fault: callback when CPU fault on an unaddressable device page
638		− *
639		− * This is a helper structure for device drivers that do not wish to implement
640		− * the gory details related to hotplugging new memoy and allocating struct
641		− * pages.
642		− *
643		− * Device drivers can directly use ZONE_DEVICE memory on their own if they
644		− * wish to do so.
645		− *
646		− * The page_fault() callback must migrate page back, from device memory to
647		− * system memory, so that the CPU can access it. This might fail for various
648		− * reasons (device issues, device have been unplugged, ...). When such error
649		− * conditions happen, the page_fault() callback must return VM_FAULT_SIGBUS and
650		− * set the CPU page table entry to "poisoned".
651		− *
652		− * Note that because memory cgroup charges are transferred to the device memory,
653		− * this should never fail due to memory restrictions. However, allocation
654		− * of a regular system page might still fail because we are out of memory. If
655		− * that happens, the page_fault() callback must return VM_FAULT_OOM.
656		− *
657		− * The page_fault() callback can also try to migrate back multiple pages in one
658		− * chunk, as an optimization. It must, however, prioritize the faulting address
659		− * over all the others.
660		− */
661		−typedef vm_fault_t (dev_page_fault_t)(struct vm_area_struct vma,
662		− unsigned long addr,
663		− const struct page *page,
664		− unsigned int flags,
665		− pmd_t *pmdp);
666		−
667		−struct hmm_devmem {
668		− struct completion completion;
669		− unsigned long pfn_first;
670		− unsigned long pfn_last;
671		− struct resource *resource;
672		− struct device *device;
673		− struct dev_pagemap pagemap;
674		− const struct hmm_devmem_ops *ops;
675		− struct percpu_ref ref;
676		− dev_page_fault_t page_fault;
677		−};
678		−
679		−/*
680		− * To add (hotplug) device memory, HMM assumes that there is no real resource
681		− * that reserves a range in the physical address space (this is intended to be
682		− * use by unaddressable device memory). It will reserve a physical range big
683		− * enough and allocate struct page for it.
684		− *
685		− * The device driver can wrap the hmm_devmem struct inside a private device
686		− * driver struct.
687		− */
688		−struct hmm_devmem hmm_devmem_add(const struct hmm_devmem_ops ops,
689		− struct device *device,
690		− unsigned long size);
691		−struct hmm_devmem hmm_devmem_add_resource(const struct hmm_devmem_ops ops,
692		− struct device *device,
693		− struct resource *res);
694		−
695		−/*
696		− * hmm_devmem_page_set_drvdata - set per-page driver data field
697		− *
698		− * @page: pointer to struct page
699		− * @data: driver data value to set
700		− *
701		− * Because page can not be on lru we have an unsigned long that driver can use
702		− * to store a per page field. This just a simple helper to do that.
703		− */
704		−static inline void hmm_devmem_page_set_drvdata(struct page *page,
705		− unsigned long data)
706		−{
707		− page->hmm_data = data;
708		−}
709		−
710		−/*
711		− * hmm_devmem_page_get_drvdata - get per page driver data field
712		− *
713		− * @page: pointer to struct page
714		− * Return: driver data value
715		− */
716		−static inline unsigned long hmm_devmem_page_get_drvdata(const struct page *page)
717		−{
718		− return page->hmm_data;
719		−}
720		−
721		−
722		−/*
723		− * struct hmm_device - fake device to hang device memory onto
724		− *
725		− * @device: device struct
726		− * @minor: device minor number
727		− */
728		−struct hmm_device {
729		− struct device device;
730		− unsigned int minor;
731		−};
732		−
733		−/*
734		− * A device driver that wants to handle multiple devices memory through a
735		− * single fake device can use hmm_device to do so. This is purely a helper and
736		− * it is not strictly needed, in order to make use of any HMM functionality.
737		− */
738		−struct hmm_device hmm_device_new(void drvdata);
739		−void hmm_device_put(struct hmm_device *hmm_device);
740		−#endif /* CONFIG_DEVICE_PRIVATE \|\| CONFIG_DEVICE_PUBLIC */
741		−#else /* IS_ENABLED(CONFIG_HMM) */
742		−static inline void hmm_mm_destroy(struct mm_struct *mm) {}
743		−static inline void hmm_mm_init(struct mm_struct *mm) {}
744		−#endif /* IS_ENABLED(CONFIG_HMM) */
745		−
746	550	#endif /* LINUX_HMM_H */

include/linux/ioport.h+2−1

@@ -132,7 +132,6 @@ enum { · 131 unmodified lines
132	132	IORES_DESC_PERSISTENT_MEMORY = 4,
133	133	IORES_DESC_PERSISTENT_MEMORY_LEGACY = 5,
134	134	IORES_DESC_DEVICE_PRIVATE_MEMORY = 6,
135		− IORES_DESC_DEVICE_PUBLIC_MEMORY = 7,
136	135	};
137	136
138	137	/* helpers to define resources */
@@ -286,6 +285,8 @@ static inline bool resource_overlaps(struct resource r1, struct resource r2) · 147 unmodified lines
286	285	return (r1->start <= r2->end && r1->end >= r2->start);
287	286	}
288	287
	288	+struct resource devm_request_free_mem_region(struct device dev,
	289	+ struct resource *base, unsigned long size);
289	290
290	291	#endif /* __ASSEMBLY__ */
291	292	#endif /* _LINUX_IOPORT_H */

include/linux/memremap.h+49−26

@@ -37,13 +37,6 @@ struct vmem_altmap { · 36 unmodified lines
37	37	* A more complete discussion of unaddressable memory may be found in
38	38	* include/linux/hmm.h and Documentation/vm/hmm.rst.
39	39	*
40		− * MEMORY_DEVICE_PUBLIC:
41		− * Device memory that is cache coherent from device and CPU point of view. This
42		− * is use on platform that have an advance system bus (like CAPI or CCIX). A
43		− * driver can hotplug the device memory using ZONE_DEVICE and with that memory
44		− * type. Any page of a process can be migrated to such memory. However no one
45		− * should be allow to pin such memory so that it can always be evicted.
46		− *
47	40	* MEMORY_DEVICE_FS_DAX:
48	41	* Host memory that has similar access semantics as System RAM i.e. DMA
49	42	* coherent and supports page pinning. In support of coordinating page
@@ -52,54 +45,84 @@ struct vmem_altmap { · 2 unmodified lines
52	45	* wakeup is used to coordinate physical address space management (ex:
53	46	* fs truncate/hole punch) vs pinned pages (ex: device dma).
54	47	*
	48	+ * MEMORY_DEVICE_DEVDAX:
	49	+ * Host memory that has similar access semantics as System RAM i.e. DMA
	50	+ * coherent and supports page pinning. In contrast to
	51	+ * MEMORY_DEVICE_FS_DAX, this memory is access via a device-dax
	52	+ * character device.
	53	+ *
55	54	* MEMORY_DEVICE_PCI_P2PDMA:
56	55	* Device memory residing in a PCI BAR intended for use with Peer-to-Peer
57	56	* transactions.
58	57	*/
59	58	enum memory_type {
	59	+ /* 0 is reserved to catch uninitialized type fields */
60	60	MEMORY_DEVICE_PRIVATE = 1,
61		− MEMORY_DEVICE_PUBLIC,
62	61	MEMORY_DEVICE_FS_DAX,
	62	+ MEMORY_DEVICE_DEVDAX,
63	63	MEMORY_DEVICE_PCI_P2PDMA,
64	64	};
65	65
66		−/*
67		− * Additional notes about MEMORY_DEVICE_PRIVATE may be found in
68		− * include/linux/hmm.h and Documentation/vm/hmm.rst. There is also a brief
69		− * explanation in include/linux/memory_hotplug.h.
70		− *
71		− * The page_free() callback is called once the page refcount reaches 1
72		− * (ZONE_DEVICE pages never reach 0 refcount unless there is a refcount bug.
73		− * This allows the device driver to implement its own memory management.)
74		− */
75		−typedef void (dev_page_free_t)(struct page page, void *data);
	66	+struct dev_pagemap_ops {
	67	+ /*
	68	+ * Called once the page refcount reaches 1. (ZONE_DEVICE pages never
	69	+ * reach 0 refcount unless there is a refcount bug. This allows the
	70	+ * device driver to implement its own memory management.)
	71	+ */
	72	+ void (page_free)(struct page page);
	73	+
	74	+ /*
	75	+ * Transition the refcount in struct dev_pagemap to the dead state.
	76	+ */
	77	+ void (kill)(struct dev_pagemap pgmap);
	78	+
	79	+ /*
	80	+ * Wait for refcount in struct dev_pagemap to be idle and reap it.
	81	+ */
	82	+ void (cleanup)(struct dev_pagemap pgmap);
	83	+
	84	+ /*
	85	+ * Used for private (un-addressable) device memory only. Must migrate
	86	+ * the page back to a CPU accessible page.
	87	+ */
	88	+ vm_fault_t (migrate_to_ram)(struct vm_fault vmf);
	89	+};
	90	+
	91	+#define PGMAP_ALTMAP_VALID (1 << 0)
76	92
77	93	/**
78	94	* struct dev_pagemap - metadata for ZONE_DEVICE mappings
79		− * @page_free: free page callback when page refcount reaches 1
80	95	* @altmap: pre-allocated/reserved memory for vmemmap allocations
81	96	* @res: physical address range covered by @ref
82	97	* @ref: reference count that pins the devm_memremap_pages() mapping
83		− * @kill: callback to transition @ref to the dead state
84		− * @cleanup: callback to wait for @ref to be idle and reap it
	98	+ * @internal_ref: internal reference if @ref is not provided by the caller
	99	+ * @done: completion for @internal_ref
85	100	* @dev: host device of the mapping for debug
86	101	* @data: private data pointer for page_free()
87	102	* @type: memory type: see MEMORY_* in memory_hotplug.h
	103	+ * @flags: PGMAP_* flags to specify defailed behavior
	104	+ * @ops: method table
88	105	*/
89	106	struct dev_pagemap {
90		− dev_page_free_t page_free;
91	107	struct vmem_altmap altmap;
92		− bool altmap_valid;
93	108	struct resource res;
94	109	struct percpu_ref *ref;
95		− void (kill)(struct percpu_ref ref);
96		− void (cleanup)(struct percpu_ref ref);
	110	+ struct percpu_ref internal_ref;
	111	+ struct completion done;
97	112	struct device *dev;
98		− void *data;
99	113	enum memory_type type;
	114	+ unsigned int flags;
100	115	u64 pci_p2pdma_bus_offset;
	116	+ const struct dev_pagemap_ops *ops;
101	117	};
102	118
	119	+static inline struct vmem_altmap pgmap_altmap(struct dev_pagemap pgmap)
	120	+{
	121	+ if (pgmap->flags & PGMAP_ALTMAP_VALID)
	122	+ return &pgmap->altmap;
	123	+ return NULL;
	124	+}
	125	+
103	126	#ifdef CONFIG_ZONE_DEVICE
104	127	void devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap);
105	128	void devm_memunmap_pages(struct device dev, struct dev_pagemap pgmap);

include/linux/mm.h+2−26

@@ -932,8 +932,6 @@ static inline bool is_zone_device_page(const struct page *page) · 931 unmodified lines
932	932	#endif
933	933
934	934	#ifdef CONFIG_DEV_PAGEMAP_OPS
935		−void dev_pagemap_get_ops(void);
936		−void dev_pagemap_put_ops(void);
937	935	void __put_devmap_managed_page(struct page *page);
938	936	DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
939	937	static inline bool put_devmap_managed_page(struct page *page)
@@ -944,7 +942,6 @@ static inline bool put_devmap_managed_page(struct page *page) · 4 unmodified lines
944	942	return false;
945	943	switch (page->pgmap->type) {
946	944	case MEMORY_DEVICE_PRIVATE:
947		− case MEMORY_DEVICE_PUBLIC:
948	945	case MEMORY_DEVICE_FS_DAX:
949	946	__put_devmap_managed_page(page);
950	947	return true;
@@ -960,12 +957,6 @@ static inline bool is_device_private_page(const struct page *page) · 9 unmodified lines
960	957	page->pgmap->type == MEMORY_DEVICE_PRIVATE;
961	958	}
962	959
963		−static inline bool is_device_public_page(const struct page *page)
964		−{
965		− return is_zone_device_page(page) &&
966		− page->pgmap->type == MEMORY_DEVICE_PUBLIC;
967		−}
968		−
969	960	#ifdef CONFIG_PCI_P2PDMA
970	961	static inline bool is_pci_p2pdma_page(const struct page *page)
971	962	{
@@ -980,14 +971,6 @@ static inline bool is_pci_p2pdma_page(const struct page *page) · 8 unmodified lines
980	971	#endif /* CONFIG_PCI_P2PDMA */
981	972
982	973	#else /* CONFIG_DEV_PAGEMAP_OPS */
983		−static inline void dev_pagemap_get_ops(void)
984		−{
985		−}
986		−
987		−static inline void dev_pagemap_put_ops(void)
988		−{
989		−}
990		−
991	974	static inline bool put_devmap_managed_page(struct page *page)
992	975	{
993	976	return false;
@@ -998,11 +981,6 @@ static inline bool is_device_private_page(const struct page *page) · 4 unmodified lines
998	981	return false;
999	982	}
1000	983
1001		−static inline bool is_device_public_page(const struct page *page)
1002		−{
1003		− return false;
1004		−}
1005		−
1006	984	static inline bool is_pci_p2pdma_page(const struct page *page)
1007	985	{
1008	986	return false;
@@ -1431,10 +1409,8 @@ struct zap_details { · 422 unmodified lines
1431	1409	pgoff_t last_index; /* Highest page->index to unmap */
1432	1410	};
1433	1411
1434		−struct page _vm_normal_page(struct vm_area_struct vma, unsigned long addr,
1435		− pte_t pte, bool with_public_device);
1436		−#define vm_normal_page(vma, addr, pte) _vm_normal_page(vma, addr, pte, false)
1437		−
	1412	+struct page vm_normal_page(struct vm_area_struct vma, unsigned long addr,
	1413	+ pte_t pte);
1438	1414	struct page vm_normal_page_pmd(struct vm_area_struct vma, unsigned long addr,
1439	1415	pmd_t pmd);
1440	1416

include/linux/mm_types.h+2−2

@@ -158,7 +158,7 @@ struct page { · 157 unmodified lines
158	158	struct { /* ZONE_DEVICE pages */
159	159	/** @pgmap: Points to the hosting device page map. */
160	160	struct dev_pagemap *pgmap;
161		− unsigned long hmm_data;
	161	+ void *zone_device_data;
162	162	unsigned long _zd_pad_1; /* uses mapping */
163	163	};
164	164
@@ -501,7 +501,7 @@ struct mm_struct { · 336 unmodified lines
501	501	#endif
502	502	struct work_struct async_put_work;
503	503
504		−#if IS_ENABLED(CONFIG_HMM)
	504	+#ifdef CONFIG_HMM_MIRROR
505	505	/* HMM needs to track a few things per mm */
506	506	struct hmm *hmm;
507	507	#endif

include/linux/swapops.h+0−15

@@ -129,12 +129,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 128 unmodified lines
129	129	{
130	130	return pfn_to_page(swp_offset(entry));
131	131	}
132		−
133		−vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
134		− unsigned long addr,
135		− swp_entry_t entry,
136		− unsigned int flags,
137		− pmd_t *pmdp);
138	132	#else /* CONFIG_DEVICE_PRIVATE */
139	133	static inline swp_entry_t make_device_private_entry(struct page *page, bool write)
140	134	{
@@ -164,15 +158,6 @@ static inline struct page *device_private_entry_to_page(swp_entry_t entry) · 23 unmodified lines
164	158	{
165	159	return NULL;
166	160	}
167		−
168		−static inline vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
169		− unsigned long addr,
170		− swp_entry_t entry,
171		− unsigned int flags,
172		− pmd_t *pmdp)
173		−{
174		− return VM_FAULT_SIGBUS;
175		−}
176	161	#endif /* CONFIG_DEVICE_PRIVATE */
177	162
178	163	#ifdef CONFIG_MIGRATION

Merge branch 'hmm-devmem-cleanup.4' into rdma.git hmm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code