Merge tag 'drm-next-2020-04-03-1' of git://anongit.freedesktop.org/drm/drm

Pull drm hugepage support from Dave Airlie: "This adds support for hugepages to TTM and has been tested with the vmwgfx drivers, though I expect other drivers to start using it" * tag 'drm-next-2020-04-03-1' of git://anongit.freedesktop.org/drm/drm: drm/vmwgfx: Hook up the helpers to align buffer objects drm/vmwgfx: Introduce a huge page aligning TTM range manager drm: Add a drm_get_unmapped_area() helper drm/vmwgfx: Support huge page faults drm/ttm, drm/vmwgfx: Support huge TTM pagefaults mm: Add vmf_insert_pfn_xxx_prot() for huge page-table entries mm: Split huge pages on write-notify or COW mm: Introduce vma_is_special_huge fs: Constify vma argument to vma_is_dax

Linus Torvalds committed 6 years ago

Commit ea9448b2 parents 83eb69f 0e7e61916 files · +692 −28Browse files

Unified · Split

16 files

drivers/gpu/drm/drm_file.c +141 −0
drivers/gpu/drm/ttm/ttm_bo_vm.c +158 −3
drivers/gpu/drm/vmwgfx/Makefile +1 −0
drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +13 −0
drivers/gpu/drm/vmwgfx/vmwgfx_drv.h +12 −0
drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +74 −2
drivers/gpu/drm/vmwgfx/vmwgfx_thp.c +166 −0
drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c +1 −1
drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c +4 −1
include/drm/drm_file.h +9 −0
include/drm/ttm/ttm_bo_api.h +2 −1
include/linux/fs.h +1 −1
include/linux/huge_mm.h +39 −2
include/linux/mm.h +17 −0
mm/huge_memory.c +35 −9
mm/memory.c +19 −8

drivers/gpu/drm/drm_file.c+141−0

@@ -48,6 +48,11 @@ · 47 unmodified lines
48	48	#include "drm_internal.h"
49	49	#include "drm_legacy.h"
50	50
	51	+#if defined(CONFIG_MMU) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
	52	+#include <uapi/asm/mman.h>
	53	+#include <drm/drm_vma_manager.h>
	54	+#endif
	55	+
51	56	/* from BKL pushdown */
52	57	DEFINE_MUTEX(drm_global_mutex);
53	58
@@ -872,3 +877,139 @@ struct file mock_drm_getfile(struct drm_minor minor, unsigned int flags) · 818 unmodified lines
872	877	return file;
873	878	}
874	879	EXPORT_SYMBOL_FOR_TESTS_ONLY(mock_drm_getfile);
	880	+
	881	+#ifdef CONFIG_MMU
	882	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	883	+/*
	884	+ * drm_addr_inflate() attempts to construct an aligned area by inflating
	885	+ * the area size and skipping the unaligned start of the area.
	886	+ * adapted from shmem_get_unmapped_area()
	887	+ */
	888	+static unsigned long drm_addr_inflate(unsigned long addr,
	889	+ unsigned long len,
	890	+ unsigned long pgoff,
	891	+ unsigned long flags,
	892	+ unsigned long huge_size)
	893	+{
	894	+ unsigned long offset, inflated_len;
	895	+ unsigned long inflated_addr;
	896	+ unsigned long inflated_offset;
	897	+
	898	+ offset = (pgoff << PAGE_SHIFT) & (huge_size - 1);
	899	+ if (offset && offset + len < 2 * huge_size)
	900	+ return addr;
	901	+ if ((addr & (huge_size - 1)) == offset)
	902	+ return addr;
	903	+
	904	+ inflated_len = len + huge_size - PAGE_SIZE;
	905	+ if (inflated_len > TASK_SIZE)
	906	+ return addr;
	907	+ if (inflated_len < len)
	908	+ return addr;
	909	+
	910	+ inflated_addr = current->mm->get_unmapped_area(NULL, 0, inflated_len,
	911	+ 0, flags);
	912	+ if (IS_ERR_VALUE(inflated_addr))
	913	+ return addr;
	914	+ if (inflated_addr & ~PAGE_MASK)
	915	+ return addr;
	916	+
	917	+ inflated_offset = inflated_addr & (huge_size - 1);
	918	+ inflated_addr += offset - inflated_offset;
	919	+ if (inflated_offset > offset)
	920	+ inflated_addr += huge_size;
	921	+
	922	+ if (inflated_addr > TASK_SIZE - len)
	923	+ return addr;
	924	+
	925	+ return inflated_addr;
	926	+}
	927	+
	928	+/**
	929	+ * drm_get_unmapped_area() - Get an unused user-space virtual memory area
	930	+ * suitable for huge page table entries.
	931	+ * @file: The struct file representing the address space being mmap()'d.
	932	+ * @uaddr: Start address suggested by user-space.
	933	+ * @len: Length of the area.
	934	+ * @pgoff: The page offset into the address space.
	935	+ * @flags: mmap flags
	936	+ * @mgr: The address space manager used by the drm driver. This argument can
	937	+ * probably be removed at some point when all drivers use the same
	938	+ * address space manager.
	939	+ *
	940	+ * This function attempts to find an unused user-space virtual memory area
	941	+ * that can accommodate the size we want to map, and that is properly
	942	+ * aligned to facilitate huge page table entries matching actual
	943	+ * huge pages or huge page aligned memory in buffer objects. Buffer objects
	944	+ * are assumed to start at huge page boundary pfns (io memory) or be
	945	+ * populated by huge pages aligned to the start of the buffer object
	946	+ * (system- or coherent memory). Adapted from shmem_get_unmapped_area.
	947	+ *
	948	+ * Return: aligned user-space address.
	949	+ */
	950	+unsigned long drm_get_unmapped_area(struct file *file,
	951	+ unsigned long uaddr, unsigned long len,
	952	+ unsigned long pgoff, unsigned long flags,
	953	+ struct drm_vma_offset_manager *mgr)
	954	+{
	955	+ unsigned long addr;
	956	+ unsigned long inflated_addr;
	957	+ struct drm_vma_offset_node *node;
	958	+
	959	+ if (len > TASK_SIZE)
	960	+ return -ENOMEM;
	961	+
	962	+ /*
	963	+ * @pgoff is the file page-offset the huge page boundaries of
	964	+ * which typically aligns to physical address huge page boundaries.
	965	+ * That's not true for DRM, however, where physical address huge
	966	+ * page boundaries instead are aligned with the offset from
	967	+ * buffer object start. So adjust @pgoff to be the offset from
	968	+ * buffer object start.
	969	+ */
	970	+ drm_vma_offset_lock_lookup(mgr);
	971	+ node = drm_vma_offset_lookup_locked(mgr, pgoff, 1);
	972	+ if (node)
	973	+ pgoff -= node->vm_node.start;
	974	+ drm_vma_offset_unlock_lookup(mgr);
	975	+
	976	+ addr = current->mm->get_unmapped_area(file, uaddr, len, pgoff, flags);
	977	+ if (IS_ERR_VALUE(addr))
	978	+ return addr;
	979	+ if (addr & ~PAGE_MASK)
	980	+ return addr;
	981	+ if (addr > TASK_SIZE - len)
	982	+ return addr;
	983	+
	984	+ if (len < HPAGE_PMD_SIZE)
	985	+ return addr;
	986	+ if (flags & MAP_FIXED)
	987	+ return addr;
	988	+ /*
	989	+ * Our priority is to support MAP_SHARED mapped hugely;
	990	+ * and support MAP_PRIVATE mapped hugely too, until it is COWed.
	991	+ * But if caller specified an address hint, respect that as before.
	992	+ */
	993	+ if (uaddr)
	994	+ return addr;
	995	+
	996	+ inflated_addr = drm_addr_inflate(addr, len, pgoff, flags,
	997	+ HPAGE_PMD_SIZE);
	998	+
	999	+ if (IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) &&
	1000	+ len >= HPAGE_PUD_SIZE)
	1001	+ inflated_addr = drm_addr_inflate(inflated_addr, len, pgoff,
	1002	+ flags, HPAGE_PUD_SIZE);
	1003	+ return inflated_addr;
	1004	+}
	1005	+#else /* CONFIG_TRANSPARENT_HUGEPAGE */
	1006	+unsigned long drm_get_unmapped_area(struct file *file,
	1007	+ unsigned long uaddr, unsigned long len,
	1008	+ unsigned long pgoff, unsigned long flags,
	1009	+ struct drm_vma_offset_manager *mgr)
	1010	+{
	1011	+ return current->mm->get_unmapped_area(file, uaddr, len, pgoff, flags);
	1012	+}
	1013	+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	1014	+EXPORT_SYMBOL_GPL(drm_get_unmapped_area);
	1015	+#endif /* CONFIG_MMU */

drivers/gpu/drm/ttm/ttm_bo_vm.c+158−3

@@ -162,6 +162,89 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, · 161 unmodified lines
162	162	}
163	163	EXPORT_SYMBOL(ttm_bo_vm_reserve);
164	164
	165	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	166	+/**
	167	+ * ttm_bo_vm_insert_huge - Insert a pfn for PUD or PMD faults
	168	+ * @vmf: Fault data
	169	+ * @bo: The buffer object
	170	+ * @page_offset: Page offset from bo start
	171	+ * @fault_page_size: The size of the fault in pages.
	172	+ * @pgprot: The page protections.
	173	+ * Does additional checking whether it's possible to insert a PUD or PMD
	174	+ * pfn and performs the insertion.
	175	+ *
	176	+ * Return: VM_FAULT_NOPAGE on successful insertion, VM_FAULT_FALLBACK if
	177	+ * a huge fault was not possible, or on insertion error.
	178	+ */
	179	+static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf,
	180	+ struct ttm_buffer_object *bo,
	181	+ pgoff_t page_offset,
	182	+ pgoff_t fault_page_size,
	183	+ pgprot_t pgprot)
	184	+{
	185	+ pgoff_t i;
	186	+ vm_fault_t ret;
	187	+ unsigned long pfn;
	188	+ pfn_t pfnt;
	189	+ struct ttm_tt *ttm = bo->ttm;
	190	+ bool write = vmf->flags & FAULT_FLAG_WRITE;
	191	+
	192	+ /* Fault should not cross bo boundary. */
	193	+ page_offset &= ~(fault_page_size - 1);
	194	+ if (page_offset + fault_page_size > bo->num_pages)
	195	+ goto out_fallback;
	196	+
	197	+ if (bo->mem.bus.is_iomem)
	198	+ pfn = ttm_bo_io_mem_pfn(bo, page_offset);
	199	+ else
	200	+ pfn = page_to_pfn(ttm->pages[page_offset]);
	201	+
	202	+ /* pfn must be fault_page_size aligned. */
	203	+ if ((pfn & (fault_page_size - 1)) != 0)
	204	+ goto out_fallback;
	205	+
	206	+ /* Check that memory is contiguous. */
	207	+ if (!bo->mem.bus.is_iomem) {
	208	+ for (i = 1; i < fault_page_size; ++i) {
	209	+ if (page_to_pfn(ttm->pages[page_offset + i]) != pfn + i)
	210	+ goto out_fallback;
	211	+ }
	212	+ } else if (bo->bdev->driver->io_mem_pfn) {
	213	+ for (i = 1; i < fault_page_size; ++i) {
	214	+ if (ttm_bo_io_mem_pfn(bo, page_offset + i) != pfn + i)
	215	+ goto out_fallback;
	216	+ }
	217	+ }
	218	+
	219	+ pfnt = __pfn_to_pfn_t(pfn, PFN_DEV);
	220	+ if (fault_page_size == (HPAGE_PMD_SIZE >> PAGE_SHIFT))
	221	+ ret = vmf_insert_pfn_pmd_prot(vmf, pfnt, pgprot, write);
	222	+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
	223	+ else if (fault_page_size == (HPAGE_PUD_SIZE >> PAGE_SHIFT))
	224	+ ret = vmf_insert_pfn_pud_prot(vmf, pfnt, pgprot, write);
	225	+#endif
	226	+ else
	227	+ WARN_ON_ONCE(ret = VM_FAULT_FALLBACK);
	228	+
	229	+ if (ret != VM_FAULT_NOPAGE)
	230	+ goto out_fallback;
	231	+
	232	+ return VM_FAULT_NOPAGE;
	233	+out_fallback:
	234	+ count_vm_event(THP_FAULT_FALLBACK);
	235	+ return VM_FAULT_FALLBACK;
	236	+}
	237	+#else
	238	+static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf,
	239	+ struct ttm_buffer_object *bo,
	240	+ pgoff_t page_offset,
	241	+ pgoff_t fault_page_size,
	242	+ pgprot_t pgprot)
	243	+{
	244	+ return VM_FAULT_FALLBACK;
	245	+}
	246	+#endif
	247	+
165	248	/**
166	249	* ttm_bo_vm_fault_reserved - TTM fault helper
167	250	* @vmf: The struct vm_fault given as argument to the fault callback
@@ -169,6 +252,7 @@ EXPORT_SYMBOL(ttm_bo_vm_reserve); · 1 unmodified line
169	252	* @num_prefault: Maximum number of prefault pages. The caller may want to
170	253	* specify this based on madvice settings and the size of the GPU object
171	254	* backed by the memory.
	255	+ * @fault_page_size: The size of the fault in pages.
172	256	*
173	257	* This function inserts one or more page table entries pointing to the
174	258	* memory backing the buffer object, and then returns a return code
@@ -182,7 +266,8 @@ EXPORT_SYMBOL(ttm_bo_vm_reserve); · 7 unmodified lines
182	266	*/
183	267	vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
184	268	pgprot_t prot,
185		− pgoff_t num_prefault)
	269	+ pgoff_t num_prefault,
	270	+ pgoff_t fault_page_size)
186	271	{
187	272	struct vm_area_struct *vma = vmf->vma;
188	273	struct ttm_buffer_object *bo = vma->vm_private_data;
@@ -274,6 +359,13 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, · 85 unmodified lines
274	359	prot = pgprot_decrypted(prot);
275	360	}
276	361
	362	+ /* We don't prefault on huge faults. Yet. */
	363	+ if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && fault_page_size != 1) {
	364	+ ret = ttm_bo_vm_insert_huge(vmf, bo, page_offset,
	365	+ fault_page_size, prot);
	366	+ goto out_io_unlock;
	367	+ }
	368	+
277	369	/*
278	370	* Speculatively prefault a number of pages. Only error on
279	371	* first page.
@@ -340,7 +432,7 @@ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) · 60 unmodified lines
340	432	return ret;
341	433
342	434	prot = vma->vm_page_prot;
343		− ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT);
	435	+ ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1);
344	436	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
345	437	return ret;
346	438
@@ -350,6 +442,66 @@ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) · 3 unmodified lines
350	442	}
351	443	EXPORT_SYMBOL(ttm_bo_vm_fault);
352	444
	445	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	446	+/**
	447	+ * ttm_pgprot_is_wrprotecting - Is a page protection value write-protecting?
	448	+ * @prot: The page protection value
	449	+ *
	450	+ * Return: true if @prot is write-protecting. false otherwise.
	451	+ */
	452	+static bool ttm_pgprot_is_wrprotecting(pgprot_t prot)
	453	+{
	454	+ /*
	455	+ * This is meant to say "pgprot_wrprotect(prot) == prot" in a generic
	456	+ * way. Unfortunately there is no generic pgprot_wrprotect.
	457	+ */
	458	+ return pte_val(pte_wrprotect(__pte(pgprot_val(prot)))) ==
	459	+ pgprot_val(prot);
	460	+}
	461	+
	462	+static vm_fault_t ttm_bo_vm_huge_fault(struct vm_fault *vmf,
	463	+ enum page_entry_size pe_size)
	464	+{
	465	+ struct vm_area_struct *vma = vmf->vma;
	466	+ pgprot_t prot;
	467	+ struct ttm_buffer_object *bo = vma->vm_private_data;
	468	+ vm_fault_t ret;
	469	+ pgoff_t fault_page_size = 0;
	470	+ bool write = vmf->flags & FAULT_FLAG_WRITE;
	471	+
	472	+ switch (pe_size) {
	473	+ case PE_SIZE_PMD:
	474	+ fault_page_size = HPAGE_PMD_SIZE >> PAGE_SHIFT;
	475	+ break;
	476	+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
	477	+ case PE_SIZE_PUD:
	478	+ fault_page_size = HPAGE_PUD_SIZE >> PAGE_SHIFT;
	479	+ break;
	480	+#endif
	481	+ default:
	482	+ WARN_ON_ONCE(1);
	483	+ return VM_FAULT_FALLBACK;
	484	+ }
	485	+
	486	+ /* Fallback on write dirty-tracking or COW */
	487	+ if (write && ttm_pgprot_is_wrprotecting(vma->vm_page_prot))
	488	+ return VM_FAULT_FALLBACK;
	489	+
	490	+ ret = ttm_bo_vm_reserve(bo, vmf);
	491	+ if (ret)
	492	+ return ret;
	493	+
	494	+ prot = vm_get_page_prot(vma->vm_flags);
	495	+ ret = ttm_bo_vm_fault_reserved(vmf, prot, 1, fault_page_size);
	496	+ if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
	497	+ return ret;
	498	+
	499	+ dma_resv_unlock(bo->base.resv);
	500	+
	501	+ return ret;
	502	+}
	503	+#endif
	504	+
353	505	void ttm_bo_vm_open(struct vm_area_struct *vma)
354	506	{
355	507	struct ttm_buffer_object *bo = vma->vm_private_data;
@@ -451,7 +603,10 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { · 95 unmodified lines
451	603	.fault = ttm_bo_vm_fault,
452	604	.open = ttm_bo_vm_open,
453	605	.close = ttm_bo_vm_close,
454		− .access = ttm_bo_vm_access
	606	+ .access = ttm_bo_vm_access,
	607	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	608	+ .huge_fault = ttm_bo_vm_huge_fault,
	609	+#endif
455	610	};
456	611
457	612	static struct ttm_buffer_object ttm_bo_vm_lookup(struct ttm_bo_device bdev,

drivers/gpu/drm/vmwgfx/Makefile+1−0

@@ -11,4 +11,5 @@ vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \ · 10 unmodified lines
11	11	vmwgfx_validation.o vmwgfx_page_dirty.o vmwgfx_streamoutput.o \
12	12	ttm_object.o ttm_lock.o
13	13
	14	+vmwgfx-$(CONFIG_TRANSPARENT_HUGEPAGE) += vmwgfx_thp.o
14	15	obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o

drivers/gpu/drm/vmwgfx/vmwgfx_drv.c+13−0

@@ -1247,6 +1247,18 @@ static void vmw_remove(struct pci_dev *pdev) · 1246 unmodified lines
1247	1247	pci_disable_device(pdev);
1248	1248	}
1249	1249
	1250	+static unsigned long
	1251	+vmw_get_unmapped_area(struct file *file, unsigned long uaddr,
	1252	+ unsigned long len, unsigned long pgoff,
	1253	+ unsigned long flags)
	1254	+{
	1255	+ struct drm_file *file_priv = file->private_data;
	1256	+ struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);
	1257	+
	1258	+ return drm_get_unmapped_area(file, uaddr, len, pgoff, flags,
	1259	+ &dev_priv->vma_manager);
	1260	+}
	1261	+
1250	1262	static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
1251	1263	void *ptr)
1252	1264	{
@@ -1418,6 +1430,7 @@ static const struct file_operations vmwgfx_driver_fops = { · 165 unmodified lines
1418	1430	.compat_ioctl = vmw_compat_ioctl,
1419	1431	#endif
1420	1432	.llseek = noop_llseek,
	1433	+ .get_unmapped_area = vmw_get_unmapped_area,
1421	1434	};
1422	1435
1423	1436	static struct drm_driver driver = {

drivers/gpu/drm/vmwgfx/vmwgfx_drv.h+12−0

@@ -1000,6 +1000,7 @@ extern int vmw_mmap(struct file filp, struct vm_area_struct vma); · 999 unmodified lines
1000	1000
1001	1001	extern void vmw_validation_mem_init_ttm(struct vmw_private *dev_priv,
1002	1002	size_t gran);
	1003	+
1003	1004	/**
1004	1005	* TTM buffer object driver - vmwgfx_ttm_buffer.c
1005	1006	*/
@@ -1510,6 +1511,17 @@ void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo, · 504 unmodified lines
1510	1511	pgoff_t start, pgoff_t end);
1511	1512	vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf);
1512	1513	vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf);
	1514	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	1515	+vm_fault_t vmw_bo_vm_huge_fault(struct vm_fault *vmf,
	1516	+ enum page_entry_size pe_size);
	1517	+#endif
	1518	+
	1519	+/* Transparent hugepage support - vmwgfx_thp.c */
	1520	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	1521	+extern const struct ttm_mem_type_manager_func vmw_thp_func;
	1522	+#else
	1523	+#define vmw_thp_func ttm_bo_manager_func
	1524	+#endif
1513	1525
1514	1526	/**
1515	1527	* VMW_DEBUG_KMS - Debug output for kernel mode-setting

drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c+74−2

@@ -473,11 +473,11 @@ vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf) · 472 unmodified lines
473	473	* a lot of unnecessary write faults.
474	474	*/
475	475	if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
476		− prot = vma->vm_page_prot;
	476	+ prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
477	477	else
478	478	prot = vm_get_page_prot(vma->vm_flags);
479	479
480		− ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
	480	+ ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault, 1);
481	481	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
482	482	return ret;
483	483
@@ -486,3 +486,75 @@ out_unlock: · 2 unmodified lines
486	486
487	487	return ret;
488	488	}
	489	+
	490	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	491	+vm_fault_t vmw_bo_vm_huge_fault(struct vm_fault *vmf,
	492	+ enum page_entry_size pe_size)
	493	+{
	494	+ struct vm_area_struct *vma = vmf->vma;
	495	+ struct ttm_buffer_object bo = (struct ttm_buffer_object )
	496	+ vma->vm_private_data;
	497	+ struct vmw_buffer_object *vbo =
	498	+ container_of(bo, struct vmw_buffer_object, base);
	499	+ pgprot_t prot;
	500	+ vm_fault_t ret;
	501	+ pgoff_t fault_page_size;
	502	+ bool write = vmf->flags & FAULT_FLAG_WRITE;
	503	+ bool is_cow_mapping =
	504	+ (vma->vm_flags & (VM_SHARED \| VM_MAYWRITE)) == VM_MAYWRITE;
	505	+
	506	+ switch (pe_size) {
	507	+ case PE_SIZE_PMD:
	508	+ fault_page_size = HPAGE_PMD_SIZE >> PAGE_SHIFT;
	509	+ break;
	510	+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
	511	+ case PE_SIZE_PUD:
	512	+ fault_page_size = HPAGE_PUD_SIZE >> PAGE_SHIFT;
	513	+ break;
	514	+#endif
	515	+ default:
	516	+ WARN_ON_ONCE(1);
	517	+ return VM_FAULT_FALLBACK;
	518	+ }
	519	+
	520	+ /* Always do write dirty-tracking and COW on PTE level. */
	521	+ if (write && (READ_ONCE(vbo->dirty) \|\| is_cow_mapping))
	522	+ return VM_FAULT_FALLBACK;
	523	+
	524	+ ret = ttm_bo_vm_reserve(bo, vmf);
	525	+ if (ret)
	526	+ return ret;
	527	+
	528	+ if (vbo->dirty) {
	529	+ pgoff_t allowed_prefault;
	530	+ unsigned long page_offset;
	531	+
	532	+ page_offset = vmf->pgoff -
	533	+ drm_vma_node_start(&bo->base.vma_node);
	534	+ if (page_offset >= bo->num_pages \|\|
	535	+ vmw_resources_clean(vbo, page_offset,
	536	+ page_offset + PAGE_SIZE,
	537	+ &allowed_prefault)) {
	538	+ ret = VM_FAULT_SIGBUS;
	539	+ goto out_unlock;
	540	+ }
	541	+
	542	+ /*
	543	+ * Write protect, so we get a new fault on write, and can
	544	+ * split.
	545	+ */
	546	+ prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
	547	+ } else {
	548	+ prot = vm_get_page_prot(vma->vm_flags);
	549	+ }
	550	+
	551	+ ret = ttm_bo_vm_fault_reserved(vmf, prot, 1, fault_page_size);
	552	+ if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
	553	+ return ret;
	554	+
	555	+out_unlock:
	556	+ dma_resv_unlock(bo->base.resv);
	557	+
	558	+ return ret;
	559	+}
	560	+#endif

drivers/gpu/drm/vmwgfx/vmwgfx_thp.c+166−0

@@ -0,0 +1,166 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR MIT
	2	+/*
	3	+ * Huge page-table-entry support for IO memory.
	4	+ *
	5	+ * Copyright (C) 2007-2019 Vmware, Inc. All rights reservedd.
	6	+ */
	7	+#include "vmwgfx_drv.h"
	8	+#include <drm/ttm/ttm_module.h>
	9	+#include <drm/ttm/ttm_bo_driver.h>
	10	+#include <drm/ttm/ttm_placement.h>
	11	+
	12	+/**
	13	+ * struct vmw_thp_manager - Range manager implementing huge page alignment
	14	+ *
	15	+ * @mm: The underlying range manager. Protected by @lock.
	16	+ * @lock: Manager lock.
	17	+ */
	18	+struct vmw_thp_manager {
	19	+ struct drm_mm mm;
	20	+ spinlock_t lock;
	21	+};
	22	+
	23	+static int vmw_thp_insert_aligned(struct drm_mm mm, struct drm_mm_node node,
	24	+ unsigned long align_pages,
	25	+ const struct ttm_place *place,
	26	+ struct ttm_mem_reg *mem,
	27	+ unsigned long lpfn,
	28	+ enum drm_mm_insert_mode mode)
	29	+{
	30	+ if (align_pages >= mem->page_alignment &&
	31	+ (!mem->page_alignment \|\| align_pages % mem->page_alignment == 0)) {
	32	+ return drm_mm_insert_node_in_range(mm, node,
	33	+ mem->num_pages,
	34	+ align_pages, 0,
	35	+ place->fpfn, lpfn, mode);
	36	+ }
	37	+
	38	+ return -ENOSPC;
	39	+}
	40	+
	41	+static int vmw_thp_get_node(struct ttm_mem_type_manager *man,
	42	+ struct ttm_buffer_object *bo,
	43	+ const struct ttm_place *place,
	44	+ struct ttm_mem_reg *mem)
	45	+{
	46	+ struct vmw_thp_manager rman = (struct vmw_thp_manager ) man->priv;
	47	+ struct drm_mm *mm = &rman->mm;
	48	+ struct drm_mm_node *node;
	49	+ unsigned long align_pages;
	50	+ unsigned long lpfn;
	51	+ enum drm_mm_insert_mode mode = DRM_MM_INSERT_BEST;
	52	+ int ret;
	53	+
	54	+ node = kzalloc(sizeof(*node), GFP_KERNEL);
	55	+ if (!node)
	56	+ return -ENOMEM;
	57	+
	58	+ lpfn = place->lpfn;
	59	+ if (!lpfn)
	60	+ lpfn = man->size;
	61	+
	62	+ mode = DRM_MM_INSERT_BEST;
	63	+ if (place->flags & TTM_PL_FLAG_TOPDOWN)
	64	+ mode = DRM_MM_INSERT_HIGH;
	65	+
	66	+ spin_lock(&rman->lock);
	67	+ if (IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) {
	68	+ align_pages = (HPAGE_PUD_SIZE >> PAGE_SHIFT);
	69	+ if (mem->num_pages >= align_pages) {
	70	+ ret = vmw_thp_insert_aligned(mm, node, align_pages,
	71	+ place, mem, lpfn, mode);
	72	+ if (!ret)
	73	+ goto found_unlock;
	74	+ }
	75	+ }
	76	+
	77	+ align_pages = (HPAGE_PMD_SIZE >> PAGE_SHIFT);
	78	+ if (mem->num_pages >= align_pages) {
	79	+ ret = vmw_thp_insert_aligned(mm, node, align_pages, place, mem,
	80	+ lpfn, mode);
	81	+ if (!ret)
	82	+ goto found_unlock;
	83	+ }
	84	+
	85	+ ret = drm_mm_insert_node_in_range(mm, node, mem->num_pages,
	86	+ mem->page_alignment, 0,
	87	+ place->fpfn, lpfn, mode);
	88	+found_unlock:
	89	+ spin_unlock(&rman->lock);
	90	+
	91	+ if (unlikely(ret)) {
	92	+ kfree(node);
	93	+ } else {
	94	+ mem->mm_node = node;
	95	+ mem->start = node->start;
	96	+ }
	97	+
	98	+ return 0;
	99	+}
	100	+
	101	+
	102	+
	103	+static void vmw_thp_put_node(struct ttm_mem_type_manager *man,
	104	+ struct ttm_mem_reg *mem)
	105	+{
	106	+ struct vmw_thp_manager rman = (struct vmw_thp_manager ) man->priv;
	107	+
	108	+ if (mem->mm_node) {
	109	+ spin_lock(&rman->lock);
	110	+ drm_mm_remove_node(mem->mm_node);
	111	+ spin_unlock(&rman->lock);
	112	+
	113	+ kfree(mem->mm_node);
	114	+ mem->mm_node = NULL;
	115	+ }
	116	+}
	117	+
	118	+static int vmw_thp_init(struct ttm_mem_type_manager *man,
	119	+ unsigned long p_size)
	120	+{
	121	+ struct vmw_thp_manager *rman;
	122	+
	123	+ rman = kzalloc(sizeof(*rman), GFP_KERNEL);
	124	+ if (!rman)
	125	+ return -ENOMEM;
	126	+
	127	+ drm_mm_init(&rman->mm, 0, p_size);
	128	+ spin_lock_init(&rman->lock);
	129	+ man->priv = rman;
	130	+ return 0;
	131	+}
	132	+
	133	+static int vmw_thp_takedown(struct ttm_mem_type_manager *man)
	134	+{
	135	+ struct vmw_thp_manager rman = (struct vmw_thp_manager ) man->priv;
	136	+ struct drm_mm *mm = &rman->mm;
	137	+
	138	+ spin_lock(&rman->lock);
	139	+ if (drm_mm_clean(mm)) {
	140	+ drm_mm_takedown(mm);
	141	+ spin_unlock(&rman->lock);
	142	+ kfree(rman);
	143	+ man->priv = NULL;
	144	+ return 0;
	145	+ }
	146	+ spin_unlock(&rman->lock);
	147	+ return -EBUSY;
	148	+}
	149	+
	150	+static void vmw_thp_debug(struct ttm_mem_type_manager *man,
	151	+ struct drm_printer *printer)
	152	+{
	153	+ struct vmw_thp_manager rman = (struct vmw_thp_manager ) man->priv;
	154	+
	155	+ spin_lock(&rman->lock);
	156	+ drm_mm_print(&rman->mm, printer);
	157	+ spin_unlock(&rman->lock);
	158	+}
	159	+
	160	+const struct ttm_mem_type_manager_func vmw_thp_func = {
	161	+ .init = vmw_thp_init,
	162	+ .takedown = vmw_thp_takedown,
	163	+ .get_node = vmw_thp_get_node,
	164	+ .put_node = vmw_thp_put_node,
	165	+ .debug = vmw_thp_debug
	166	+};

drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c+1−1

@@ -749,7 +749,7 @@ static int vmw_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, · 748 unmodified lines
749	749	break;
750	750	case TTM_PL_VRAM:
751	751	/* "On-card" video ram */
752		− man->func = &ttm_bo_manager_func;
	752	+ man->func = &vmw_thp_func;
753	753	man->gpu_offset = 0;
754	754	man->flags = TTM_MEMTYPE_FLAG_FIXED \| TTM_MEMTYPE_FLAG_MAPPABLE;
755	755	man->available_caching = TTM_PL_FLAG_CACHED;

drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c+4−1

@@ -34,7 +34,10 @@ int vmw_mmap(struct file filp, struct vm_area_struct vma) · 33 unmodified lines
34	34	.page_mkwrite = vmw_bo_vm_mkwrite,
35	35	.fault = vmw_bo_vm_fault,
36	36	.open = ttm_bo_vm_open,
37		− .close = ttm_bo_vm_close
	37	+ .close = ttm_bo_vm_close,
	38	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	39	+ .huge_fault = vmw_bo_vm_huge_fault,
	40	+#endif
38	41	};
39	42	struct drm_file *file_priv = filp->private_data;
40	43	struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);

include/drm/drm_file.h+9−0

@@ -391,4 +391,13 @@ void drm_send_event(struct drm_device dev, struct drm_pending_event e); · 390 unmodified lines
391	391
392	392	struct file mock_drm_getfile(struct drm_minor minor, unsigned int flags);
393	393
	394	+#ifdef CONFIG_MMU
	395	+struct drm_vma_offset_manager;
	396	+unsigned long drm_get_unmapped_area(struct file *file,
	397	+ unsigned long uaddr, unsigned long len,
	398	+ unsigned long pgoff, unsigned long flags,
	399	+ struct drm_vma_offset_manager *mgr);
	400	+#endif /* CONFIG_MMU */
	401	+
	402	+
394	403	#endif /* _DRM_FILE_H_ */

include/drm/ttm/ttm_bo_api.h+2−1

@@ -727,7 +727,8 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, · 726 unmodified lines
727	727
728	728	vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
729	729	pgprot_t prot,
730		− pgoff_t num_prefault);
	730	+ pgoff_t num_prefault,
	731	+ pgoff_t fault_page_size);
731	732
732	733	vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf);
733	734

include/linux/fs.h+1−1

@@ -3399,7 +3399,7 @@ static inline bool io_is_direct(struct file *filp) · 3398 unmodified lines
3399	3399	return (filp->f_flags & O_DIRECT) \|\| IS_DAX(filp->f_mapping->host);
3400	3400	}
3401	3401
3402		−static inline bool vma_is_dax(struct vm_area_struct *vma)
	3402	+static inline bool vma_is_dax(const struct vm_area_struct *vma)
3403	3403	{
3404	3404	return vma->vm_file && IS_DAX(vma->vm_file->f_mapping->host);
3405	3405	}

include/linux/huge_mm.h+39−2

@@ -47,8 +47,45 @@ extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, · 46 unmodified lines
47	47	extern int change_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
48	48	unsigned long addr, pgprot_t newprot,
49	49	int prot_numa);
50		−vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write);
51		−vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write);
	50	+vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,
	51	+ pgprot_t pgprot, bool write);
	52	+
	53	+/**
	54	+ * vmf_insert_pfn_pmd - insert a pmd size pfn
	55	+ * @vmf: Structure describing the fault
	56	+ * @pfn: pfn to insert
	57	+ * @pgprot: page protection to use
	58	+ * @write: whether it's a write fault
	59	+ *
	60	+ * Insert a pmd size pfn. See vmf_insert_pfn() for additional info.
	61	+ *
	62	+ * Return: vm_fault_t value.
	63	+ */
	64	+static inline vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn,
	65	+ bool write)
	66	+{
	67	+ return vmf_insert_pfn_pmd_prot(vmf, pfn, vmf->vma->vm_page_prot, write);
	68	+}
	69	+vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,
	70	+ pgprot_t pgprot, bool write);
	71	+
	72	+/**
	73	+ * vmf_insert_pfn_pud - insert a pud size pfn
	74	+ * @vmf: Structure describing the fault
	75	+ * @pfn: pfn to insert
	76	+ * @pgprot: page protection to use
	77	+ * @write: whether it's a write fault
	78	+ *
	79	+ * Insert a pud size pfn. See vmf_insert_pfn() for additional info.
	80	+ *
	81	+ * Return: vm_fault_t value.
	82	+ */
	83	+static inline vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn,
	84	+ bool write)
	85	+{
	86	+ return vmf_insert_pfn_pud_prot(vmf, pfn, vmf->vma->vm_page_prot, write);
	87	+}
	88	+
52	89	enum transparent_hugepage_flag {
53	90	TRANSPARENT_HUGEPAGE_FLAG,
54	91	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,

include/linux/mm.h+17−0

@@ -3014,6 +3014,23 @@ extern long copy_huge_page_from_user(struct page *dst_page, · 3013 unmodified lines
3014	3014	const void __user *usr_src,
3015	3015	unsigned int pages_per_huge_page,
3016	3016	bool allow_pagefault);
	3017	+
	3018	+/**
	3019	+ * vma_is_special_huge - Are transhuge page-table entries considered special?
	3020	+ * @vma: Pointer to the struct vm_area_struct to consider
	3021	+ *
	3022	+ * Whether transhuge page-table entries are considered "special" following
	3023	+ * the definition in vm_normal_page().
	3024	+ *
	3025	+ * Return: true if transhuge page-table entries should be considered special,
	3026	+ * false otherwise.
	3027	+ */
	3028	+static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
	3029	+{
	3030	+ return vma_is_dax(vma) \|\| (vma->vm_file &&
	3031	+ (vma->vm_flags & (VM_PFNMAP \| VM_MIXEDMAP)));
	3032	+}
	3033	+
3017	3034	#endif /* CONFIG_TRANSPARENT_HUGEPAGE \|\| CONFIG_HUGETLBFS */
3018	3035
3019	3036	#ifdef CONFIG_DEBUG_PAGEALLOC

mm/huge_memory.c+35−9

@@ -824,11 +824,24 @@ out_unlock: · 823 unmodified lines
824	824	pte_free(mm, pgtable);
825	825	}
826	826
827		−vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write)
	827	+/**
	828	+ * vmf_insert_pfn_pmd_prot - insert a pmd size pfn
	829	+ * @vmf: Structure describing the fault
	830	+ * @pfn: pfn to insert
	831	+ * @pgprot: page protection to use
	832	+ * @write: whether it's a write fault
	833	+ *
	834	+ * Insert a pmd size pfn. See vmf_insert_pfn() for additional info and
	835	+ * also consult the vmf_insert_mixed_prot() documentation when
	836	+ * @pgprot != @vmf->vma->vm_page_prot.
	837	+ *
	838	+ * Return: vm_fault_t value.
	839	+ */
	840	+vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,
	841	+ pgprot_t pgprot, bool write)
828	842	{
829	843	unsigned long addr = vmf->address & PMD_MASK;
830	844	struct vm_area_struct *vma = vmf->vma;
831		− pgprot_t pgprot = vma->vm_page_prot;
832	845	pgtable_t pgtable = NULL;
833	846
834	847	/*
@@ -856,7 +869,7 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) · 21 unmodified lines
856	869	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);
857	870	return VM_FAULT_NOPAGE;
858	871	}
859		−EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
	872	+EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd_prot);
860	873
861	874	#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
862	875	static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
@@ -902,11 +915,24 @@ out_unlock: · 39 unmodified lines
902	915	spin_unlock(ptl);
903	916	}
904	917
905		−vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write)
	918	+/**
	919	+ * vmf_insert_pfn_pud_prot - insert a pud size pfn
	920	+ * @vmf: Structure describing the fault
	921	+ * @pfn: pfn to insert
	922	+ * @pgprot: page protection to use
	923	+ * @write: whether it's a write fault
	924	+ *
	925	+ * Insert a pud size pfn. See vmf_insert_pfn() for additional info and
	926	+ * also consult the vmf_insert_mixed_prot() documentation when
	927	+ * @pgprot != @vmf->vma->vm_page_prot.
	928	+ *
	929	+ * Return: vm_fault_t value.
	930	+ */
	931	+vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,
	932	+ pgprot_t pgprot, bool write)
906	933	{
907	934	unsigned long addr = vmf->address & PUD_MASK;
908	935	struct vm_area_struct *vma = vmf->vma;
909		− pgprot_t pgprot = vma->vm_page_prot;
910	936
911	937	/*
912	938	* If we had pud_special, we could avoid all these restrictions,
@@ -927,7 +953,7 @@ vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write) · 14 unmodified lines
927	953	insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);
928	954	return VM_FAULT_NOPAGE;
929	955	}
930		−EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
	956	+EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud_prot);
931	957	#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
932	958
933	959	static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
@@ -1819,7 +1845,7 @@ int zap_huge_pmd(struct mmu_gather tlb, struct vm_area_struct vma, · 885 unmodified lines
1819	1845	orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1820	1846	tlb->fullmm);
1821	1847	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1822		− if (vma_is_dax(vma)) {
	1848	+ if (vma_is_special_huge(vma)) {
1823	1849	if (arch_needs_pgtable_deposit())
1824	1850	zap_deposited_table(tlb->mm, pmd);
1825	1851	spin_unlock(ptl);
@@ -2083,7 +2109,7 @@ int zap_huge_pud(struct mmu_gather tlb, struct vm_area_struct vma, · 257 unmodified lines
2083	2109	*/
2084	2110	pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm);
2085	2111	tlb_remove_pud_tlb_entry(tlb, pud, addr);
2086		− if (vma_is_dax(vma)) {
	2112	+ if (vma_is_special_huge(vma)) {
2087	2113	spin_unlock(ptl);
2088	2114	/* No zero page support yet */
2089	2115	} else {
@@ -2192,7 +2218,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct vma, pmd_t pmd, · 102 unmodified lines
2192	2218	*/
2193	2219	if (arch_needs_pgtable_deposit())
2194	2220	zap_deposited_table(mm, pmd);
2195		− if (vma_is_dax(vma))
	2221	+ if (vma_is_special_huge(vma))
2196	2222	return;
2197	2223	page = pmd_page(_pmd);
2198	2224	if (!PageDirty(page) && pmd_dirty(_pmd))

mm/memory.c+19−8

@@ -3951,11 +3951,14 @@ static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd) · 3950 unmodified lines
3951	3951	{
3952	3952	if (vma_is_anonymous(vmf->vma))
3953	3953	return do_huge_pmd_wp_page(vmf, orig_pmd);
3954		− if (vmf->vma->vm_ops->huge_fault)
3955		− return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
	3954	+ if (vmf->vma->vm_ops->huge_fault) {
	3955	+ vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
3956	3956
3957		− /* COW handled on pte level: split pmd */
3958		− VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);
	3957	+ if (!(ret & VM_FAULT_FALLBACK))
	3958	+ return ret;
	3959	+ }
	3960	+
	3961	+ /* COW or write-notify handled on pte level: split pmd. */
3959	3962	__split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
3960	3963
3961	3964	return VM_FAULT_FALLBACK;
@@ -3968,12 +3971,20 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma) · 6 unmodified lines
3968	3971
3969	3972	static vm_fault_t create_huge_pud(struct vm_fault *vmf)
3970	3973	{
3971		−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	3974	+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
	3975	+ defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
3972	3976	/* No support for anonymous transparent PUD pages yet */
3973	3977	if (vma_is_anonymous(vmf->vma))
3974		− return VM_FAULT_FALLBACK;
3975		− if (vmf->vma->vm_ops->huge_fault)
3976		− return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
	3978	+ goto split;
	3979	+ if (vmf->vma->vm_ops->huge_fault) {
	3980	+ vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
	3981	+
	3982	+ if (!(ret & VM_FAULT_FALLBACK))
	3983	+ return ret;
	3984	+ }
	3985	+split:
	3986	+ /* COW or write-notify not handled on PUD level: split pud.*/
	3987	+ __split_huge_pud(vmf->vma, vmf->pud, vmf->address);
3977	3988	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3978	3989	return VM_FAULT_FALLBACK;
3979	3990	}