History

Merge tag 'drm-next-2020-04-03-1' of git://anongit.freedesktop.org/drm/drm

Pull drm hugepage support from Dave Airlie: "This adds support for hugepages to TTM and has been tested with the vmwgfx drivers, though I expect other drivers to start using it" * tag 'drm-next-2020-04-03-1' of git://anongit.freedesktop.org/drm/drm: drm/vmwgfx: Hook up the helpers to align buffer objects drm/vmwgfx: Introduce a huge page aligning TTM range manager drm: Add a drm_get_unmapped_area() helper drm/vmwgfx: Support huge page faults drm/ttm, drm/vmwgfx: Support huge TTM pagefaults mm: Add vmf_insert_pfn_xxx_prot() for huge page-table entries mm: Split huge pages on write-notify or COW mm: Introduce vma_is_special_huge fs: Constify vma argument to vma_is_dax

Linus Torvalds committed 6 years ago

Commit ea9448b2 parents 83eb69f 0e7e61916 files · +692 −28Browse files

Unified · Split

16 files
  1. drivers/gpu/drm/drm_file.c +141 −0
  2. drivers/gpu/drm/ttm/ttm_bo_vm.c +158 −3
  3. drivers/gpu/drm/vmwgfx/Makefile +1 −0
  4. drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +13 −0
  5. drivers/gpu/drm/vmwgfx/vmwgfx_drv.h +12 −0
  6. drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +74 −2
  7. drivers/gpu/drm/vmwgfx/vmwgfx_thp.c +166 −0
  8. drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c +1 −1
  9. drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c +4 −1
  10. include/drm/drm_file.h +9 −0
  11. include/drm/ttm/ttm_bo_api.h +2 −1
  12. include/linux/fs.h +1 −1
  13. include/linux/huge_mm.h +39 −2
  14. include/linux/mm.h +17 −0
  15. mm/huge_memory.c +35 −9
  16. mm/memory.c +19 −8
drivers/gpu/drm/drm_file.c+141−0
@@ -48,6 +48,11 @@ · 47 unmodified lines
48#include "drm_internal.h"48#include "drm_internal.h"
49#include "drm_legacy.h"49#include "drm_legacy.h"
50 50
51#if defined(CONFIG_MMU) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
52#include <uapi/asm/mman.h>
53#include <drm/drm_vma_manager.h>
54#endif
55
51/* from BKL pushdown */56/* from BKL pushdown */
52DEFINE_MUTEX(drm_global_mutex);57DEFINE_MUTEX(drm_global_mutex);
53 58
@@ -872,3 +877,139 @@ struct file *mock_drm_getfile(struct drm_minor *minor, unsigned int flags) · 818 unmodified lines
872 return file;877 return file;
873}878}
874EXPORT_SYMBOL_FOR_TESTS_ONLY(mock_drm_getfile);879EXPORT_SYMBOL_FOR_TESTS_ONLY(mock_drm_getfile);
880
881#ifdef CONFIG_MMU
882#ifdef CONFIG_TRANSPARENT_HUGEPAGE
883/*
884 * drm_addr_inflate() attempts to construct an aligned area by inflating
885 * the area size and skipping the unaligned start of the area.
886 * adapted from shmem_get_unmapped_area()
887 */
888static unsigned long drm_addr_inflate(unsigned long addr,
889 unsigned long len,
890 unsigned long pgoff,
891 unsigned long flags,
892 unsigned long huge_size)
893{
894 unsigned long offset, inflated_len;
895 unsigned long inflated_addr;
896 unsigned long inflated_offset;
897
898 offset = (pgoff << PAGE_SHIFT) & (huge_size - 1);
899 if (offset && offset + len < 2 * huge_size)
900 return addr;
901 if ((addr & (huge_size - 1)) == offset)
902 return addr;
903
904 inflated_len = len + huge_size - PAGE_SIZE;
905 if (inflated_len > TASK_SIZE)
906 return addr;
907 if (inflated_len < len)
908 return addr;
909
910 inflated_addr = current->mm->get_unmapped_area(NULL, 0, inflated_len,
911 0, flags);
912 if (IS_ERR_VALUE(inflated_addr))
913 return addr;
914 if (inflated_addr & ~PAGE_MASK)
915 return addr;
916
917 inflated_offset = inflated_addr & (huge_size - 1);
918 inflated_addr += offset - inflated_offset;
919 if (inflated_offset > offset)
920 inflated_addr += huge_size;
921
922 if (inflated_addr > TASK_SIZE - len)
923 return addr;
924
925 return inflated_addr;
926}
927
928/**
929 * drm_get_unmapped_area() - Get an unused user-space virtual memory area
930 * suitable for huge page table entries.
931 * @file: The struct file representing the address space being mmap()'d.
932 * @uaddr: Start address suggested by user-space.
933 * @len: Length of the area.
934 * @pgoff: The page offset into the address space.
935 * @flags: mmap flags
936 * @mgr: The address space manager used by the drm driver. This argument can
937 * probably be removed at some point when all drivers use the same
938 * address space manager.
939 *
940 * This function attempts to find an unused user-space virtual memory area
941 * that can accommodate the size we want to map, and that is properly
942 * aligned to facilitate huge page table entries matching actual
943 * huge pages or huge page aligned memory in buffer objects. Buffer objects
944 * are assumed to start at huge page boundary pfns (io memory) or be
945 * populated by huge pages aligned to the start of the buffer object
946 * (system- or coherent memory). Adapted from shmem_get_unmapped_area.
947 *
948 * Return: aligned user-space address.
949 */
950unsigned long drm_get_unmapped_area(struct file *file,
951 unsigned long uaddr, unsigned long len,
952 unsigned long pgoff, unsigned long flags,
953 struct drm_vma_offset_manager *mgr)
954{
955 unsigned long addr;
956 unsigned long inflated_addr;
957 struct drm_vma_offset_node *node;
958
959 if (len > TASK_SIZE)
960 return -ENOMEM;
961
962 /*
963 * @pgoff is the file page-offset the huge page boundaries of
964 * which typically aligns to physical address huge page boundaries.
965 * That's not true for DRM, however, where physical address huge
966 * page boundaries instead are aligned with the offset from
967 * buffer object start. So adjust @pgoff to be the offset from
968 * buffer object start.
969 */
970 drm_vma_offset_lock_lookup(mgr);
971 node = drm_vma_offset_lookup_locked(mgr, pgoff, 1);
972 if (node)
973 pgoff -= node->vm_node.start;
974 drm_vma_offset_unlock_lookup(mgr);
975
976 addr = current->mm->get_unmapped_area(file, uaddr, len, pgoff, flags);
977 if (IS_ERR_VALUE(addr))
978 return addr;
979 if (addr & ~PAGE_MASK)
980 return addr;
981 if (addr > TASK_SIZE - len)
982 return addr;
983
984 if (len < HPAGE_PMD_SIZE)
985 return addr;
986 if (flags & MAP_FIXED)
987 return addr;
988 /*
989 * Our priority is to support MAP_SHARED mapped hugely;
990 * and support MAP_PRIVATE mapped hugely too, until it is COWed.
991 * But if caller specified an address hint, respect that as before.
992 */
993 if (uaddr)
994 return addr;
995
996 inflated_addr = drm_addr_inflate(addr, len, pgoff, flags,
997 HPAGE_PMD_SIZE);
998
999 if (IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) &&
1000 len >= HPAGE_PUD_SIZE)
1001 inflated_addr = drm_addr_inflate(inflated_addr, len, pgoff,
1002 flags, HPAGE_PUD_SIZE);
1003 return inflated_addr;
1004}
1005#else /* CONFIG_TRANSPARENT_HUGEPAGE */
1006unsigned long drm_get_unmapped_area(struct file *file,
1007 unsigned long uaddr, unsigned long len,
1008 unsigned long pgoff, unsigned long flags,
1009 struct drm_vma_offset_manager *mgr)
1010{
1011 return current->mm->get_unmapped_area(file, uaddr, len, pgoff, flags);
1012}
1013#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1014EXPORT_SYMBOL_GPL(drm_get_unmapped_area);
1015#endif /* CONFIG_MMU */
drivers/gpu/drm/ttm/ttm_bo_vm.c+158−3
@@ -162,6 +162,89 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, · 161 unmodified lines
162}162}
163EXPORT_SYMBOL(ttm_bo_vm_reserve);163EXPORT_SYMBOL(ttm_bo_vm_reserve);
164 164
165#ifdef CONFIG_TRANSPARENT_HUGEPAGE
166/**
167 * ttm_bo_vm_insert_huge - Insert a pfn for PUD or PMD faults
168 * @vmf: Fault data
169 * @bo: The buffer object
170 * @page_offset: Page offset from bo start
171 * @fault_page_size: The size of the fault in pages.
172 * @pgprot: The page protections.
173 * Does additional checking whether it's possible to insert a PUD or PMD
174 * pfn and performs the insertion.
175 *
176 * Return: VM_FAULT_NOPAGE on successful insertion, VM_FAULT_FALLBACK if
177 * a huge fault was not possible, or on insertion error.
178 */
179static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf,
180 struct ttm_buffer_object *bo,
181 pgoff_t page_offset,
182 pgoff_t fault_page_size,
183 pgprot_t pgprot)
184{
185 pgoff_t i;
186 vm_fault_t ret;
187 unsigned long pfn;
188 pfn_t pfnt;
189 struct ttm_tt *ttm = bo->ttm;
190 bool write = vmf->flags & FAULT_FLAG_WRITE;
191
192 /* Fault should not cross bo boundary. */
193 page_offset &= ~(fault_page_size - 1);
194 if (page_offset + fault_page_size > bo->num_pages)
195 goto out_fallback;
196
197 if (bo->mem.bus.is_iomem)
198 pfn = ttm_bo_io_mem_pfn(bo, page_offset);
199 else
200 pfn = page_to_pfn(ttm->pages[page_offset]);
201
202 /* pfn must be fault_page_size aligned. */
203 if ((pfn & (fault_page_size - 1)) != 0)
204 goto out_fallback;
205
206 /* Check that memory is contiguous. */
207 if (!bo->mem.bus.is_iomem) {
208 for (i = 1; i < fault_page_size; ++i) {
209 if (page_to_pfn(ttm->pages[page_offset + i]) != pfn + i)
210 goto out_fallback;
211 }
212 } else if (bo->bdev->driver->io_mem_pfn) {
213 for (i = 1; i < fault_page_size; ++i) {
214 if (ttm_bo_io_mem_pfn(bo, page_offset + i) != pfn + i)
215 goto out_fallback;
216 }
217 }
218
219 pfnt = __pfn_to_pfn_t(pfn, PFN_DEV);
220 if (fault_page_size == (HPAGE_PMD_SIZE >> PAGE_SHIFT))
221 ret = vmf_insert_pfn_pmd_prot(vmf, pfnt, pgprot, write);
222#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
223 else if (fault_page_size == (HPAGE_PUD_SIZE >> PAGE_SHIFT))
224 ret = vmf_insert_pfn_pud_prot(vmf, pfnt, pgprot, write);
225#endif
226 else
227 WARN_ON_ONCE(ret = VM_FAULT_FALLBACK);
228
229 if (ret != VM_FAULT_NOPAGE)
230 goto out_fallback;
231
232 return VM_FAULT_NOPAGE;
233out_fallback:
234 count_vm_event(THP_FAULT_FALLBACK);
235 return VM_FAULT_FALLBACK;
236}
237#else
238static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf,
239 struct ttm_buffer_object *bo,
240 pgoff_t page_offset,
241 pgoff_t fault_page_size,
242 pgprot_t pgprot)
243{
244 return VM_FAULT_FALLBACK;
245}
246#endif
247
165/**248/**
166 * ttm_bo_vm_fault_reserved - TTM fault helper249 * ttm_bo_vm_fault_reserved - TTM fault helper
167 * @vmf: The struct vm_fault given as argument to the fault callback250 * @vmf: The struct vm_fault given as argument to the fault callback
@@ -169,6 +252,7 @@ EXPORT_SYMBOL(ttm_bo_vm_reserve); · 1 unmodified line
169 * @num_prefault: Maximum number of prefault pages. The caller may want to252 * @num_prefault: Maximum number of prefault pages. The caller may want to
170 * specify this based on madvice settings and the size of the GPU object253 * specify this based on madvice settings and the size of the GPU object
171 * backed by the memory.254 * backed by the memory.
255 * @fault_page_size: The size of the fault in pages.
172 *256 *
173 * This function inserts one or more page table entries pointing to the257 * This function inserts one or more page table entries pointing to the
174 * memory backing the buffer object, and then returns a return code258 * memory backing the buffer object, and then returns a return code
@@ -182,7 +266,8 @@ EXPORT_SYMBOL(ttm_bo_vm_reserve); · 7 unmodified lines
182 */266 */
183vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,267vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
184 pgprot_t prot,268 pgprot_t prot,
185 pgoff_t num_prefault)269 pgoff_t num_prefault,
270 pgoff_t fault_page_size)
186{271{
187 struct vm_area_struct *vma = vmf->vma;272 struct vm_area_struct *vma = vmf->vma;
188 struct ttm_buffer_object *bo = vma->vm_private_data;273 struct ttm_buffer_object *bo = vma->vm_private_data;
@@ -274,6 +359,13 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, · 85 unmodified lines
274 prot = pgprot_decrypted(prot);359 prot = pgprot_decrypted(prot);
275 }360 }
276 361
362 /* We don't prefault on huge faults. Yet. */
363 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && fault_page_size != 1) {
364 ret = ttm_bo_vm_insert_huge(vmf, bo, page_offset,
365 fault_page_size, prot);
366 goto out_io_unlock;
367 }
368
277 /*369 /*
278 * Speculatively prefault a number of pages. Only error on370 * Speculatively prefault a number of pages. Only error on
279 * first page.371 * first page.
@@ -340,7 +432,7 @@ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) · 60 unmodified lines
340 return ret;432 return ret;
341 433
342 prot = vma->vm_page_prot;434 prot = vma->vm_page_prot;
343 ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT);435 ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1);
344 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))436 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
345 return ret;437 return ret;
346 438
@@ -350,6 +442,66 @@ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) · 3 unmodified lines
350}442}
351EXPORT_SYMBOL(ttm_bo_vm_fault);443EXPORT_SYMBOL(ttm_bo_vm_fault);
352 444
445#ifdef CONFIG_TRANSPARENT_HUGEPAGE
446/**
447 * ttm_pgprot_is_wrprotecting - Is a page protection value write-protecting?
448 * @prot: The page protection value
449 *
450 * Return: true if @prot is write-protecting. false otherwise.
451 */
452static bool ttm_pgprot_is_wrprotecting(pgprot_t prot)
453{
454 /*
455 * This is meant to say "pgprot_wrprotect(prot) == prot" in a generic
456 * way. Unfortunately there is no generic pgprot_wrprotect.
457 */
458 return pte_val(pte_wrprotect(__pte(pgprot_val(prot)))) ==
459 pgprot_val(prot);
460}
461
462static vm_fault_t ttm_bo_vm_huge_fault(struct vm_fault *vmf,
463 enum page_entry_size pe_size)
464{
465 struct vm_area_struct *vma = vmf->vma;
466 pgprot_t prot;
467 struct ttm_buffer_object *bo = vma->vm_private_data;
468 vm_fault_t ret;
469 pgoff_t fault_page_size = 0;
470 bool write = vmf->flags & FAULT_FLAG_WRITE;
471
472 switch (pe_size) {
473 case PE_SIZE_PMD:
474 fault_page_size = HPAGE_PMD_SIZE >> PAGE_SHIFT;
475 break;
476#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
477 case PE_SIZE_PUD:
478 fault_page_size = HPAGE_PUD_SIZE >> PAGE_SHIFT;
479 break;
480#endif
481 default:
482 WARN_ON_ONCE(1);
483 return VM_FAULT_FALLBACK;
484 }
485
486 /* Fallback on write dirty-tracking or COW */
487 if (write && ttm_pgprot_is_wrprotecting(vma->vm_page_prot))
488 return VM_FAULT_FALLBACK;
489
490 ret = ttm_bo_vm_reserve(bo, vmf);
491 if (ret)
492 return ret;
493
494 prot = vm_get_page_prot(vma->vm_flags);
495 ret = ttm_bo_vm_fault_reserved(vmf, prot, 1, fault_page_size);
496 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
497 return ret;
498
499 dma_resv_unlock(bo->base.resv);
500
501 return ret;
502}
503#endif
504
353void ttm_bo_vm_open(struct vm_area_struct *vma)505void ttm_bo_vm_open(struct vm_area_struct *vma)
354{506{
355 struct ttm_buffer_object *bo = vma->vm_private_data;507 struct ttm_buffer_object *bo = vma->vm_private_data;
@@ -451,7 +603,10 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { · 95 unmodified lines
451 .fault = ttm_bo_vm_fault,603 .fault = ttm_bo_vm_fault,
452 .open = ttm_bo_vm_open,604 .open = ttm_bo_vm_open,
453 .close = ttm_bo_vm_close,605 .close = ttm_bo_vm_close,
454 .access = ttm_bo_vm_access606 .access = ttm_bo_vm_access,
607#ifdef CONFIG_TRANSPARENT_HUGEPAGE
608 .huge_fault = ttm_bo_vm_huge_fault,
609#endif
455};610};
456 611
457static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,612static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
drivers/gpu/drm/vmwgfx/Makefile+1−0
@@ -11,4 +11,5 @@ vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \ · 10 unmodified lines
11 vmwgfx_validation.o vmwgfx_page_dirty.o vmwgfx_streamoutput.o \11 vmwgfx_validation.o vmwgfx_page_dirty.o vmwgfx_streamoutput.o \
12 ttm_object.o ttm_lock.o12 ttm_object.o ttm_lock.o
13 13
14vmwgfx-$(CONFIG_TRANSPARENT_HUGEPAGE) += vmwgfx_thp.o
14obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o15obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o
drivers/gpu/drm/vmwgfx/vmwgfx_drv.c+13−0
@@ -1247,6 +1247,18 @@ static void vmw_remove(struct pci_dev *pdev) · 1246 unmodified lines
1247 pci_disable_device(pdev);1247 pci_disable_device(pdev);
1248}1248}
1249 1249
1250static unsigned long
1251vmw_get_unmapped_area(struct file *file, unsigned long uaddr,
1252 unsigned long len, unsigned long pgoff,
1253 unsigned long flags)
1254{
1255 struct drm_file *file_priv = file->private_data;
1256 struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);
1257
1258 return drm_get_unmapped_area(file, uaddr, len, pgoff, flags,
1259 &dev_priv->vma_manager);
1260}
1261
1250static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,1262static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
1251 void *ptr)1263 void *ptr)
1252{1264{
@@ -1418,6 +1430,7 @@ static const struct file_operations vmwgfx_driver_fops = { · 165 unmodified lines
1418 .compat_ioctl = vmw_compat_ioctl,1430 .compat_ioctl = vmw_compat_ioctl,
1419#endif1431#endif
1420 .llseek = noop_llseek,1432 .llseek = noop_llseek,
1433 .get_unmapped_area = vmw_get_unmapped_area,
1421};1434};
1422 1435
1423static struct drm_driver driver = {1436static struct drm_driver driver = {
drivers/gpu/drm/vmwgfx/vmwgfx_drv.h+12−0
@@ -1000,6 +1000,7 @@ extern int vmw_mmap(struct file *filp, struct vm_area_struct *vma); · 999 unmodified lines
1000 1000
1001extern void vmw_validation_mem_init_ttm(struct vmw_private *dev_priv,1001extern void vmw_validation_mem_init_ttm(struct vmw_private *dev_priv,
1002 size_t gran);1002 size_t gran);
1003
1003/**1004/**
1004 * TTM buffer object driver - vmwgfx_ttm_buffer.c1005 * TTM buffer object driver - vmwgfx_ttm_buffer.c
1005 */1006 */
@@ -1510,6 +1511,17 @@ void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo, · 504 unmodified lines
1510 pgoff_t start, pgoff_t end);1511 pgoff_t start, pgoff_t end);
1511vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf);1512vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf);
1512vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf);1513vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf);
1514#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1515vm_fault_t vmw_bo_vm_huge_fault(struct vm_fault *vmf,
1516 enum page_entry_size pe_size);
1517#endif
1518
1519/* Transparent hugepage support - vmwgfx_thp.c */
1520#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1521extern const struct ttm_mem_type_manager_func vmw_thp_func;
1522#else
1523#define vmw_thp_func ttm_bo_manager_func
1524#endif
1513 1525
1514/**1526/**
1515 * VMW_DEBUG_KMS - Debug output for kernel mode-setting1527 * VMW_DEBUG_KMS - Debug output for kernel mode-setting
drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c+74−2
@@ -473,11 +473,11 @@ vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf) · 472 unmodified lines
473 * a lot of unnecessary write faults.473 * a lot of unnecessary write faults.
474 */474 */
475 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)475 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
476 prot = vma->vm_page_prot;476 prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
477 else477 else
478 prot = vm_get_page_prot(vma->vm_flags);478 prot = vm_get_page_prot(vma->vm_flags);
479 479
480 ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);480 ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault, 1);
481 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))481 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
482 return ret;482 return ret;
483 483
@@ -486,3 +486,75 @@ out_unlock: · 2 unmodified lines
486 486
487 return ret;487 return ret;
488}488}
489
490#ifdef CONFIG_TRANSPARENT_HUGEPAGE
491vm_fault_t vmw_bo_vm_huge_fault(struct vm_fault *vmf,
492 enum page_entry_size pe_size)
493{
494 struct vm_area_struct *vma = vmf->vma;
495 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
496 vma->vm_private_data;
497 struct vmw_buffer_object *vbo =
498 container_of(bo, struct vmw_buffer_object, base);
499 pgprot_t prot;
500 vm_fault_t ret;
501 pgoff_t fault_page_size;
502 bool write = vmf->flags & FAULT_FLAG_WRITE;
503 bool is_cow_mapping =
504 (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
505
506 switch (pe_size) {
507 case PE_SIZE_PMD:
508 fault_page_size = HPAGE_PMD_SIZE >> PAGE_SHIFT;
509 break;
510#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
511 case PE_SIZE_PUD:
512 fault_page_size = HPAGE_PUD_SIZE >> PAGE_SHIFT;
513 break;
514#endif
515 default:
516 WARN_ON_ONCE(1);
517 return VM_FAULT_FALLBACK;
518 }
519
520 /* Always do write dirty-tracking and COW on PTE level. */
521 if (write && (READ_ONCE(vbo->dirty) || is_cow_mapping))
522 return VM_FAULT_FALLBACK;
523
524 ret = ttm_bo_vm_reserve(bo, vmf);
525 if (ret)
526 return ret;
527
528 if (vbo->dirty) {
529 pgoff_t allowed_prefault;
530 unsigned long page_offset;
531
532 page_offset = vmf->pgoff -
533 drm_vma_node_start(&bo->base.vma_node);
534 if (page_offset >= bo->num_pages ||
535 vmw_resources_clean(vbo, page_offset,
536 page_offset + PAGE_SIZE,
537 &allowed_prefault)) {
538 ret = VM_FAULT_SIGBUS;
539 goto out_unlock;
540 }
541
542 /*
543 * Write protect, so we get a new fault on write, and can
544 * split.
545 */
546 prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
547 } else {
548 prot = vm_get_page_prot(vma->vm_flags);
549 }
550
551 ret = ttm_bo_vm_fault_reserved(vmf, prot, 1, fault_page_size);
552 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
553 return ret;
554
555out_unlock:
556 dma_resv_unlock(bo->base.resv);
557
558 return ret;
559}
560#endif
drivers/gpu/drm/vmwgfx/vmwgfx_thp.c+166−0
@@ -0,0 +1,166 @@
1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Huge page-table-entry support for IO memory.
4 *
5 * Copyright (C) 2007-2019 Vmware, Inc. All rights reservedd.
6 */
7#include "vmwgfx_drv.h"
8#include <drm/ttm/ttm_module.h>
9#include <drm/ttm/ttm_bo_driver.h>
10#include <drm/ttm/ttm_placement.h>
11
12/**
13 * struct vmw_thp_manager - Range manager implementing huge page alignment
14 *
15 * @mm: The underlying range manager. Protected by @lock.
16 * @lock: Manager lock.
17 */
18struct vmw_thp_manager {
19 struct drm_mm mm;
20 spinlock_t lock;
21};
22
23static int vmw_thp_insert_aligned(struct drm_mm *mm, struct drm_mm_node *node,
24 unsigned long align_pages,
25 const struct ttm_place *place,
26 struct ttm_mem_reg *mem,
27 unsigned long lpfn,
28 enum drm_mm_insert_mode mode)
29{
30 if (align_pages >= mem->page_alignment &&
31 (!mem->page_alignment || align_pages % mem->page_alignment == 0)) {
32 return drm_mm_insert_node_in_range(mm, node,
33 mem->num_pages,
34 align_pages, 0,
35 place->fpfn, lpfn, mode);
36 }
37
38 return -ENOSPC;
39}
40
41static int vmw_thp_get_node(struct ttm_mem_type_manager *man,
42 struct ttm_buffer_object *bo,
43 const struct ttm_place *place,
44 struct ttm_mem_reg *mem)
45{
46 struct vmw_thp_manager *rman = (struct vmw_thp_manager *) man->priv;
47 struct drm_mm *mm = &rman->mm;
48 struct drm_mm_node *node;
49 unsigned long align_pages;
50 unsigned long lpfn;
51 enum drm_mm_insert_mode mode = DRM_MM_INSERT_BEST;
52 int ret;
53
54 node = kzalloc(sizeof(*node), GFP_KERNEL);
55 if (!node)
56 return -ENOMEM;
57
58 lpfn = place->lpfn;
59 if (!lpfn)
60 lpfn = man->size;
61
62 mode = DRM_MM_INSERT_BEST;
63 if (place->flags & TTM_PL_FLAG_TOPDOWN)
64 mode = DRM_MM_INSERT_HIGH;
65
66 spin_lock(&rman->lock);
67 if (IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) {
68 align_pages = (HPAGE_PUD_SIZE >> PAGE_SHIFT);
69 if (mem->num_pages >= align_pages) {
70 ret = vmw_thp_insert_aligned(mm, node, align_pages,
71 place, mem, lpfn, mode);
72 if (!ret)
73 goto found_unlock;
74 }
75 }
76
77 align_pages = (HPAGE_PMD_SIZE >> PAGE_SHIFT);
78 if (mem->num_pages >= align_pages) {
79 ret = vmw_thp_insert_aligned(mm, node, align_pages, place, mem,
80 lpfn, mode);
81 if (!ret)
82 goto found_unlock;
83 }
84
85 ret = drm_mm_insert_node_in_range(mm, node, mem->num_pages,
86 mem->page_alignment, 0,
87 place->fpfn, lpfn, mode);
88found_unlock:
89 spin_unlock(&rman->lock);
90
91 if (unlikely(ret)) {
92 kfree(node);
93 } else {
94 mem->mm_node = node;
95 mem->start = node->start;
96 }
97
98 return 0;
99}
100
101
102
103static void vmw_thp_put_node(struct ttm_mem_type_manager *man,
104 struct ttm_mem_reg *mem)
105{
106 struct vmw_thp_manager *rman = (struct vmw_thp_manager *) man->priv;
107
108 if (mem->mm_node) {
109 spin_lock(&rman->lock);
110 drm_mm_remove_node(mem->mm_node);
111 spin_unlock(&rman->lock);
112
113 kfree(mem->mm_node);
114 mem->mm_node = NULL;
115 }
116}
117
118static int vmw_thp_init(struct ttm_mem_type_manager *man,
119 unsigned long p_size)
120{
121 struct vmw_thp_manager *rman;
122
123 rman = kzalloc(sizeof(*rman), GFP_KERNEL);
124 if (!rman)
125 return -ENOMEM;
126
127 drm_mm_init(&rman->mm, 0, p_size);
128 spin_lock_init(&rman->lock);
129 man->priv = rman;
130 return 0;
131}
132
133static int vmw_thp_takedown(struct ttm_mem_type_manager *man)
134{
135 struct vmw_thp_manager *rman = (struct vmw_thp_manager *) man->priv;
136 struct drm_mm *mm = &rman->mm;
137
138 spin_lock(&rman->lock);
139 if (drm_mm_clean(mm)) {
140 drm_mm_takedown(mm);
141 spin_unlock(&rman->lock);
142 kfree(rman);
143 man->priv = NULL;
144 return 0;
145 }
146 spin_unlock(&rman->lock);
147 return -EBUSY;
148}
149
150static void vmw_thp_debug(struct ttm_mem_type_manager *man,
151 struct drm_printer *printer)
152{
153 struct vmw_thp_manager *rman = (struct vmw_thp_manager *) man->priv;
154
155 spin_lock(&rman->lock);
156 drm_mm_print(&rman->mm, printer);
157 spin_unlock(&rman->lock);
158}
159
160const struct ttm_mem_type_manager_func vmw_thp_func = {
161 .init = vmw_thp_init,
162 .takedown = vmw_thp_takedown,
163 .get_node = vmw_thp_get_node,
164 .put_node = vmw_thp_put_node,
165 .debug = vmw_thp_debug
166};
drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c+1−1
@@ -749,7 +749,7 @@ static int vmw_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, · 748 unmodified lines
749 break;749 break;
750 case TTM_PL_VRAM:750 case TTM_PL_VRAM:
751 /* "On-card" video ram */751 /* "On-card" video ram */
752 man->func = &ttm_bo_manager_func;752 man->func = &vmw_thp_func;
753 man->gpu_offset = 0;753 man->gpu_offset = 0;
754 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE;754 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE;
755 man->available_caching = TTM_PL_FLAG_CACHED;755 man->available_caching = TTM_PL_FLAG_CACHED;
drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c+4−1
@@ -34,7 +34,10 @@ int vmw_mmap(struct file *filp, struct vm_area_struct *vma) · 33 unmodified lines
34 .page_mkwrite = vmw_bo_vm_mkwrite,34 .page_mkwrite = vmw_bo_vm_mkwrite,
35 .fault = vmw_bo_vm_fault,35 .fault = vmw_bo_vm_fault,
36 .open = ttm_bo_vm_open,36 .open = ttm_bo_vm_open,
37 .close = ttm_bo_vm_close37 .close = ttm_bo_vm_close,
38#ifdef CONFIG_TRANSPARENT_HUGEPAGE
39 .huge_fault = vmw_bo_vm_huge_fault,
40#endif
38 };41 };
39 struct drm_file *file_priv = filp->private_data;42 struct drm_file *file_priv = filp->private_data;
40 struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);43 struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);
include/drm/drm_file.h+9−0
@@ -391,4 +391,13 @@ void drm_send_event(struct drm_device *dev, struct drm_pending_event *e); · 390 unmodified lines
391 391
392struct file *mock_drm_getfile(struct drm_minor *minor, unsigned int flags);392struct file *mock_drm_getfile(struct drm_minor *minor, unsigned int flags);
393 393
394#ifdef CONFIG_MMU
395struct drm_vma_offset_manager;
396unsigned long drm_get_unmapped_area(struct file *file,
397 unsigned long uaddr, unsigned long len,
398 unsigned long pgoff, unsigned long flags,
399 struct drm_vma_offset_manager *mgr);
400#endif /* CONFIG_MMU */
401
402
394#endif /* _DRM_FILE_H_ */403#endif /* _DRM_FILE_H_ */
include/drm/ttm/ttm_bo_api.h+2−1
@@ -727,7 +727,8 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, · 726 unmodified lines
727 727
728vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,728vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
729 pgprot_t prot,729 pgprot_t prot,
730 pgoff_t num_prefault);730 pgoff_t num_prefault,
731 pgoff_t fault_page_size);
731 732
732vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf);733vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf);
733 734
include/linux/fs.h+1−1
@@ -3399,7 +3399,7 @@ static inline bool io_is_direct(struct file *filp) · 3398 unmodified lines
3399 return (filp->f_flags & O_DIRECT) || IS_DAX(filp->f_mapping->host);3399 return (filp->f_flags & O_DIRECT) || IS_DAX(filp->f_mapping->host);
3400}3400}
3401 3401
3402static inline bool vma_is_dax(struct vm_area_struct *vma)3402static inline bool vma_is_dax(const struct vm_area_struct *vma)
3403{3403{
3404 return vma->vm_file && IS_DAX(vma->vm_file->f_mapping->host);3404 return vma->vm_file && IS_DAX(vma->vm_file->f_mapping->host);
3405}3405}
include/linux/huge_mm.h+39−2
@@ -47,8 +47,45 @@ extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, · 46 unmodified lines
47extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,47extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
48 unsigned long addr, pgprot_t newprot,48 unsigned long addr, pgprot_t newprot,
49 int prot_numa);49 int prot_numa);
50vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write);50vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,
51vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write);51 pgprot_t pgprot, bool write);
52
53/**
54 * vmf_insert_pfn_pmd - insert a pmd size pfn
55 * @vmf: Structure describing the fault
56 * @pfn: pfn to insert
57 * @pgprot: page protection to use
58 * @write: whether it's a write fault
59 *
60 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info.
61 *
62 * Return: vm_fault_t value.
63 */
64static inline vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn,
65 bool write)
66{
67 return vmf_insert_pfn_pmd_prot(vmf, pfn, vmf->vma->vm_page_prot, write);
68}
69vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,
70 pgprot_t pgprot, bool write);
71
72/**
73 * vmf_insert_pfn_pud - insert a pud size pfn
74 * @vmf: Structure describing the fault
75 * @pfn: pfn to insert
76 * @pgprot: page protection to use
77 * @write: whether it's a write fault
78 *
79 * Insert a pud size pfn. See vmf_insert_pfn() for additional info.
80 *
81 * Return: vm_fault_t value.
82 */
83static inline vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn,
84 bool write)
85{
86 return vmf_insert_pfn_pud_prot(vmf, pfn, vmf->vma->vm_page_prot, write);
87}
88
52enum transparent_hugepage_flag {89enum transparent_hugepage_flag {
53 TRANSPARENT_HUGEPAGE_FLAG,90 TRANSPARENT_HUGEPAGE_FLAG,
54 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,91 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
include/linux/mm.h+17−0
@@ -3014,6 +3014,23 @@ extern long copy_huge_page_from_user(struct page *dst_page, · 3013 unmodified lines
3014 const void __user *usr_src,3014 const void __user *usr_src,
3015 unsigned int pages_per_huge_page,3015 unsigned int pages_per_huge_page,
3016 bool allow_pagefault);3016 bool allow_pagefault);
3017
3018/**
3019 * vma_is_special_huge - Are transhuge page-table entries considered special?
3020 * @vma: Pointer to the struct vm_area_struct to consider
3021 *
3022 * Whether transhuge page-table entries are considered "special" following
3023 * the definition in vm_normal_page().
3024 *
3025 * Return: true if transhuge page-table entries should be considered special,
3026 * false otherwise.
3027 */
3028static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
3029{
3030 return vma_is_dax(vma) || (vma->vm_file &&
3031 (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
3032}
3033
3017#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */3034#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
3018 3035
3019#ifdef CONFIG_DEBUG_PAGEALLOC3036#ifdef CONFIG_DEBUG_PAGEALLOC
mm/huge_memory.c+35−9
@@ -824,11 +824,24 @@ out_unlock: · 823 unmodified lines
824 pte_free(mm, pgtable);824 pte_free(mm, pgtable);
825}825}
826 826
827vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write)827/**
828 * vmf_insert_pfn_pmd_prot - insert a pmd size pfn
829 * @vmf: Structure describing the fault
830 * @pfn: pfn to insert
831 * @pgprot: page protection to use
832 * @write: whether it's a write fault
833 *
834 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info and
835 * also consult the vmf_insert_mixed_prot() documentation when
836 * @pgprot != @vmf->vma->vm_page_prot.
837 *
838 * Return: vm_fault_t value.
839 */
840vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,
841 pgprot_t pgprot, bool write)
828{842{
829 unsigned long addr = vmf->address & PMD_MASK;843 unsigned long addr = vmf->address & PMD_MASK;
830 struct vm_area_struct *vma = vmf->vma;844 struct vm_area_struct *vma = vmf->vma;
831 pgprot_t pgprot = vma->vm_page_prot;
832 pgtable_t pgtable = NULL;845 pgtable_t pgtable = NULL;
833 846
834 /*847 /*
@@ -856,7 +869,7 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) · 21 unmodified lines
856 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);869 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);
857 return VM_FAULT_NOPAGE;870 return VM_FAULT_NOPAGE;
858}871}
859EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);872EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd_prot);
860 873
861#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD874#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
862static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)875static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
@@ -902,11 +915,24 @@ out_unlock: · 39 unmodified lines
902 spin_unlock(ptl);915 spin_unlock(ptl);
903}916}
904 917
905vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write)918/**
919 * vmf_insert_pfn_pud_prot - insert a pud size pfn
920 * @vmf: Structure describing the fault
921 * @pfn: pfn to insert
922 * @pgprot: page protection to use
923 * @write: whether it's a write fault
924 *
925 * Insert a pud size pfn. See vmf_insert_pfn() for additional info and
926 * also consult the vmf_insert_mixed_prot() documentation when
927 * @pgprot != @vmf->vma->vm_page_prot.
928 *
929 * Return: vm_fault_t value.
930 */
931vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,
932 pgprot_t pgprot, bool write)
906{933{
907 unsigned long addr = vmf->address & PUD_MASK;934 unsigned long addr = vmf->address & PUD_MASK;
908 struct vm_area_struct *vma = vmf->vma;935 struct vm_area_struct *vma = vmf->vma;
909 pgprot_t pgprot = vma->vm_page_prot;
910 936
911 /*937 /*
912 * If we had pud_special, we could avoid all these restrictions,938 * If we had pud_special, we could avoid all these restrictions,
@@ -927,7 +953,7 @@ vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write) · 14 unmodified lines
927 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);953 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);
928 return VM_FAULT_NOPAGE;954 return VM_FAULT_NOPAGE;
929}955}
930EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);956EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud_prot);
931#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */957#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
932 958
933static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,959static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
@@ -1819,7 +1845,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · 885 unmodified lines
1819 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,1845 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1820 tlb->fullmm);1846 tlb->fullmm);
1821 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);1847 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1822 if (vma_is_dax(vma)) {1848 if (vma_is_special_huge(vma)) {
1823 if (arch_needs_pgtable_deposit())1849 if (arch_needs_pgtable_deposit())
1824 zap_deposited_table(tlb->mm, pmd);1850 zap_deposited_table(tlb->mm, pmd);
1825 spin_unlock(ptl);1851 spin_unlock(ptl);
@@ -2083,7 +2109,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, · 257 unmodified lines
2083 */2109 */
2084 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm);2110 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm);
2085 tlb_remove_pud_tlb_entry(tlb, pud, addr);2111 tlb_remove_pud_tlb_entry(tlb, pud, addr);
2086 if (vma_is_dax(vma)) {2112 if (vma_is_special_huge(vma)) {
2087 spin_unlock(ptl);2113 spin_unlock(ptl);
2088 /* No zero page support yet */2114 /* No zero page support yet */
2089 } else {2115 } else {
@@ -2192,7 +2218,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, · 102 unmodified lines
2192 */2218 */
2193 if (arch_needs_pgtable_deposit())2219 if (arch_needs_pgtable_deposit())
2194 zap_deposited_table(mm, pmd);2220 zap_deposited_table(mm, pmd);
2195 if (vma_is_dax(vma))2221 if (vma_is_special_huge(vma))
2196 return;2222 return;
2197 page = pmd_page(_pmd);2223 page = pmd_page(_pmd);
2198 if (!PageDirty(page) && pmd_dirty(_pmd))2224 if (!PageDirty(page) && pmd_dirty(_pmd))
mm/memory.c+19−8
@@ -3951,11 +3951,14 @@ static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd) · 3950 unmodified lines
3951{3951{
3952 if (vma_is_anonymous(vmf->vma))3952 if (vma_is_anonymous(vmf->vma))
3953 return do_huge_pmd_wp_page(vmf, orig_pmd);3953 return do_huge_pmd_wp_page(vmf, orig_pmd);
3954 if (vmf->vma->vm_ops->huge_fault)3954 if (vmf->vma->vm_ops->huge_fault) {
3955 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);3955 vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
3956 3956
3957 /* COW handled on pte level: split pmd */3957 if (!(ret & VM_FAULT_FALLBACK))
3958 VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);3958 return ret;
3959 }
3960
3961 /* COW or write-notify handled on pte level: split pmd. */
3959 __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);3962 __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
3960 3963
3961 return VM_FAULT_FALLBACK;3964 return VM_FAULT_FALLBACK;
@@ -3968,12 +3971,20 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma) · 6 unmodified lines
3968 3971
3969static vm_fault_t create_huge_pud(struct vm_fault *vmf)3972static vm_fault_t create_huge_pud(struct vm_fault *vmf)
3970{3973{
3971#ifdef CONFIG_TRANSPARENT_HUGEPAGE3974#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
3975 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
3972 /* No support for anonymous transparent PUD pages yet */3976 /* No support for anonymous transparent PUD pages yet */
3973 if (vma_is_anonymous(vmf->vma))3977 if (vma_is_anonymous(vmf->vma))
3974 return VM_FAULT_FALLBACK;3978 goto split;
3975 if (vmf->vma->vm_ops->huge_fault)3979 if (vmf->vma->vm_ops->huge_fault) {
3976 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);3980 vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
3981
3982 if (!(ret & VM_FAULT_FALLBACK))
3983 return ret;
3984 }
3985split:
3986 /* COW or write-notify not handled on PUD level: split pud.*/
3987 __split_huge_pud(vmf->vma, vmf->pud, vmf->address);
3977#endif /* CONFIG_TRANSPARENT_HUGEPAGE */3988#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3978 return VM_FAULT_FALLBACK;3989 return VM_FAULT_FALLBACK;
3979}3990}