History

Merge tag 'folio-5.18c' of git://git.infradead.org/users/willy/pagecache

Pull folio updates from Matthew Wilcox: - Rewrite how munlock works to massively reduce the contention on i_mmap_rwsem (Hugh Dickins): https://lore.kernel.org/linux-mm/8e4356d-9622-a7f0-b2c-f116b5f2efea@google.com/ - Sort out the page refcount mess for ZONE_DEVICE pages (Christoph Hellwig): https://lore.kernel.org/linux-mm/20220210072828.2930359-1-hch@lst.de/ - Convert GUP to use folios and make pincount available for order-1 pages. (Matthew Wilcox) - Convert a few more truncation functions to use folios (Matthew Wilcox) - Convert page_vma_mapped_walk to use PFNs instead of pages (Matthew Wilcox) - Convert rmap_walk to use folios (Matthew Wilcox) - Convert most of shrink_page_list() to use a folio (Matthew Wilcox) - Add support for creating large folios in readahead (Matthew Wilcox) * tag 'folio-5.18c' of git://git.infradead.org/users/willy/pagecache: (114 commits) mm/damon: minor cleanup for damon_pa_young selftests/vm/transhuge-stress: Support file-backed PMD folios mm/filemap: Support VM_HUGEPAGE for file mappings mm/readahead: Switch to page_cache_ra_order mm/readahead: Align file mappings for non-DAX mm/readahead: Add large folio readahead mm: Support arbitrary THP sizes mm: Make large folios depend on THP mm: Fix READ_ONLY_THP warning mm/filemap: Allow large folios to be added to the page cache mm: Turn can_split_huge_page() into can_split_folio() mm/vmscan: Convert pageout() to take a folio mm/vmscan: Turn page_check_references() into folio_check_references() mm/vmscan: Account large folios correctly mm/vmscan: Optimise shrink_page_list for non-PMD-sized folios mm/vmscan: Free non-shmem folios without splitting them mm/rmap: Constify the rmap_walk_control argument mm/rmap: Convert rmap_walk() to take a folio mm: Turn page_anon_vma() into folio_anon_vma() mm/rmap: Turn page_lock_anon_vma_read() into folio_lock_anon_vma_read() ...

Linus Torvalds committed 4 years ago

Commit 9030fb02 parents 3bf03b9 2a3c4bc100 files · +2900 −3020Browse files

Unified · Split

100 files
  1. Documentation/core-api/pin_user_pages.rst +9 −9
  2. arch/alpha/include/asm/pgtable.h +1 −0
  3. arch/arc/include/asm/hugepage.h +0 −1
  4. arch/arc/include/asm/pgtable-levels.h +1 −0
  5. arch/arm/include/asm/pgtable-2level.h +2 −0
  6. arch/arm64/mm/mmu.c +1 −0
  7. arch/csky/include/asm/pgtable.h +1 −0
  8. arch/hexagon/include/asm/pgtable.h +5 −0
  9. arch/ia64/include/asm/pgtable.h +1 −0
  10. arch/m68k/include/asm/mcf_pgtable.h +1 −0
  11. arch/m68k/include/asm/motorola_pgtable.h +1 −0
  12. arch/m68k/include/asm/sun3_pgtable.h +1 −0
  13. arch/microblaze/include/asm/pgtable.h +3 −0
  14. arch/mips/include/asm/pgtable.h +5 −5
  15. arch/nds32/include/asm/pgtable.h +1 −0
  16. arch/nios2/include/asm/pgtable.h +1 −0
  17. arch/openrisc/include/asm/pgtable.h +1 −0
  18. arch/parisc/include/asm/pgtable.h +1 −0
  19. arch/powerpc/include/asm/book3s/32/pgtable.h +2 −2
  20. arch/powerpc/include/asm/mmu_context.h +0 −1
  21. arch/powerpc/include/asm/nohash/32/pgtable.h +3 −4
  22. arch/powerpc/include/asm/nohash/64/pgtable.h +1 −0
  23. arch/powerpc/kvm/book3s_hv_uvmem.c +1 −1
  24. arch/powerpc/mm/book3s64/pgtable.c +1 −0
  25. arch/sh/include/asm/pgtable_32.h +1 −0
  26. arch/sparc/include/asm/pgtable_32.h +6 −1
  27. arch/um/include/asm/pgtable.h +1 −0
  28. arch/xtensa/include/asm/pgtable.h +1 −0
  29. drivers/block/xen-blkback/xenbus.c +1 −0
  30. drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 −1
  31. drivers/gpu/drm/amd/amdkfd/kfd_priv.h +1 −0
  32. drivers/gpu/drm/drm_cache.c +1 −1
  33. drivers/gpu/drm/nouveau/nouveau_dmem.c +2 −1
  34. drivers/gpu/drm/nouveau/nouveau_svm.c +1 −0
  35. drivers/infiniband/core/rw.c +1 −0
  36. drivers/nvdimm/pmem.h +1 −0
  37. drivers/nvme/host/pci.c +1 −0
  38. drivers/nvme/target/io-cmd-bdev.c +1 −0
  39. drivers/usb/gadget/function/f_mass_storage.c +1 −0
  40. fs/Kconfig +1 −1
  41. fs/coda/file.c +1 −0
  42. fs/fuse/virtio_fs.c +1 −0
  43. fs/iomap/fiemap.c +1 −0
  44. fs/nfsd/filecache.c +1 −0
  45. fs/nfsd/vfs.c +1 −0
  46. fs/proc/page.c +1 −0
  47. fs/splice.c +12 −12
  48. fs/vboxsf/utils.c +1 −0
  49. include/linux/fs.h +0 −120
  50. include/linux/hmm.h +2 −7
  51. include/linux/huge_mm.h +9 −50
  52. include/linux/hugetlb.h +5 −0
  53. include/linux/ksm.h +3 −3
  54. include/linux/memremap.h +24 −3
  55. include/linux/mm.h +152 −144
  56. include/linux/mm_inline.h +8 −3
  57. include/linux/mm_types.h +22 −4
  58. include/linux/pagemap.h +133 −13
  59. include/linux/rmap.h +47 −29
  60. include/linux/swap.h +4 −5
  61. include/trace/events/vmscan.h +5 −5
  62. kernel/events/uprobes.c +3 −10
  63. kernel/futex/core.c +1 −1
  64. lib/test_hmm.c +3 −1
  65. mm/Kconfig +3 −4
  66. mm/Makefile +1 −0
  67. mm/damon/paddr.c +29 −33
  68. mm/debug.c +7 −11
  69. mm/filemap.c +41 −18
  70. mm/folio-compat.c +13 −0
  71. mm/gup.c +205 −281
  72. mm/huge_memory.c +62 −116
  73. mm/hugetlb.c +9 −6
  74. mm/internal.h +79 −38
  75. mm/khugepaged.c +11 −4
  76. mm/ksm.c +10 −22
  77. mm/madvise.c +5 −0
  78. mm/memcontrol.c +16 −21
  79. mm/memory-failure.c +7 −5
  80. mm/memory.c +15 −28
  81. mm/memory_hotplug.c +8 −5
  82. mm/memremap.c +34 −31
  83. mm/migrate.c +56 −814
  84. mm/migrate_device.c +773 −0
  85. mm/mlock.c +266 −372
  86. mm/mmap.c +2 −30
  87. mm/mmzone.c +7 −0
  88. mm/oom_kill.c +1 −1
  89. mm/page_alloc.c +1 −2
  90. mm/page_idle.c +14 −16
  91. mm/page_vma_mapped.c +26 −32
  92. mm/readahead.c +100 −8
  93. mm/rmap.c +239 −339
  94. mm/swap.c +68 −105
  95. mm/truncate.c +51 −60
  96. mm/userfaultfd.c +8 −6
  97. mm/util.c +34 −2
  98. mm/vmscan.c +156 −149
  99. mm/workingset.c +13 −12
  100. tools/testing/selftests/vm/transhuge-stress.c +24 −11
Documentation/core-api/pin_user_pages.rst+9−9
@@ -55,18 +55,18 @@ flags the caller provides. The caller is required to pass in a non-null struct · 54 unmodified lines
5555 pages* array, and the function then pins pages by incrementing each by a special
5656 value: GUP_PIN_COUNTING_BIAS.
5757
58−For huge pages (and in fact, any compound page of more than 2 pages), the
59−GUP_PIN_COUNTING_BIAS scheme is not used. Instead, an exact form of pin counting
60−is achieved, by using the 3rd struct page in the compound page. A new struct
61−page field, hpage_pinned_refcount, has been added in order to support this.
58+For compound pages, the GUP_PIN_COUNTING_BIAS scheme is not used. Instead,
59+an exact form of pin counting is achieved, by using the 2nd struct page
60+in the compound page. A new struct page field, compound_pincount, has
61+been added in order to support this.
6262
6363 This approach for compound pages avoids the counting upper limit problems that
6464 are discussed below. Those limitations would have been aggravated severely by
6565 huge pages, because each tail page adds a refcount to the head page. And in
66−fact, testing revealed that, without a separate hpage_pinned_refcount field,
66+fact, testing revealed that, without a separate compound_pincount field,
6767 page overflows were seen in some huge page stress tests.
6868
69−This also means that huge pages and compound pages (of order > 1) do not suffer
69+This also means that huge pages and compound pages do not suffer
7070 from the false positives problem that is mentioned below.::
7171
7272 Function
@@ -264,9 +264,9 @@ place.) · 191 unmodified lines
264264 Other diagnostics
265265 =================
266266
267−dump_page() has been enhanced slightly, to handle these new counting fields, and
268−to better report on compound pages in general. Specifically, for compound pages
269−with order > 1, the exact (hpage_pinned_refcount) pincount is reported.
267+dump_page() has been enhanced slightly, to handle these new counting
268+fields, and to better report on compound pages in general. Specifically,
269+for compound pages, the exact (compound_pincount) pincount is reported.
270270
271271 References
272272 ==========
arch/alpha/include/asm/pgtable.h+1−0
@@ -233,6 +233,7 @@ pmd_page_vaddr(pmd_t pmd) · 232 unmodified lines
233233 return ((pmd_val(pmd) & _PFN_MASK) >> (32-PAGE_SHIFT)) + PAGE_OFFSET;
234234 }
235235
236+#define pmd_pfn(pmd) (pmd_val(pmd) >> 32)
236237 #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> 32))
237238 #define pud_page(pud) (pfn_to_page(pud_val(pud) >> 32))
238239
arch/arc/include/asm/hugepage.h+0−1
@@ -31,7 +31,6 @@ static inline pmd_t pte_pmd(pte_t pte) · 30 unmodified lines
3131
3232 #define pmd_write(pmd) pte_write(pmd_pte(pmd))
3333 #define pmd_young(pmd) pte_young(pmd_pte(pmd))
34−#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
3534 #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
3635
3736 #define mk_pmd(page, prot) pte_pmd(mk_pte(page, prot))
arch/arc/include/asm/pgtable-levels.h+1−0
@@ -161,6 +161,7 @@ · 160 unmodified lines
161161 #define pmd_present(x) (pmd_val(x))
162162 #define pmd_clear(xp) do { pmd_val(*(xp)) = 0; } while (0)
163163 #define pmd_page_vaddr(pmd) (pmd_val(pmd) & PAGE_MASK)
164+#define pmd_pfn(pmd) ((pmd_val(pmd) & PAGE_MASK) >> PAGE_SHIFT)
164165 #define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd))
165166 #define set_pmd(pmdp, pmd) (*(pmdp) = pmd)
166167 #define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd))
arch/arm/include/asm/pgtable-2level.h+2−0
@@ -208,6 +208,8 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) · 207 unmodified lines
208208 }
209209 #define pmd_offset pmd_offset
210210
211+#define pmd_pfn(pmd) (__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
212+
211213 #define pmd_large(pmd) (pmd_val(pmd) & 2)
212214 #define pmd_leaf(pmd) (pmd_val(pmd) & 2)
213215 #define pmd_bad(pmd) (pmd_val(pmd) & 2)
arch/arm64/mm/mmu.c+1−0
@@ -17,6 +17,7 @@ · 16 unmodified lines
1717 #include <linux/mman.h>
1818 #include <linux/nodemask.h>
1919 #include <linux/memblock.h>
20+#include <linux/memremap.h>
2021 #include <linux/memory.h>
2122 #include <linux/fs.h>
2223 #include <linux/io.h>
arch/csky/include/asm/pgtable.h+1−0
@@ -30,6 +30,7 @@ · 29 unmodified lines
3030 #define pgd_ERROR(e) \
3131 pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
3232
33+#define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT)
3334 #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
3435 #define pte_clear(mm, addr, ptep) set_pte((ptep), \
3536 (((unsigned int) addr >= PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0)))
arch/hexagon/include/asm/pgtable.h+5−0
@@ -235,6 +235,11 @@ static inline int pmd_bad(pmd_t pmd) · 234 unmodified lines
235235 return 0;
236236 }
237237
238+/*
239+ * pmd_pfn - converts a PMD entry to a page frame number
240+ */
241+#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
242+
238243 /*
239244 * pmd_page - converts a PMD entry to a page pointer
240245 */
arch/ia64/include/asm/pgtable.h+1−0
@@ -267,6 +267,7 @@ ia64_phys_addr_valid (unsigned long addr) · 266 unmodified lines
267267 #define pmd_present(pmd) (pmd_val(pmd) != 0UL)
268268 #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
269269 #define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val(pmd) & _PFN_MASK))
270+#define pmd_pfn(pmd) ((pmd_val(pmd) & _PFN_MASK) >> PAGE_SHIFT)
270271 #define pmd_page(pmd) virt_to_page((pmd_val(pmd) + PAGE_OFFSET))
271272
272273 #define pud_none(pud) (!pud_val(pud))
arch/m68k/include/asm/mcf_pgtable.h+1−0
@@ -322,6 +322,7 @@ extern pgd_t kernel_pg_dir[PTRS_PER_PGD]; · 321 unmodified lines
322322 #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
323323 #define __swp_entry_to_pte(x) (__pte((x).val))
324324
325+#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
325326 #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT))
326327
327328 #define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot))
arch/m68k/include/asm/motorola_pgtable.h+1−0
@@ -147,6 +147,7 @@ static inline void pud_set(pud_t *pudp, pmd_t *pmdp) · 146 unmodified lines
147147 #define pmd_present(pmd) (pmd_val(pmd) & _PAGE_TABLE)
148148 #define pmd_clear(pmdp) ({ pmd_val(*pmdp) = 0; })
149149
150+#define pmd_pfn(pmd) ((pmd_val(pmd) & _TABLE_MASK) >> PAGE_SHIFT)
150151 /*
151152 * m68k does not have huge pages (020/030 actually could), but generic code
152153 * expects pmd_page() to exists, only to then DCE it all. Provide a dummy to
arch/m68k/include/asm/sun3_pgtable.h+1−0
@@ -130,6 +130,7 @@ static inline void pte_clear (struct mm_struct *mm, unsigned long addr, pte_t *p · 129 unmodified lines
130130 ({ pte_t __pte; pte_val(__pte) = pfn | pgprot_val(pgprot); __pte; })
131131
132132 #define pte_page(pte) virt_to_page(__pte_page(pte))
133+#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
133134 #define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd))
134135
135136
arch/microblaze/include/asm/pgtable.h+3−0
@@ -399,6 +399,9 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 398 unmodified lines
399399 return ((unsigned long) (pmd_val(pmd) & PAGE_MASK));
400400 }
401401
402+/* returns pfn of the pmd entry*/
403+#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT)
404+
402405 /* returns struct *page of the pmd entry*/
403406 #define pmd_page(pmd) (pfn_to_page(__pa(pmd_val(pmd)) >> PAGE_SHIFT))
404407
arch/mips/include/asm/pgtable.h+5−5
@@ -86,6 +86,11 @@ extern void paging_init(void); · 85 unmodified lines
8686 */
8787 #define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd))
8888
89+static inline unsigned long pmd_pfn(pmd_t pmd)
90+{
91+ return pmd_val(pmd) >> _PFN_SHIFT;
92+}
93+
8994 #ifndef CONFIG_MIPS_HUGE_TLB_SUPPORT
9095 #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
9196 #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
@@ -422,11 +427,6 @@ static inline int pmd_write(pmd_t pmd) · 330 unmodified lines
422427 return !!(pmd_val(pmd) & _PAGE_WRITE);
423428 }
424429
425−static inline unsigned long pmd_pfn(pmd_t pmd)
426−{
427− return pmd_val(pmd) >> _PFN_SHIFT;
428−}
429−
430430 static inline struct page *pmd_page(pmd_t pmd)
431431 {
432432 if (pmd_val(pmd) & _PAGE_HUGE)
arch/nds32/include/asm/pgtable.h+1−0
@@ -308,6 +308,7 @@ static inline pmd_t __mk_pmd(pte_t * ptep, unsigned long prot) · 307 unmodified lines
308308 return pmd;
309309 }
310310
311+#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
311312 #define pmd_page(pmd) virt_to_page(__va(pmd_val(pmd)))
312313
313314 /*
arch/nios2/include/asm/pgtable.h+1−0
@@ -235,6 +235,7 @@ static inline void pte_clear(struct mm_struct *mm, · 234 unmodified lines
235235 * and a page entry and page directory to the page they refer to.
236236 */
237237 #define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd))
238+#define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT)
238239 #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
239240
240241 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
arch/openrisc/include/asm/pgtable.h+1−0
@@ -361,6 +361,7 @@ static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) · 360 unmodified lines
361361 pmd_val(*pmdp) = _KERNPG_TABLE | (unsigned long) ptep;
362362 }
363363
364+#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
364365 #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT))
365366
366367 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
arch/parisc/include/asm/pgtable.h+1−0
@@ -408,6 +408,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 407 unmodified lines
408408 return ((unsigned long) __va(pmd_address(pmd)));
409409 }
410410
411+#define pmd_pfn(pmd) (pmd_address(pmd) >> PAGE_SHIFT)
411412 #define __pmd_page(pmd) ((unsigned long) __va(pmd_address(pmd)))
412413 #define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd))
413414
arch/powerpc/include/asm/book3s/32/pgtable.h+2−2
@@ -372,8 +372,8 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, · 371 unmodified lines
372372 #define __HAVE_ARCH_PTE_SAME
373373 #define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)
374374
375−#define pmd_page(pmd) \
376− pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
375+#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
376+#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
377377
378378 /*
379379 * Encode and decode a swap entry.
arch/powerpc/include/asm/mmu_context.h+0−1
@@ -21,7 +21,6 @@ extern void destroy_context(struct mm_struct *mm); · 20 unmodified lines
2121 #ifdef CONFIG_SPAPR_TCE_IOMMU
2222 struct mm_iommu_table_group_mem_t;
2323
24−extern int isolate_lru_page(struct page *page); /* from internal.h */
2524 extern bool mm_iommu_preregistered(struct mm_struct *mm);
2625 extern long mm_iommu_new(struct mm_struct *mm,
2726 unsigned long ua, unsigned long entries,
arch/powerpc/include/asm/nohash/32/pgtable.h+3−4
@@ -349,15 +349,14 @@ static inline int pte_young(pte_t pte) · 348 unmodified lines
349349 * of the pte page. -- paulus
350350 */
351351 #ifndef CONFIG_BOOKE
352−#define pmd_page(pmd) \
353− pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
352+#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
354353 #else
355354 #define pmd_page_vaddr(pmd) \
356355 ((unsigned long)(pmd_val(pmd) & ~(PTE_TABLE_SIZE - 1)))
357−#define pmd_page(pmd) \
358− pfn_to_page((__pa(pmd_val(pmd)) >> PAGE_SHIFT))
356+#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT)
359357 #endif
360358
359+#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
361360 /*
362361 * Encode and decode a swap entry.
363362 * Note that the bits we use in a PTE for representing a swap entry
arch/powerpc/include/asm/nohash/64/pgtable.h+1−0
@@ -142,6 +142,7 @@ static inline pte_t pmd_pte(pmd_t pmd) · 141 unmodified lines
142142 #define pmd_present(pmd) (!pmd_none(pmd))
143143 #define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS)
144144 extern struct page *pmd_page(pmd_t pmd);
145+#define pmd_pfn(pmd) (page_to_pfn(pmd_page(pmd)))
145146
146147 static inline void pud_set(pud_t *pudp, unsigned long val)
147148 {
arch/powerpc/kvm/book3s_hv_uvmem.c+1−1
@@ -91,6 +91,7 @@ · 90 unmodified lines
9191 #include <linux/kvm_host.h>
9292 #include <linux/ksm.h>
9393 #include <linux/of.h>
94+#include <linux/memremap.h>
9495 #include <asm/ultravisor.h>
9596 #include <asm/mman.h>
9697 #include <asm/kvm_ppc.h>
@@ -712,7 +713,6 @@ static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm) · 615 unmodified lines
712713
713714 dpage = pfn_to_page(uvmem_pfn);
714715 dpage->zone_device_data = pvt;
715− get_page(dpage);
716716 lock_page(dpage);
717717 return dpage;
718718 out_clear:
arch/powerpc/mm/book3s64/pgtable.c+1−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #include <linux/sched.h>
77 #include <linux/mm_types.h>
88 #include <linux/memblock.h>
9+#include <linux/memremap.h>
910 #include <linux/debugfs.h>
1011 #include <misc/cxl-base.h>
1112
arch/sh/include/asm/pgtable_32.h+1−0
@@ -406,6 +406,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 405 unmodified lines
406406 return (unsigned long)pmd_val(pmd);
407407 }
408408
409+#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT)
409410 #define pmd_page(pmd) (virt_to_page(pmd_val(pmd)))
410411
411412 #ifdef CONFIG_X2TLB
arch/sparc/include/asm/pgtable_32.h+6−1
@@ -127,11 +127,16 @@ static inline int srmmu_device_memory(unsigned long x) · 126 unmodified lines
127127 return ((x & 0xF0000000) != 0);
128128 }
129129
130+static inline unsigned long pmd_pfn(pmd_t pmd)
131+{
132+ return (pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4);
133+}
134+
130135 static inline struct page *pmd_page(pmd_t pmd)
131136 {
132137 if (srmmu_device_memory(pmd_val(pmd)))
133138 BUG();
134− return pfn_to_page((pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4));
139+ return pfn_to_page(pmd_pfn(pmd));
135140 }
136141
137142 static inline unsigned long __pmd_page(pmd_t pmd)
arch/um/include/asm/pgtable.h+1−0
@@ -109,6 +109,7 @@ extern unsigned long end_iomem; · 108 unmodified lines
109109 #define p4d_newpage(x) (p4d_val(x) & _PAGE_NEWPAGE)
110110 #define p4d_mkuptodate(x) (p4d_val(x) &= ~_PAGE_NEWPAGE)
111111
112+#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
112113 #define pmd_page(pmd) phys_to_page(pmd_val(pmd) & PAGE_MASK)
113114
114115 #define pte_page(x) pfn_to_page(pte_pfn(x))
arch/xtensa/include/asm/pgtable.h+1−0
@@ -241,6 +241,7 @@ static inline void paging_init(void) { } · 240 unmodified lines
241241 * The pmd contains the kernel virtual address of the pte page.
242242 */
243243 #define pmd_page_vaddr(pmd) ((unsigned long)(pmd_val(pmd) & PAGE_MASK))
244+#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT)
244245 #define pmd_page(pmd) virt_to_page(pmd_val(pmd))
245246
246247 /*
drivers/block/xen-blkback/xenbus.c+1−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010
1111 #include <linux/module.h>
1212 #include <linux/kthread.h>
13+#include <linux/pagemap.h>
1314 #include <xen/events.h>
1415 #include <xen/grant_table.h>
1516 #include "common.h"
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c+1−1
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424 #include <linux/hmm.h>
2525 #include <linux/dma-direction.h>
2626 #include <linux/dma-mapping.h>
27+#include <linux/migrate.h>
2728 #include "amdgpu_sync.h"
2829 #include "amdgpu_object.h"
2930 #include "amdgpu_vm.h"
@@ -224,7 +225,6 @@ svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn) · 194 unmodified lines
224225 page = pfn_to_page(pfn);
225226 svm_range_bo_ref(prange->svm_bo);
226227 page->zone_device_data = prange->svm_bo;
227− get_page(page);
228228 lock_page(page);
229229 }
230230
drivers/gpu/drm/amd/amdkfd/kfd_priv.h+1−0
@@ -25,6 +25,7 @@ · 24 unmodified lines
2525
2626 #include <linux/hashtable.h>
2727 #include <linux/mmu_notifier.h>
28+#include <linux/memremap.h>
2829 #include <linux/mutex.h>
2930 #include <linux/types.h>
3031 #include <linux/atomic.h>
drivers/gpu/drm/drm_cache.c+1−1
@@ -27,11 +27,11 @@ · 26 unmodified lines
2727 /*
2828 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
2929 */
30−
3130 #include <linux/dma-buf-map.h>
3231 #include <linux/export.h>
3332 #include <linux/highmem.h>
3433 #include <linux/cc_platform.h>
34+#include <linux/ioport.h>
3535 #include <xen/xen.h>
3636
3737 #include <drm/drm_cache.h>
drivers/gpu/drm/nouveau/nouveau_dmem.c+2−1
@@ -39,6 +39,8 @@ · 38 unmodified lines
3939
4040 #include <linux/sched/mm.h>
4141 #include <linux/hmm.h>
42+#include <linux/memremap.h>
43+#include <linux/migrate.h>
4244
4345 /*
4446 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
@@ -324,7 +326,6 @@ nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) · 279 unmodified lines
324326 return NULL;
325327 }
326328
327− get_page(page);
328329 lock_page(page);
329330 return page;
330331 }
drivers/gpu/drm/nouveau/nouveau_svm.c+1−0
@@ -35,6 +35,7 @@ · 34 unmodified lines
3535 #include <linux/sched/mm.h>
3636 #include <linux/sort.h>
3737 #include <linux/hmm.h>
38+#include <linux/memremap.h>
3839 #include <linux/rmap.h>
3940
4041 struct nouveau_svm {
drivers/infiniband/core/rw.c+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
22 /*
33 * Copyright (c) 2016 HGST, a Western Digital Company.
44 */
5+#include <linux/memremap.h>
56 #include <linux/moduleparam.h>
67 #include <linux/slab.h>
78 #include <linux/pci-p2pdma.h>
drivers/nvdimm/pmem.h+1−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
33 #define __NVDIMM_PMEM_H__
44 #include <linux/page-flags.h>
55 #include <linux/badblocks.h>
6+#include <linux/memremap.h>
67 #include <linux/types.h>
78 #include <linux/pfn_t.h>
89 #include <linux/fs.h>
drivers/nvme/host/pci.c+1−0
@@ -15,6 +15,7 @@ · 14 unmodified lines
1515 #include <linux/init.h>
1616 #include <linux/interrupt.h>
1717 #include <linux/io.h>
18+#include <linux/memremap.h>
1819 #include <linux/mm.h>
1920 #include <linux/module.h>
2021 #include <linux/mutex.h>
drivers/nvme/target/io-cmd-bdev.c+1−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
77 #include <linux/blkdev.h>
88 #include <linux/blk-integrity.h>
9+#include <linux/memremap.h>
910 #include <linux/module.h>
1011 #include "nvmet.h"
1112
drivers/usb/gadget/function/f_mass_storage.c+1−0
@@ -179,6 +179,7 @@ · 178 unmodified lines
179179 #include <linux/kthread.h>
180180 #include <linux/sched/signal.h>
181181 #include <linux/limits.h>
182+#include <linux/pagemap.h>
182183 #include <linux/rwsem.h>
183184 #include <linux/slab.h>
184185 #include <linux/spinlock.h>
fs/Kconfig+1−1
@@ -48,7 +48,7 @@ config FS_DAX · 47 unmodified lines
4848 bool "File system based Direct Access (DAX) support"
4949 depends on MMU
5050 depends on !(ARM || MIPS || SPARC)
51− select DEV_PAGEMAP_OPS if (ZONE_DEVICE && !FS_DAX_LIMITED)
51+ depends on ZONE_DEVICE || FS_DAX_LIMITED
5252 select FS_IOMAP
5353 select DAX
5454 help
fs/coda/file.c+1−0
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <linux/time.h>
1515 #include <linux/file.h>
1616 #include <linux/fs.h>
17+#include <linux/pagemap.h>
1718 #include <linux/stat.h>
1819 #include <linux/cred.h>
1920 #include <linux/errno.h>
fs/fuse/virtio_fs.c+1−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include <linux/dax.h>
99 #include <linux/pci.h>
1010 #include <linux/pfn_t.h>
11+#include <linux/memremap.h>
1112 #include <linux/module.h>
1213 #include <linux/virtio.h>
1314 #include <linux/virtio_fs.h>
fs/iomap/fiemap.c+1−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include <linux/fs.h>
88 #include <linux/iomap.h>
99 #include <linux/fiemap.h>
10+#include <linux/pagemap.h>
1011
1112 static int iomap_to_fiemap(struct fiemap_extent_info *fi,
1213 const struct iomap *iomap, u32 flags)
fs/nfsd/filecache.c+1−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include <linux/hash.h>
88 #include <linux/slab.h>
99 #include <linux/file.h>
10+#include <linux/pagemap.h>
1011 #include <linux/sched.h>
1112 #include <linux/list_lru.h>
1213 #include <linux/fsnotify_backend.h>
fs/nfsd/vfs.c+1−0
@@ -26,6 +26,7 @@ · 25 unmodified lines
2626 #include <linux/xattr.h>
2727 #include <linux/jhash.h>
2828 #include <linux/ima.h>
29+#include <linux/pagemap.h>
2930 #include <linux/slab.h>
3031 #include <linux/uaccess.h>
3132 #include <linux/exportfs.h>
fs/proc/page.c+1−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010 #include <linux/proc_fs.h>
1111 #include <linux/seq_file.h>
1212 #include <linux/hugetlb.h>
13+#include <linux/memremap.h>
1314 #include <linux/memcontrol.h>
1415 #include <linux/mmu_notifier.h>
1516 #include <linux/page_idle.h>
fs/splice.c+12−12
@@ -46,45 +46,45 @@ · 45 unmodified lines
4646 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
4747 struct pipe_buffer *buf)
4848 {
49− struct page *page = buf->page;
49+ struct folio *folio = page_folio(buf->page);
5050 struct address_space *mapping;
5151
52− lock_page(page);
52+ folio_lock(folio);
5353
54− mapping = page_mapping(page);
54+ mapping = folio_mapping(folio);
5555 if (mapping) {
56− WARN_ON(!PageUptodate(page));
56+ WARN_ON(!folio_test_uptodate(folio));
5757
5858 /*
5959 * At least for ext2 with nobh option, we need to wait on
60− * writeback completing on this page, since we'll remove it
60+ * writeback completing on this folio, since we'll remove it
6161 * from the pagecache. Otherwise truncate wont wait on the
62− * page, allowing the disk blocks to be reused by someone else
62+ * folio, allowing the disk blocks to be reused by someone else
6363 * before we actually wrote our data to them. fs corruption
6464 * ensues.
6565 */
66− wait_on_page_writeback(page);
66+ folio_wait_writeback(folio);
6767
68− if (page_has_private(page) &&
69− !try_to_release_page(page, GFP_KERNEL))
68+ if (folio_has_private(folio) &&
69+ !filemap_release_folio(folio, GFP_KERNEL))
7070 goto out_unlock;
7171
7272 /*
7373 * If we succeeded in removing the mapping, set LRU flag
7474 * and return good.
7575 */
76− if (remove_mapping(mapping, page)) {
76+ if (remove_mapping(mapping, folio)) {
7777 buf->flags |= PIPE_BUF_FLAG_LRU;
7878 return true;
7979 }
8080 }
8181
8282 /*
83− * Raced with truncate or failed to remove page from current
83+ * Raced with truncate or failed to remove folio from current
8484 * address space, unlock and return failure.
8585 */
8686 out_unlock:
87− unlock_page(page);
87+ folio_unlock(folio);
8888 return false;
8989 }
9090
fs/vboxsf/utils.c+1−0
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include <linux/namei.h>
1010 #include <linux/nls.h>
1111 #include <linux/sizes.h>
12+#include <linux/pagemap.h>
1213 #include <linux/vfs.h>
1314 #include "vfsmod.h"
1415
include/linux/fs.h+0−120
@@ -2753,54 +2753,6 @@ extern void init_special_inode(struct inode *, umode_t, dev_t); · 2752 unmodified lines
27532753 extern void make_bad_inode(struct inode *);
27542754 extern bool is_bad_inode(struct inode *);
27552755
2756−unsigned long invalidate_mapping_pages(struct address_space *mapping,
2757− pgoff_t start, pgoff_t end);
2758−
2759−void invalidate_mapping_pagevec(struct address_space *mapping,
2760− pgoff_t start, pgoff_t end,
2761− unsigned long *nr_pagevec);
2762−
2763−static inline void invalidate_remote_inode(struct inode *inode)
2764−{
2765− if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
2766− S_ISLNK(inode->i_mode))
2767− invalidate_mapping_pages(inode->i_mapping, 0, -1);
2768−}
2769−extern int invalidate_inode_pages2(struct address_space *mapping);
2770−extern int invalidate_inode_pages2_range(struct address_space *mapping,
2771− pgoff_t start, pgoff_t end);
2772−extern int write_inode_now(struct inode *, int);
2773−extern int filemap_fdatawrite(struct address_space *);
2774−extern int filemap_flush(struct address_space *);
2775−extern int filemap_fdatawait_keep_errors(struct address_space *mapping);
2776−extern int filemap_fdatawait_range(struct address_space *, loff_t lstart,
2777− loff_t lend);
2778−extern int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
2779− loff_t start_byte, loff_t end_byte);
2780−
2781−static inline int filemap_fdatawait(struct address_space *mapping)
2782−{
2783− return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
2784−}
2785−
2786−extern bool filemap_range_has_page(struct address_space *, loff_t lstart,
2787− loff_t lend);
2788−extern int filemap_write_and_wait_range(struct address_space *mapping,
2789− loff_t lstart, loff_t lend);
2790−extern int __filemap_fdatawrite_range(struct address_space *mapping,
2791− loff_t start, loff_t end, int sync_mode);
2792−extern int filemap_fdatawrite_range(struct address_space *mapping,
2793− loff_t start, loff_t end);
2794−extern int filemap_check_errors(struct address_space *mapping);
2795−extern void __filemap_set_wb_err(struct address_space *mapping, int err);
2796−int filemap_fdatawrite_wbc(struct address_space *mapping,
2797− struct writeback_control *wbc);
2798−
2799−static inline int filemap_write_and_wait(struct address_space *mapping)
2800−{
2801− return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
2802−}
2803−
28042756 extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,
28052757 loff_t lend);
28062758 extern int __must_check file_check_and_advance_wb_err(struct file *file);
@@ -2812,67 +2764,6 @@ static inline int file_write_and_wait(struct file *file) · 5 unmodified lines
28122764 return file_write_and_wait_range(file, 0, LLONG_MAX);
28132765 }
28142766
2815−/**
2816− * filemap_set_wb_err - set a writeback error on an address_space
2817− * @mapping: mapping in which to set writeback error
2818− * @err: error to be set in mapping
2819− *
2820− * When writeback fails in some way, we must record that error so that
2821− * userspace can be informed when fsync and the like are called. We endeavor
2822− * to report errors on any file that was open at the time of the error. Some
2823− * internal callers also need to know when writeback errors have occurred.
2824− *
2825− * When a writeback error occurs, most filesystems will want to call
2826− * filemap_set_wb_err to record the error in the mapping so that it will be
2827− * automatically reported whenever fsync is called on the file.
2828− */
2829−static inline void filemap_set_wb_err(struct address_space *mapping, int err)
2830−{
2831− /* Fastpath for common case of no error */
2832− if (unlikely(err))
2833− __filemap_set_wb_err(mapping, err);
2834−}
2835−
2836−/**
2837− * filemap_check_wb_err - has an error occurred since the mark was sampled?
2838− * @mapping: mapping to check for writeback errors
2839− * @since: previously-sampled errseq_t
2840− *
2841− * Grab the errseq_t value from the mapping, and see if it has changed "since"
2842− * the given value was sampled.
2843− *
2844− * If it has then report the latest error set, otherwise return 0.
2845− */
2846−static inline int filemap_check_wb_err(struct address_space *mapping,
2847− errseq_t since)
2848−{
2849− return errseq_check(&mapping->wb_err, since);
2850−}
2851−
2852−/**
2853− * filemap_sample_wb_err - sample the current errseq_t to test for later errors
2854− * @mapping: mapping to be sampled
2855− *
2856− * Writeback errors are always reported relative to a particular sample point
2857− * in the past. This function provides those sample points.
2858− */
2859−static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
2860−{
2861− return errseq_sample(&mapping->wb_err);
2862−}
2863−
2864−/**
2865− * file_sample_sb_err - sample the current errseq_t to test for later errors
2866− * @file: file pointer to be sampled
2867− *
2868− * Grab the most current superblock-level errseq_t value for the given
2869− * struct file.
2870− */
2871−static inline errseq_t file_sample_sb_err(struct file *file)
2872−{
2873− return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
2874−}
2875−
28762767 extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,
28772768 int datasync);
28782769 extern int vfs_fsync(struct file *file, int datasync);
@@ -3627,15 +3518,4 @@ extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len, · 748 unmodified lines
36273518 extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
36283519 int advice);
36293520
3630−/*
3631− * Flush file data before changing attributes. Caller must hold any locks
3632− * required to prevent further writes to this file until we're done setting
3633− * flags.
3634− */
3635−static inline int inode_drain_writes(struct inode *inode)
3636−{
3637− inode_dio_wait(inode);
3638− return filemap_write_and_wait(inode->i_mapping);
3639−}
3640−
36413521 #endif /* _LINUX_FS_H */
include/linux/hmm.h+2−7
@@ -9,14 +9,9 @@ · 8 unmodified lines
99 #ifndef LINUX_HMM_H
1010 #define LINUX_HMM_H
1111
12−#include <linux/kconfig.h>
13−#include <linux/pgtable.h>
12+#include <linux/mm.h>
1413
15−#include <linux/device.h>
16−#include <linux/migrate.h>
17−#include <linux/memremap.h>
18−#include <linux/completion.h>
19−#include <linux/mmu_notifier.h>
14+struct mmu_interval_notifier;
2015
2116 /*
2217 * On output:
include/linux/huge_mm.h+9−50
@@ -185,7 +185,7 @@ void prep_transhuge_page(struct page *page); · 184 unmodified lines
185185 void free_transhuge_page(struct page *page);
186186 bool is_transparent_hugepage(struct page *page);
187187
188−bool can_split_huge_page(struct page *page, int *pextra_pins);
188+bool can_split_folio(struct folio *folio, int *pextra_pins);
189189 int split_huge_page_to_list(struct page *page, struct list_head *list);
190190 static inline int split_huge_page(struct page *page)
191191 {
@@ -194,7 +194,7 @@ static inline int split_huge_page(struct page *page) · 2 unmodified lines
194194 void deferred_split_huge_page(struct page *page);
195195
196196 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
197− unsigned long address, bool freeze, struct page *page);
197+ unsigned long address, bool freeze, struct folio *folio);
198198
199199 #define split_huge_pmd(__vma, __pmd, __address) \
200200 do { \
@@ -207,7 +207,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, · 6 unmodified lines
207207
208208
209209 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
210− bool freeze, struct page *page);
210+ bool freeze, struct folio *folio);
211211
212212 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
213213 unsigned long address);
@@ -250,30 +250,6 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, · 36 unmodified lines
250250 return NULL;
251251 }
252252
253−/**
254− * thp_order - Order of a transparent huge page.
255− * @page: Head page of a transparent huge page.
256− */
257−static inline unsigned int thp_order(struct page *page)
258−{
259− VM_BUG_ON_PGFLAGS(PageTail(page), page);
260− if (PageHead(page))
261− return HPAGE_PMD_ORDER;
262− return 0;
263−}
264−
265−/**
266− * thp_nr_pages - The number of regular pages in this huge page.
267− * @page: The head page of a huge page.
268− */
269−static inline int thp_nr_pages(struct page *page)
270−{
271− VM_BUG_ON_PGFLAGS(PageTail(page), page);
272− if (PageHead(page))
273− return HPAGE_PMD_NR;
274− return 1;
275−}
276−
277253 /**
278254 * folio_test_pmd_mappable - Can we map this folio with a PMD?
279255 * @folio: The folio to test
@@ -336,18 +312,6 @@ static inline struct list_head *page_deferred_list(struct page *page) · 56 unmodified lines
336312 #define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
337313 #define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
338314
339−static inline unsigned int thp_order(struct page *page)
340−{
341− VM_BUG_ON_PGFLAGS(PageTail(page), page);
342− return 0;
343−}
344−
345−static inline int thp_nr_pages(struct page *page)
346−{
347− VM_BUG_ON_PGFLAGS(PageTail(page), page);
348− return 1;
349−}
350−
351315 static inline bool folio_test_pmd_mappable(struct folio *folio)
352316 {
353317 return false;
@@ -387,7 +351,7 @@ static inline bool is_transparent_hugepage(struct page *page) · 33 unmodified lines
387351 #define thp_get_unmapped_area NULL
388352
389353 static inline bool
390−can_split_huge_page(struct page *page, int *pextra_pins)
354+can_split_folio(struct folio *folio, int *pextra_pins)
391355 {
392356 BUILD_BUG();
393357 return false;
@@ -406,9 +370,9 @@ static inline void deferred_split_huge_page(struct page *page) {} · 12 unmodified lines
406370 do { } while (0)
407371
408372 static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
409− unsigned long address, bool freeze, struct page *page) {}
373+ unsigned long address, bool freeze, struct folio *folio) {}
410374 static inline void split_huge_pmd_address(struct vm_area_struct *vma,
411− unsigned long address, bool freeze, struct page *page) {}
375+ unsigned long address, bool freeze, struct folio *folio) {}
412376
413377 #define split_huge_pud(__vma, __pmd, __address) \
414378 do { } while (0)
@@ -483,15 +447,10 @@ static inline bool thp_migration_supported(void) · 68 unmodified lines
483447 }
484448 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
485449
486−/**
487− * thp_size - Size of a transparent huge page.
488− * @page: Head page of a transparent huge page.
489− *
490− * Return: Number of bytes in this page.
491− */
492−static inline unsigned long thp_size(struct page *page)
450+static inline int split_folio_to_list(struct folio *folio,
451+ struct list_head *list)
493452 {
494− return PAGE_SIZE << thp_order(page);
453+ return split_huge_page_to_list(&folio->page, list);
495454 }
496455
497456 #endif /* _LINUX_HUGE_MM_H */
include/linux/hugetlb.h+5−0
@@ -970,6 +970,11 @@ static inline struct hstate *page_hstate(struct page *page) · 969 unmodified lines
970970 return NULL;
971971 }
972972
973+static inline struct hstate *size_to_hstate(unsigned long size)
974+{
975+ return NULL;
976+}
977+
973978 static inline unsigned long huge_page_size(struct hstate *h)
974979 {
975980 return PAGE_SIZE;
include/linux/ksm.h+3−3
@@ -51,7 +51,7 @@ static inline void ksm_exit(struct mm_struct *mm) · 50 unmodified lines
5151 struct page *ksm_might_need_to_copy(struct page *page,
5252 struct vm_area_struct *vma, unsigned long address);
5353
54−void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);
54+void rmap_walk_ksm(struct folio *folio, const struct rmap_walk_control *rwc);
5555 void folio_migrate_ksm(struct folio *newfolio, struct folio *folio);
5656
5757 #else /* !CONFIG_KSM */
@@ -78,8 +78,8 @@ static inline struct page *ksm_might_need_to_copy(struct page *page, · 20 unmodified lines
7878 return page;
7979 }
8080
81−static inline void rmap_walk_ksm(struct page *page,
82− struct rmap_walk_control *rwc)
81+static inline void rmap_walk_ksm(struct folio *folio,
82+ const struct rmap_walk_control *rwc)
8383 {
8484 }
8585
include/linux/memremap.h+24−3
@@ -1,6 +1,8 @@
11 /* SPDX-License-Identifier: GPL-2.0 */
22 #ifndef _LINUX_MEMREMAP_H_
33 #define _LINUX_MEMREMAP_H_
4+
5+#include <linux/mm.h>
46 #include <linux/range.h>
57 #include <linux/ioport.h>
68 #include <linux/percpu-refcount.h>
@@ -66,9 +68,9 @@ enum memory_type { · 59 unmodified lines
6668
6769 struct dev_pagemap_ops {
6870 /*
69− * Called once the page refcount reaches 1. (ZONE_DEVICE pages never
70− * reach 0 refcount unless there is a refcount bug. This allows the
71− * device driver to implement its own memory management.)
71+ * Called once the page refcount reaches 0. The reference count will be
72+ * reset to one by the core code after the method is called to prepare
73+ * for handing out the page again.
7274 */
7375 void (*page_free)(struct page *page);
7476
@@ -129,6 +131,25 @@ static inline unsigned long pgmap_vmemmap_nr(struct dev_pagemap *pgmap) · 54 unmodified lines
129131 return 1 << pgmap->vmemmap_shift;
130132 }
131133
134+static inline bool is_device_private_page(const struct page *page)
135+{
136+ return IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
137+ is_zone_device_page(page) &&
138+ page->pgmap->type == MEMORY_DEVICE_PRIVATE;
139+}
140+
141+static inline bool folio_is_device_private(const struct folio *folio)
142+{
143+ return is_device_private_page(&folio->page);
144+}
145+
146+static inline bool is_pci_p2pdma_page(const struct page *page)
147+{
148+ return IS_ENABLED(CONFIG_PCI_P2PDMA) &&
149+ is_zone_device_page(page) &&
150+ page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
151+}
152+
132153 #ifdef CONFIG_ZONE_DEVICE
133154 void *memremap_pages(struct dev_pagemap *pgmap, int nid);
134155 void memunmap_pages(struct dev_pagemap *pgmap);
include/linux/mm.h+152−144
@@ -3,9 +3,6 @@ · 2 unmodified lines
33 #define _LINUX_MM_H
44
55 #include <linux/errno.h>
6−
7−#ifdef __KERNEL__
8−
96 #include <linux/mmdebug.h>
107 #include <linux/gfp.h>
118 #include <linux/bug.h>
@@ -26,7 +23,6 @@ · 14 unmodified lines
2623 #include <linux/err.h>
2724 #include <linux/page-flags.h>
2825 #include <linux/page_ref.h>
29−#include <linux/memremap.h>
3026 #include <linux/overflow.h>
3127 #include <linux/sizes.h>
3228 #include <linux/sched.h>
@@ -216,8 +212,10 @@ int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, · 183 unmodified lines
216212
217213 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
218214 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
215+#define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio))
219216 #else
220217 #define nth_page(page,n) ((page) + (n))
218+#define folio_page_idx(folio, p) ((p) - &(folio)->page)
221219 #endif
222220
223221 /* to align the pointer to the (next) page boundary */
@@ -227,6 +225,10 @@ int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, · 3 unmodified lines
227225 #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
228226
229227 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
228+static inline struct folio *lru_to_folio(struct list_head *head)
229+{
230+ return list_entry((head)->prev, struct folio, lru);
231+}
230232
231233 void setup_initial_init_mm(void *start_code, void *end_code,
232234 void *end_data, void *brk);
@@ -775,21 +777,26 @@ static inline int is_vmalloc_or_module_addr(const void *x) · 542 unmodified lines
775777 }
776778 #endif
777779
778−static inline int head_compound_mapcount(struct page *head)
780+/*
781+ * How many times the entire folio is mapped as a single unit (eg by a
782+ * PMD or PUD entry). This is probably not what you want, except for
783+ * debugging purposes; look at folio_mapcount() or page_mapcount()
784+ * instead.
785+ */
786+static inline int folio_entire_mapcount(struct folio *folio)
779787 {
780− return atomic_read(compound_mapcount_ptr(head)) + 1;
788+ VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
789+ return atomic_read(folio_mapcount_ptr(folio)) + 1;
781790 }
782791
783792 /*
784793 * Mapcount of compound page as a whole, does not include mapped sub-pages.
785794 *
786− * Must be called only for compound pages or any their tail sub-pages.
795+ * Must be called only for compound pages.
787796 */
788797 static inline int compound_mapcount(struct page *page)
789798 {
790− VM_BUG_ON_PAGE(!PageCompound(page), page);
791− page = compound_head(page);
792− return head_compound_mapcount(page);
799+ return folio_entire_mapcount(page_folio(page));
793800 }
794801
795802 /*
@@ -819,8 +826,14 @@ static inline int page_mapcount(struct page *page) · 23 unmodified lines
819826 return atomic_read(&page->_mapcount) + 1;
820827 }
821828
829+int folio_mapcount(struct folio *folio);
830+
822831 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
823−int total_mapcount(struct page *page);
832+static inline int total_mapcount(struct page *page)
833+{
834+ return folio_mapcount(page_folio(page));
835+}
836+
824837 int page_trans_huge_mapcount(struct page *page);
825838 #else
826839 static inline int total_mapcount(struct page *page)
@@ -890,33 +903,17 @@ static inline void destroy_compound_page(struct page *page) · 63 unmodified lines
890903 compound_page_dtors[page[1].compound_dtor](page);
891904 }
892905
893−static inline bool hpage_pincount_available(struct page *page)
894−{
895− /*
896− * Can the page->hpage_pinned_refcount field be used? That field is in
897− * the 3rd page of the compound page, so the smallest (2-page) compound
898− * pages cannot support it.
899− */
900− page = compound_head(page);
901− return PageCompound(page) && compound_order(page) > 1;
902−}
903−
904906 static inline int head_compound_pincount(struct page *head)
905907 {
906908 return atomic_read(compound_pincount_ptr(head));
907909 }
908910
909−static inline int compound_pincount(struct page *page)
910−{
911− VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
912− page = compound_head(page);
913− return head_compound_pincount(page);
914−}
915−
916911 static inline void set_compound_order(struct page *page, unsigned int order)
917912 {
918913 page[1].compound_order = order;
914+#ifdef CONFIG_64BIT
919915 page[1].compound_nr = 1U << order;
916+#endif
920917 }
921918
922919 /* Returns the number of pages in this potentially compound page. */
@@ -924,7 +921,11 @@ static inline unsigned long compound_nr(struct page *page) · 1 unmodified line
924921 {
925922 if (!PageHead(page))
926923 return 1;
924+#ifdef CONFIG_64BIT
927925 return page[1].compound_nr;
926+#else
927+ return 1UL << compound_order(page);
928+#endif
928929 }
929930
930931 /* Returns the number of bytes in this potentially compound page. */
@@ -939,6 +940,37 @@ static inline unsigned int page_shift(struct page *page) · 8 unmodified lines
939940 return PAGE_SHIFT + compound_order(page);
940941 }
941942
943+/**
944+ * thp_order - Order of a transparent huge page.
945+ * @page: Head page of a transparent huge page.
946+ */
947+static inline unsigned int thp_order(struct page *page)
948+{
949+ VM_BUG_ON_PGFLAGS(PageTail(page), page);
950+ return compound_order(page);
951+}
952+
953+/**
954+ * thp_nr_pages - The number of regular pages in this huge page.
955+ * @page: The head page of a huge page.
956+ */
957+static inline int thp_nr_pages(struct page *page)
958+{
959+ VM_BUG_ON_PGFLAGS(PageTail(page), page);
960+ return compound_nr(page);
961+}
962+
963+/**
964+ * thp_size - Size of a transparent huge page.
965+ * @page: Head page of a transparent huge page.
966+ *
967+ * Return: Number of bytes in this page.
968+ */
969+static inline unsigned long thp_size(struct page *page)
970+{
971+ return PAGE_SIZE << thp_order(page);
972+}
973+
942974 void free_compound_page(struct page *page);
943975
944976 #ifdef CONFIG_MMU
@@ -1090,59 +1122,35 @@ static inline bool is_zone_device_page(const struct page *page) · 145 unmodified lines
10901122 }
10911123 #endif
10921124
1125+static inline bool folio_is_zone_device(const struct folio *folio)
1126+{
1127+ return is_zone_device_page(&folio->page);
1128+}
1129+
10931130 static inline bool is_zone_movable_page(const struct page *page)
10941131 {
10951132 return page_zonenum(page) == ZONE_MOVABLE;
10961133 }
10971134
1098−#ifdef CONFIG_DEV_PAGEMAP_OPS
1099−void free_devmap_managed_page(struct page *page);
1135+#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX)
11001136 DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
11011137
1102−static inline bool page_is_devmap_managed(struct page *page)
1138+bool __put_devmap_managed_page(struct page *page);
1139+static inline bool put_devmap_managed_page(struct page *page)
11031140 {
11041141 if (!static_branch_unlikely(&devmap_managed_key))
11051142 return false;
11061143 if (!is_zone_device_page(page))
11071144 return false;
1108− switch (page->pgmap->type) {
1109− case MEMORY_DEVICE_PRIVATE:
1110− case MEMORY_DEVICE_FS_DAX:
1111− return true;
1112− default:
1113− break;
1114− }
1115− return false;
1145+ return __put_devmap_managed_page(page);
11161146 }
11171147
1118−void put_devmap_managed_page(struct page *page);
1119−
1120−#else /* CONFIG_DEV_PAGEMAP_OPS */
1121−static inline bool page_is_devmap_managed(struct page *page)
1148+#else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
1149+static inline bool put_devmap_managed_page(struct page *page)
11221150 {
11231151 return false;
11241152 }
1125−
1126−static inline void put_devmap_managed_page(struct page *page)
1127−{
1128−}
1129−#endif /* CONFIG_DEV_PAGEMAP_OPS */
1130−
1131−static inline bool is_device_private_page(const struct page *page)
1132−{
1133− return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
1134− IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
1135− is_zone_device_page(page) &&
1136− page->pgmap->type == MEMORY_DEVICE_PRIVATE;
1137−}
1138−
1139−static inline bool is_pci_p2pdma_page(const struct page *page)
1140−{
1141− return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
1142− IS_ENABLED(CONFIG_PCI_P2PDMA) &&
1143− is_zone_device_page(page) &&
1144− page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
1145−}
1153+#endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
11461154
11471155 /* 127: arbitrary random number, small enough to assemble well */
11481156 #define folio_ref_zero_or_close_to_overflow(folio) \
@@ -1168,9 +1176,6 @@ static inline void get_page(struct page *page) · 19 unmodified lines
11681176 }
11691177
11701178 bool __must_check try_grab_page(struct page *page, unsigned int flags);
1171−struct page *try_grab_compound_head(struct page *page, int refs,
1172− unsigned int flags);
1173−
11741179
11751180 static inline __must_check bool try_get_page(struct page *page)
11761181 {
@@ -1225,16 +1230,11 @@ static inline void put_page(struct page *page) · 48 unmodified lines
12251230 struct folio *folio = page_folio(page);
12261231
12271232 /*
1228− * For devmap managed pages we need to catch refcount transition from
1229− * 2 to 1, when refcount reach one it means the page is free and we
1230− * need to inform the device driver through callback. See
1231− * include/linux/memremap.h and HMM for details.
1233+ * For some devmap managed pages we need to catch refcount transition
1234+ * from 2 to 1:
12321235 */
1233− if (page_is_devmap_managed(&folio->page)) {
1234− put_devmap_managed_page(&folio->page);
1236+ if (put_devmap_managed_page(&folio->page))
12351237 return;
1236− }
1237−
12381238 folio_put(folio);
12391239 }
12401240
@@ -1264,10 +1264,9 @@ static inline void put_page(struct page *page) · 23 unmodified lines
12641264 * applications that don't have huge page reference counts, this won't be an
12651265 * issue.
12661266 *
1267− * Locking: the lockless algorithm described in page_cache_get_speculative()
1268− * and page_cache_gup_pin_speculative() provides safe operation for
1269− * get_user_pages and page_mkclean and other calls that race to set up page
1270− * table entries.
1267+ * Locking: the lockless algorithm described in folio_try_get_rcu()
1268+ * provides safe operation for get_user_pages(), page_mkclean() and
1269+ * other calls that race to set up page table entries.
12711270 */
12721271 #define GUP_PIN_COUNTING_BIAS (1U << 10)
12731272
@@ -1278,70 +1277,11 @@ void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, · 4 unmodified lines
12781277 bool make_dirty);
12791278 void unpin_user_pages(struct page **pages, unsigned long npages);
12801279
1281−/**
1282− * page_maybe_dma_pinned - Report if a page is pinned for DMA.
1283− * @page: The page.
1284− *
1285− * This function checks if a page has been pinned via a call to
1286− * a function in the pin_user_pages() family.
1287− *
1288− * For non-huge pages, the return value is partially fuzzy: false is not fuzzy,
1289− * because it means "definitely not pinned for DMA", but true means "probably
1290− * pinned for DMA, but possibly a false positive due to having at least
1291− * GUP_PIN_COUNTING_BIAS worth of normal page references".
1292− *
1293− * False positives are OK, because: a) it's unlikely for a page to get that many
1294− * refcounts, and b) all the callers of this routine are expected to be able to
1295− * deal gracefully with a false positive.
1296− *
1297− * For huge pages, the result will be exactly correct. That's because we have
1298− * more tracking data available: the 3rd struct page in the compound page is
1299− * used to track the pincount (instead using of the GUP_PIN_COUNTING_BIAS
1300− * scheme).
1301− *
1302− * For more information, please see Documentation/core-api/pin_user_pages.rst.
1303− *
1304− * Return: True, if it is likely that the page has been "dma-pinned".
1305− * False, if the page is definitely not dma-pinned.
1306− */
1307−static inline bool page_maybe_dma_pinned(struct page *page)
1308−{
1309− if (hpage_pincount_available(page))
1310− return compound_pincount(page) > 0;
1311−
1312− /*
1313− * page_ref_count() is signed. If that refcount overflows, then
1314− * page_ref_count() returns a negative value, and callers will avoid
1315− * further incrementing the refcount.
1316− *
1317− * Here, for that overflow case, use the signed bit to count a little
1318− * bit higher via unsigned math, and thus still get an accurate result.
1319− */
1320− return ((unsigned int)page_ref_count(compound_head(page))) >=
1321− GUP_PIN_COUNTING_BIAS;
1322−}
1323−
13241280 static inline bool is_cow_mapping(vm_flags_t flags)
13251281 {
13261282 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
13271283 }
13281284
1329−/*
1330− * This should most likely only be called during fork() to see whether we
1331− * should break the cow immediately for a page on the src mm.
1332− */
1333−static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma,
1334− struct page *page)
1335−{
1336− if (!is_cow_mapping(vma->vm_flags))
1337− return false;
1338−
1339− if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
1340− return false;
1341−
1342− return page_maybe_dma_pinned(page);
1343−}
1344−
13451285 #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
13461286 #define SECTION_IN_PAGE_FLAGS
13471287 #endif
@@ -1586,6 +1526,74 @@ static inline unsigned long folio_pfn(struct folio *folio) · 238 unmodified lines
15861526 return page_to_pfn(&folio->page);
15871527 }
15881528
1529+static inline atomic_t *folio_pincount_ptr(struct folio *folio)
1530+{
1531+ return &folio_page(folio, 1)->compound_pincount;
1532+}
1533+
1534+/**
1535+ * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA.
1536+ * @folio: The folio.
1537+ *
1538+ * This function checks if a folio has been pinned via a call to
1539+ * a function in the pin_user_pages() family.
1540+ *
1541+ * For small folios, the return value is partially fuzzy: false is not fuzzy,
1542+ * because it means "definitely not pinned for DMA", but true means "probably
1543+ * pinned for DMA, but possibly a false positive due to having at least
1544+ * GUP_PIN_COUNTING_BIAS worth of normal folio references".
1545+ *
1546+ * False positives are OK, because: a) it's unlikely for a folio to
1547+ * get that many refcounts, and b) all the callers of this routine are
1548+ * expected to be able to deal gracefully with a false positive.
1549+ *
1550+ * For large folios, the result will be exactly correct. That's because
1551+ * we have more tracking data available: the compound_pincount is used
1552+ * instead of the GUP_PIN_COUNTING_BIAS scheme.
1553+ *
1554+ * For more information, please see Documentation/core-api/pin_user_pages.rst.
1555+ *
1556+ * Return: True, if it is likely that the page has been "dma-pinned".
1557+ * False, if the page is definitely not dma-pinned.
1558+ */
1559+static inline bool folio_maybe_dma_pinned(struct folio *folio)
1560+{
1561+ if (folio_test_large(folio))
1562+ return atomic_read(folio_pincount_ptr(folio)) > 0;
1563+
1564+ /*
1565+ * folio_ref_count() is signed. If that refcount overflows, then
1566+ * folio_ref_count() returns a negative value, and callers will avoid
1567+ * further incrementing the refcount.
1568+ *
1569+ * Here, for that overflow case, use the sign bit to count a little
1570+ * bit higher via unsigned math, and thus still get an accurate result.
1571+ */
1572+ return ((unsigned int)folio_ref_count(folio)) >=
1573+ GUP_PIN_COUNTING_BIAS;
1574+}
1575+
1576+static inline bool page_maybe_dma_pinned(struct page *page)
1577+{
1578+ return folio_maybe_dma_pinned(page_folio(page));
1579+}
1580+
1581+/*
1582+ * This should most likely only be called during fork() to see whether we
1583+ * should break the cow immediately for a page on the src mm.
1584+ */
1585+static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma,
1586+ struct page *page)
1587+{
1588+ if (!is_cow_mapping(vma->vm_flags))
1589+ return false;
1590+
1591+ if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
1592+ return false;
1593+
1594+ return page_maybe_dma_pinned(page);
1595+}
1596+
15891597 /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */
15901598 #ifdef CONFIG_MIGRATION
15911599 static inline bool is_pinnable_page(struct page *page)
@@ -1600,6 +1608,11 @@ static inline bool is_pinnable_page(struct page *page) · 8 unmodified lines
16001608 }
16011609 #endif
16021610
1611+static inline bool folio_is_pinnable(struct folio *folio)
1612+{
1613+ return is_pinnable_page(&folio->page);
1614+}
1615+
16031616 static inline void set_page_zone(struct page *page, enum zone_type zone)
16041617 {
16051618 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
@@ -1749,7 +1762,6 @@ static inline void *folio_address(const struct folio *folio) · 143 unmodified lines
17491762 }
17501763
17511764 extern void *page_rmapping(struct page *page);
1752−extern struct anon_vma *page_anon_vma(struct page *page);
17531765 extern pgoff_t __page_file_index(struct page *page);
17541766
17551767 /*
@@ -1855,7 +1867,6 @@ extern void truncate_setsize(struct inode *inode, loff_t newsize); · 99 unmodified lines
18551867 void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
18561868 void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
18571869 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1858−int invalidate_inode_page(struct page *page);
18591870
18601871 #ifdef CONFIG_MMU
18611872 extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
@@ -2921,13 +2932,11 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, · 1059 unmodified lines
29212932 #define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
29222933 #define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
29232934 * and return without waiting upon it */
2924−#define FOLL_POPULATE 0x40 /* fault in pages (with FOLL_MLOCK) */
29252935 #define FOLL_NOFAULT 0x80 /* do not fault in pages */
29262936 #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
29272937 #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
29282938 #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
29292939 #define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
2930−#define FOLL_MLOCK 0x1000 /* lock present pages */
29312940 #define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
29322941 #define FOLL_COW 0x4000 /* internal GUP flag */
29332942 #define FOLL_ANON 0x8000 /* don't do file mappings */
@@ -3381,5 +3390,4 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start, · 447 unmodified lines
33813390 }
33823391 #endif
33833392
3384−#endif /* __KERNEL__ */
33853393 #endif /* _LINUX_MM_H */
include/linux/mm_inline.h+8−3
@@ -99,7 +99,8 @@ void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) · 98 unmodified lines
9999
100100 update_lru_size(lruvec, lru, folio_zonenum(folio),
101101 folio_nr_pages(folio));
102− list_add(&folio->lru, &lruvec->lists[lru]);
102+ if (lru != LRU_UNEVICTABLE)
103+ list_add(&folio->lru, &lruvec->lists[lru]);
103104 }
104105
105106 static __always_inline void add_page_to_lru_list(struct page *page,
@@ -115,6 +116,7 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) · 9 unmodified lines
115116
116117 update_lru_size(lruvec, lru, folio_zonenum(folio),
117118 folio_nr_pages(folio));
119+ /* This is not expected to be used on LRU_UNEVICTABLE */
118120 list_add_tail(&folio->lru, &lruvec->lists[lru]);
119121 }
120122
@@ -127,8 +129,11 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page, · 6 unmodified lines
127129 static __always_inline
128130 void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
129131 {
130− list_del(&folio->lru);
131− update_lru_size(lruvec, folio_lru_list(folio), folio_zonenum(folio),
132+ enum lru_list lru = folio_lru_list(folio);
133+
134+ if (lru != LRU_UNEVICTABLE)
135+ list_del(&folio->lru);
136+ update_lru_size(lruvec, lru, folio_zonenum(folio),
132137 -folio_nr_pages(folio));
133138 }
134139
include/linux/mm_types.h+22−4
@@ -85,7 +85,16 @@ struct page { · 84 unmodified lines
8585 * lruvec->lru_lock. Sometimes used as a generic list
8686 * by the page owner.
8787 */
88− struct list_head lru;
88+ union {
89+ struct list_head lru;
90+ /* Or, for the Unevictable "LRU list" slot */
91+ struct {
92+ /* Always even, to negate PageTail */
93+ void *__filler;
94+ /* Count page's or folio's mlocks */
95+ unsigned int mlock_count;
96+ };
97+ };
8998 /* See page-flags.h for PAGE_MAPPING_FLAGS */
9099 struct address_space *mapping;
91100 pgoff_t index; /* Our offset within mapping. */
@@ -126,11 +135,14 @@ struct page { · 34 unmodified lines
126135 unsigned char compound_dtor;
127136 unsigned char compound_order;
128137 atomic_t compound_mapcount;
138+ atomic_t compound_pincount;
139+#ifdef CONFIG_64BIT
129140 unsigned int compound_nr; /* 1 << compound_order */
141+#endif
130142 };
131143 struct { /* Second tail page of compound page */
132144 unsigned long _compound_pad_1; /* compound_head */
133− atomic_t hpage_pinned_refcount;
145+ unsigned long _compound_pad_2;
134146 /* For both global and memcg */
135147 struct list_head deferred_list;
136148 };
@@ -241,7 +253,13 @@ struct folio { · 104 unmodified lines
241253 struct {
242254 /* public: */
243255 unsigned long flags;
244− struct list_head lru;
256+ union {
257+ struct list_head lru;
258+ struct {
259+ void *__filler;
260+ unsigned int mlock_count;
261+ };
262+ };
245263 struct address_space *mapping;
246264 pgoff_t index;
247265 void *private;
@@ -285,7 +303,7 @@ static inline atomic_t *compound_mapcount_ptr(struct page *page) · 37 unmodified lines
285303
286304 static inline atomic_t *compound_pincount_ptr(struct page *page)
287305 {
288− return &page[2].hpage_pinned_refcount;
306+ return &page[1].compound_pincount;
289307 }
290308
291309 /*
include/linux/pagemap.h+133−13
@@ -18,6 +18,120 @@ · 17 unmodified lines
1818
1919 struct folio_batch;
2020
21+unsigned long invalidate_mapping_pages(struct address_space *mapping,
22+ pgoff_t start, pgoff_t end);
23+
24+static inline void invalidate_remote_inode(struct inode *inode)
25+{
26+ if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
27+ S_ISLNK(inode->i_mode))
28+ invalidate_mapping_pages(inode->i_mapping, 0, -1);
29+}
30+int invalidate_inode_pages2(struct address_space *mapping);
31+int invalidate_inode_pages2_range(struct address_space *mapping,
32+ pgoff_t start, pgoff_t end);
33+int write_inode_now(struct inode *, int sync);
34+int filemap_fdatawrite(struct address_space *);
35+int filemap_flush(struct address_space *);
36+int filemap_fdatawait_keep_errors(struct address_space *mapping);
37+int filemap_fdatawait_range(struct address_space *, loff_t lstart, loff_t lend);
38+int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
39+ loff_t start_byte, loff_t end_byte);
40+
41+static inline int filemap_fdatawait(struct address_space *mapping)
42+{
43+ return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
44+}
45+
46+bool filemap_range_has_page(struct address_space *, loff_t lstart, loff_t lend);
47+int filemap_write_and_wait_range(struct address_space *mapping,
48+ loff_t lstart, loff_t lend);
49+int __filemap_fdatawrite_range(struct address_space *mapping,
50+ loff_t start, loff_t end, int sync_mode);
51+int filemap_fdatawrite_range(struct address_space *mapping,
52+ loff_t start, loff_t end);
53+int filemap_check_errors(struct address_space *mapping);
54+void __filemap_set_wb_err(struct address_space *mapping, int err);
55+int filemap_fdatawrite_wbc(struct address_space *mapping,
56+ struct writeback_control *wbc);
57+
58+static inline int filemap_write_and_wait(struct address_space *mapping)
59+{
60+ return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
61+}
62+
63+/**
64+ * filemap_set_wb_err - set a writeback error on an address_space
65+ * @mapping: mapping in which to set writeback error
66+ * @err: error to be set in mapping
67+ *
68+ * When writeback fails in some way, we must record that error so that
69+ * userspace can be informed when fsync and the like are called. We endeavor
70+ * to report errors on any file that was open at the time of the error. Some
71+ * internal callers also need to know when writeback errors have occurred.
72+ *
73+ * When a writeback error occurs, most filesystems will want to call
74+ * filemap_set_wb_err to record the error in the mapping so that it will be
75+ * automatically reported whenever fsync is called on the file.
76+ */
77+static inline void filemap_set_wb_err(struct address_space *mapping, int err)
78+{
79+ /* Fastpath for common case of no error */
80+ if (unlikely(err))
81+ __filemap_set_wb_err(mapping, err);
82+}
83+
84+/**
85+ * filemap_check_wb_err - has an error occurred since the mark was sampled?
86+ * @mapping: mapping to check for writeback errors
87+ * @since: previously-sampled errseq_t
88+ *
89+ * Grab the errseq_t value from the mapping, and see if it has changed "since"
90+ * the given value was sampled.
91+ *
92+ * If it has then report the latest error set, otherwise return 0.
93+ */
94+static inline int filemap_check_wb_err(struct address_space *mapping,
95+ errseq_t since)
96+{
97+ return errseq_check(&mapping->wb_err, since);
98+}
99+
100+/**
101+ * filemap_sample_wb_err - sample the current errseq_t to test for later errors
102+ * @mapping: mapping to be sampled
103+ *
104+ * Writeback errors are always reported relative to a particular sample point
105+ * in the past. This function provides those sample points.
106+ */
107+static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
108+{
109+ return errseq_sample(&mapping->wb_err);
110+}
111+
112+/**
113+ * file_sample_sb_err - sample the current errseq_t to test for later errors
114+ * @file: file pointer to be sampled
115+ *
116+ * Grab the most current superblock-level errseq_t value for the given
117+ * struct file.
118+ */
119+static inline errseq_t file_sample_sb_err(struct file *file)
120+{
121+ return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
122+}
123+
124+/*
125+ * Flush file data before changing attributes. Caller must hold any locks
126+ * required to prevent further writes to this file until we're done setting
127+ * flags.
128+ */
129+static inline int inode_drain_writes(struct inode *inode)
130+{
131+ inode_dio_wait(inode);
132+ return filemap_write_and_wait(inode->i_mapping);
133+}
134+
21135 static inline bool mapping_empty(struct address_space *mapping)
22136 {
23137 return xa_empty(&mapping->i_pages);
@@ -192,9 +306,14 @@ static inline void mapping_set_large_folios(struct address_space *mapping) · 168 unmodified lines
192306 __set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
193307 }
194308
309+/*
310+ * Large folio support currently depends on THP. These dependencies are
311+ * being worked on but are not yet fixed.
312+ */
195313 static inline bool mapping_large_folio_support(struct address_space *mapping)
196314 {
197− return test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
315+ return IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
316+ test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
198317 }
199318
200319 static inline int filemap_nr_thps(struct address_space *mapping)
@@ -212,7 +331,7 @@ static inline void filemap_nr_thps_inc(struct address_space *mapping) · 11 unmodified lines
212331 if (!mapping_large_folio_support(mapping))
213332 atomic_inc(&mapping->nr_thps);
214333 #else
215− WARN_ON_ONCE(1);
334+ WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
216335 #endif
217336 }
218337
@@ -222,7 +341,7 @@ static inline void filemap_nr_thps_dec(struct address_space *mapping) · 3 unmodified lines
222341 if (!mapping_large_folio_support(mapping))
223342 atomic_dec(&mapping->nr_thps);
224343 #else
225− WARN_ON_ONCE(1);
344+ WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
226345 #endif
227346 }
228347
@@ -283,16 +402,6 @@ static inline struct inode *folio_inode(struct folio *folio) · 54 unmodified lines
283402 return folio->mapping->host;
284403 }
285404
286−static inline bool page_cache_add_speculative(struct page *page, int count)
287−{
288− return folio_ref_try_add_rcu((struct folio *)page, count);
289−}
290−
291−static inline bool page_cache_get_speculative(struct page *page)
292−{
293− return page_cache_add_speculative(page, 1);
294−}
295−
296405 /**
297406 * folio_attach_private - Attach private data to a folio.
298407 * @folio: Folio to attach data to.
@@ -706,6 +815,17 @@ static inline loff_t folio_file_pos(struct folio *folio) · 407 unmodified lines
706815 return page_file_offset(&folio->page);
707816 }
708817
818+/*
819+ * Get the offset in PAGE_SIZE (even for hugetlb folios).
820+ * (TODO: hugetlb folios should have ->index in PAGE_SIZE)
821+ */
822+static inline pgoff_t folio_pgoff(struct folio *folio)
823+{
824+ if (unlikely(folio_test_hugetlb(folio)))
825+ return hugetlb_basepage_index(&folio->page);
826+ return folio->index;
827+}
828+
709829 extern pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
710830 unsigned long address);
711831
include/linux/rmap.h+47−29
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 #include <linux/rwsem.h>
1212 #include <linux/memcontrol.h>
1313 #include <linux/highmem.h>
14+#include <linux/pagemap.h>
1415
1516 /*
1617 * The anon_vma heads a list of private "related" vmas, to scan if
@@ -167,18 +168,19 @@ struct anon_vma *page_get_anon_vma(struct page *page); · 150 unmodified lines
167168 */
168169 void page_move_anon_rmap(struct page *, struct vm_area_struct *);
169170 void page_add_anon_rmap(struct page *, struct vm_area_struct *,
170− unsigned long, bool);
171+ unsigned long address, bool compound);
171172 void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
172− unsigned long, int);
173+ unsigned long address, int flags);
173174 void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
174− unsigned long, bool);
175−void page_add_file_rmap(struct page *, bool);
176−void page_remove_rmap(struct page *, bool);
177−
175+ unsigned long address, bool compound);
176+void page_add_file_rmap(struct page *, struct vm_area_struct *,
177+ bool compound);
178+void page_remove_rmap(struct page *, struct vm_area_struct *,
179+ bool compound);
178180 void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,
179− unsigned long);
181+ unsigned long address);
180182 void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *,
181− unsigned long);
183+ unsigned long address);
182184
183185 static inline void page_dup_rmap(struct page *page, bool compound)
184186 {
@@ -188,11 +190,11 @@ static inline void page_dup_rmap(struct page *page, bool compound) · 3 unmodified lines
188190 /*
189191 * Called from mm/vmscan.c to handle paging out
190192 */
191−int page_referenced(struct page *, int is_locked,
193+int folio_referenced(struct folio *, int is_locked,
192194 struct mem_cgroup *memcg, unsigned long *vm_flags);
193195
194−void try_to_migrate(struct page *page, enum ttu_flags flags);
195−void try_to_unmap(struct page *, enum ttu_flags flags);
196+void try_to_migrate(struct folio *folio, enum ttu_flags flags);
197+void try_to_unmap(struct folio *, enum ttu_flags flags);
196198
197199 int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
198200 unsigned long end, struct page **pages,
@@ -200,11 +202,13 @@ int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, · 1 unmodified line
200202
201203 /* Avoid racy checks */
202204 #define PVMW_SYNC (1 << 0)
203−/* Look for migarion entries rather than present PTEs */
205+/* Look for migration entries rather than present PTEs */
204206 #define PVMW_MIGRATION (1 << 1)
205207
206208 struct page_vma_mapped_walk {
207− struct page *page;
209+ unsigned long pfn;
210+ unsigned long nr_pages;
211+ pgoff_t pgoff;
208212 struct vm_area_struct *vma;
209213 unsigned long address;
210214 pmd_t *pmd;
@@ -213,10 +217,30 @@ struct page_vma_mapped_walk { · 2 unmodified lines
213217 unsigned int flags;
214218 };
215219
220+#define DEFINE_PAGE_VMA_WALK(name, _page, _vma, _address, _flags) \
221+ struct page_vma_mapped_walk name = { \
222+ .pfn = page_to_pfn(_page), \
223+ .nr_pages = compound_nr(page), \
224+ .pgoff = page_to_pgoff(page), \
225+ .vma = _vma, \
226+ .address = _address, \
227+ .flags = _flags, \
228+ }
229+
230+#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \
231+ struct page_vma_mapped_walk name = { \
232+ .pfn = folio_pfn(_folio), \
233+ .nr_pages = folio_nr_pages(_folio), \
234+ .pgoff = folio_pgoff(_folio), \
235+ .vma = _vma, \
236+ .address = _address, \
237+ .flags = _flags, \
238+ }
239+
216240 static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
217241 {
218242 /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
219− if (pvmw->pte && !PageHuge(pvmw->page))
243+ if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma))
220244 pte_unmap(pvmw->pte);
221245 if (pvmw->ptl)
222246 spin_unlock(pvmw->ptl);
@@ -237,18 +261,12 @@ unsigned long page_address_in_vma(struct page *, struct vm_area_struct *); · 14 unmodified lines
237261 */
238262 int folio_mkclean(struct folio *);
239263
240−/*
241− * called in munlock()/munmap() path to check for other vmas holding
242− * the page mlocked.
243− */
244−void page_mlock(struct page *page);
245−
246−void remove_migration_ptes(struct page *old, struct page *new, bool locked);
264+void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked);
247265
248266 /*
249267 * Called by memory-failure.c to kill processes.
250268 */
251−struct anon_vma *page_lock_anon_vma_read(struct page *page);
269+struct anon_vma *folio_lock_anon_vma_read(struct folio *folio);
252270 void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
253271 int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);
254272
@@ -267,15 +285,15 @@ struct rmap_walk_control { · 12 unmodified lines
267285 * Return false if page table scanning in rmap_walk should be stopped.
268286 * Otherwise, return true.
269287 */
270− bool (*rmap_one)(struct page *page, struct vm_area_struct *vma,
288+ bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma,
271289 unsigned long addr, void *arg);
272− int (*done)(struct page *page);
273− struct anon_vma *(*anon_lock)(struct page *page);
290+ int (*done)(struct folio *folio);
291+ struct anon_vma *(*anon_lock)(struct folio *folio);
274292 bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);
275293 };
276294
277−void rmap_walk(struct page *page, struct rmap_walk_control *rwc);
278−void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc);
295+void rmap_walk(struct folio *folio, const struct rmap_walk_control *rwc);
296+void rmap_walk_locked(struct folio *folio, const struct rmap_walk_control *rwc);
279297
280298 #else /* !CONFIG_MMU */
281299
@@ -283,7 +301,7 @@ void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc); · 1 unmodified line
283301 #define anon_vma_prepare(vma) (0)
284302 #define anon_vma_link(vma) do {} while (0)
285303
286−static inline int page_referenced(struct page *page, int is_locked,
304+static inline int folio_referenced(struct folio *folio, int is_locked,
287305 struct mem_cgroup *memcg,
288306 unsigned long *vm_flags)
289307 {
@@ -291,7 +309,7 @@ static inline int page_referenced(struct page *page, int is_locked, · 1 unmodified line
291309 return 0;
292310 }
293311
294−static inline void try_to_unmap(struct page *page, enum ttu_flags flags)
312+static inline void try_to_unmap(struct folio *folio, enum ttu_flags flags)
295313 {
296314 }
297315
include/linux/swap.h+4−5
@@ -328,7 +328,7 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio) · 327 unmodified lines
328328
329329 /* linux/mm/workingset.c */
330330 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
331−void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg);
331+void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg);
332332 void workingset_refault(struct folio *folio, void *shadow);
333333 void workingset_activation(struct folio *folio);
334334
@@ -375,7 +375,6 @@ extern void lru_add_drain(void); · 40 unmodified lines
375375 extern void lru_add_drain_cpu(int cpu);
376376 extern void lru_add_drain_cpu_zone(struct zone *zone);
377377 extern void lru_add_drain_all(void);
378−extern void deactivate_file_page(struct page *page);
379378 extern void deactivate_page(struct page *page);
380379 extern void mark_page_lazyfree(struct page *page);
381380 extern void swap_setup(void);
@@ -397,7 +396,7 @@ extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, · 15 unmodified lines
397396 unsigned long *nr_scanned);
398397 extern unsigned long shrink_all_memory(unsigned long nr_pages);
399398 extern int vm_swappiness;
400−extern int remove_mapping(struct address_space *mapping, struct page *page);
399+long remove_mapping(struct address_space *mapping, struct folio *folio);
401400
402401 extern unsigned long reclaim_pages(struct list_head *page_list);
403402 #ifdef CONFIG_NUMA
@@ -743,7 +742,7 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · 339 unmodified lines
743742 #endif
744743
745744 #ifdef CONFIG_MEMCG_SWAP
746−extern void mem_cgroup_swapout(struct page *page, swp_entry_t entry);
745+void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry);
747746 extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
748747 static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
749748 {
@@ -763,7 +762,7 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_p · 13 unmodified lines
763762 extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
764763 extern bool mem_cgroup_swap_full(struct page *page);
765764 #else
766−static inline void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
765+static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
767766 {
768767 }
769768
include/trace/events/vmscan.h+5−5
@@ -327,11 +327,11 @@ TRACE_EVENT(mm_vmscan_lru_isolate, · 326 unmodified lines
327327 __print_symbolic(__entry->lru, LRU_NAMES))
328328 );
329329
330−TRACE_EVENT(mm_vmscan_writepage,
330+TRACE_EVENT(mm_vmscan_write_folio,
331331
332− TP_PROTO(struct page *page),
332+ TP_PROTO(struct folio *folio),
333333
334− TP_ARGS(page),
334+ TP_ARGS(folio),
335335
336336 TP_STRUCT__entry(
337337 __field(unsigned long, pfn)
@@ -339,9 +339,9 @@ TRACE_EVENT(mm_vmscan_writepage, · 1 unmodified line
339339 ),
340340
341341 TP_fast_assign(
342− __entry->pfn = page_to_pfn(page);
342+ __entry->pfn = folio_pfn(folio);
343343 __entry->reclaim_flags = trace_reclaim_flags(
344− page_is_file_lru(page));
344+ folio_is_file_lru(folio));
345345 ),
346346
347347 TP_printk("page=%p pfn=0x%lx flags=%s",
kernel/events/uprobes.c+3−10
@@ -155,11 +155,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 154 unmodified lines
155155 struct page *old_page, struct page *new_page)
156156 {
157157 struct mm_struct *mm = vma->vm_mm;
158− struct page_vma_mapped_walk pvmw = {
159− .page = compound_head(old_page),
160− .vma = vma,
161− .address = addr,
162− };
158+ DEFINE_FOLIO_VMA_WALK(pvmw, page_folio(old_page), vma, addr, 0);
163159 int err;
164160 struct mmu_notifier_range range;
165161
@@ -173,7 +169,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 7 unmodified lines
173169 return err;
174170 }
175171
176− /* For try_to_free_swap() and munlock_vma_page() below */
172+ /* For try_to_free_swap() below */
177173 lock_page(old_page);
178174
179175 mmu_notifier_invalidate_range_start(&range);
@@ -201,13 +197,10 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 21 unmodified lines
201197 set_pte_at_notify(mm, addr, pvmw.pte,
202198 mk_pte(new_page, vma->vm_page_prot));
203199
204− page_remove_rmap(old_page, false);
200+ page_remove_rmap(old_page, vma, false);
205201 if (!page_mapped(old_page))
206202 try_to_free_swap(old_page);
207203 page_vma_mapped_walk_done(&pvmw);
208−
209− if ((vma->vm_flags & VM_LOCKED) && !PageCompound(old_page))
210− munlock_vma_page(old_page);
211204 put_page(old_page);
212205
213206 err = 0;
kernel/futex/core.c+1−1
@@ -302,7 +302,7 @@ again: · 301 unmodified lines
302302 * found it, but truncated or holepunched or subjected to
303303 * invalidate_complete_page2 before we got the page lock (also
304304 * cases which we are happy to fail). And we hold a reference,
305− * so refcount care in invalidate_complete_page's remove_mapping
305+ * so refcount care in invalidate_inode_page's remove_mapping
306306 * prevents drop_caches from setting mapping to NULL beneath us.
307307 *
308308 * The case we do have to guard against is when memory pressure made
lib/test_hmm.c+3−1
@@ -12,6 +12,7 @@ · 11 unmodified lines
1212 #include <linux/kernel.h>
1313 #include <linux/cdev.h>
1414 #include <linux/device.h>
15+#include <linux/memremap.h>
1516 #include <linux/mutex.h>
1617 #include <linux/rwsem.h>
1718 #include <linux/sched.h>
@@ -26,6 +27,8 @@ · 8 unmodified lines
2627 #include <linux/sched/mm.h>
2728 #include <linux/platform_device.h>
2829 #include <linux/rmap.h>
30+#include <linux/mmu_notifier.h>
31+#include <linux/migrate.h>
2932
3033 #include "test_hmm_uapi.h"
3134
@@ -563,7 +566,6 @@ static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) · 531 unmodified lines
563566 }
564567
565568 dpage->zone_device_data = rpage;
566− get_page(dpage);
567569 lock_page(dpage);
568570 return dpage;
569571
mm/Kconfig+3−4
@@ -249,6 +249,9 @@ config MIGRATION · 248 unmodified lines
249249 pages as migration can relocate pages to satisfy a huge page
250250 allocation instead of reclaiming.
251251
252+config DEVICE_MIGRATION
253+ def_bool MIGRATION && ZONE_DEVICE
254+
252255 config ARCH_ENABLE_HUGEPAGE_MIGRATION
253256 bool
254257
@@ -791,9 +794,6 @@ config ZONE_DEVICE · 536 unmodified lines
791794
792795 If FS_DAX is enabled, then say Y.
793796
794−config DEV_PAGEMAP_OPS
795− bool
796−
797797 #
798798 # Helpers to mirror range of the CPU page tables of a process into device page
799799 # tables.
@@ -805,7 +805,6 @@ config HMM_MIRROR · 5 unmodified lines
805805 config DEVICE_PRIVATE
806806 bool "Unaddressable device memory (GPU memory, ...)"
807807 depends on ZONE_DEVICE
808− select DEV_PAGEMAP_OPS
809808
810809 help
811810 Allows creation of struct pages to represent unaddressable device
mm/Makefile+1−0
@@ -92,6 +92,7 @@ obj-$(CONFIG_KFENCE) += kfence/ · 91 unmodified lines
9292 obj-$(CONFIG_FAILSLAB) += failslab.o
9393 obj-$(CONFIG_MEMTEST) += memtest.o
9494 obj-$(CONFIG_MIGRATION) += migrate.o
95+obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o
9596 obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o
9697 obj-$(CONFIG_PAGE_COUNTER) += page_counter.o
9798 obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o
mm/damon/paddr.c+29−33
@@ -16,14 +16,10 @@ · 15 unmodified lines
1616 #include "../internal.h"
1717 #include "ops-common.h"
1818
19−static bool __damon_pa_mkold(struct page *page, struct vm_area_struct *vma,
19+static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma,
2020 unsigned long addr, void *arg)
2121 {
22− struct page_vma_mapped_walk pvmw = {
23− .page = page,
24− .vma = vma,
25− .address = addr,
26− };
22+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
2723
2824 while (page_vma_mapped_walk(&pvmw)) {
2925 addr = pvmw.address;
@@ -37,32 +33,34 @@ static bool __damon_pa_mkold(struct page *page, struct vm_area_struct *vma, · 7 unmodified lines
3733
3834 static void damon_pa_mkold(unsigned long paddr)
3935 {
36+ struct folio *folio;
4037 struct page *page = damon_get_page(PHYS_PFN(paddr));
4138 struct rmap_walk_control rwc = {
4239 .rmap_one = __damon_pa_mkold,
43− .anon_lock = page_lock_anon_vma_read,
40+ .anon_lock = folio_lock_anon_vma_read,
4441 };
4542 bool need_lock;
4643
4744 if (!page)
4845 return;
46+ folio = page_folio(page);
4947
50− if (!page_mapped(page) || !page_rmapping(page)) {
51− set_page_idle(page);
48+ if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
49+ folio_set_idle(folio);
5250 goto out;
5351 }
5452
55− need_lock = !PageAnon(page) || PageKsm(page);
56− if (need_lock && !trylock_page(page))
53+ need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
54+ if (need_lock && !folio_trylock(folio))
5755 goto out;
5856
59− rmap_walk(page, &rwc);
57+ rmap_walk(folio, &rwc);
6058
6159 if (need_lock)
62− unlock_page(page);
60+ folio_unlock(folio);
6361
6462 out:
65− put_page(page);
63+ folio_put(folio);
6664 }
6765
6866 static void __damon_pa_prepare_access_check(struct damon_ctx *ctx,
@@ -89,15 +87,11 @@ struct damon_pa_access_chk_result { · 20 unmodified lines
8987 bool accessed;
9088 };
9189
92−static bool __damon_pa_young(struct page *page, struct vm_area_struct *vma,
90+static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma,
9391 unsigned long addr, void *arg)
9492 {
9593 struct damon_pa_access_chk_result *result = arg;
96− struct page_vma_mapped_walk pvmw = {
97− .page = page,
98− .vma = vma,
99− .address = addr,
100− };
94+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
10195
10296 result->accessed = false;
10397 result->page_sz = PAGE_SIZE;
@@ -105,12 +99,12 @@ static bool __damon_pa_young(struct page *page, struct vm_area_struct *vma, · 1 unmodified line
10599 addr = pvmw.address;
106100 if (pvmw.pte) {
107101 result->accessed = pte_young(*pvmw.pte) ||
108− !page_is_idle(page) ||
102+ !folio_test_idle(folio) ||
109103 mmu_notifier_test_young(vma->vm_mm, addr);
110104 } else {
111105 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
112106 result->accessed = pmd_young(*pvmw.pmd) ||
113− !page_is_idle(page) ||
107+ !folio_test_idle(folio) ||
114108 mmu_notifier_test_young(vma->vm_mm, addr);
115109 result->page_sz = ((1UL) << HPAGE_PMD_SHIFT);
116110 #else
@@ -129,6 +123,7 @@ static bool __damon_pa_young(struct page *page, struct vm_area_struct *vma, · 12 unmodified lines
129123
130124 static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz)
131125 {
126+ struct folio *folio;
132127 struct page *page = damon_get_page(PHYS_PFN(paddr));
133128 struct damon_pa_access_chk_result result = {
134129 .page_sz = PAGE_SIZE,
@@ -137,33 +132,34 @@ static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz) · 2 unmodified lines
137132 struct rmap_walk_control rwc = {
138133 .arg = &result,
139134 .rmap_one = __damon_pa_young,
140− .anon_lock = page_lock_anon_vma_read,
135+ .anon_lock = folio_lock_anon_vma_read,
141136 };
142137 bool need_lock;
143138
144139 if (!page)
145140 return false;
141+ folio = page_folio(page);
146142
147− if (!page_mapped(page) || !page_rmapping(page)) {
148− if (page_is_idle(page))
143+ if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
144+ if (folio_test_idle(folio))
149145 result.accessed = false;
150146 else
151147 result.accessed = true;
152− put_page(page);
148+ folio_put(folio);
153149 goto out;
154150 }
155151
156− need_lock = !PageAnon(page) || PageKsm(page);
157− if (need_lock && !trylock_page(page)) {
158− put_page(page);
159− return NULL;
152+ need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
153+ if (need_lock && !folio_trylock(folio)) {
154+ folio_put(folio);
155+ return false;
160156 }
161157
162− rmap_walk(page, &rwc);
158+ rmap_walk(folio, &rwc);
163159
164160 if (need_lock)
165− unlock_page(page);
166− put_page(page);
161+ folio_unlock(folio);
162+ folio_put(folio);
167163
168164 out:
169165 *page_sz = result.page_sz;
mm/debug.c+7−11
@@ -48,7 +48,8 @@ const struct trace_print_flags vmaflag_names[] = { · 47 unmodified lines
4848
4949 static void __dump_page(struct page *page)
5050 {
51− struct page *head = compound_head(page);
51+ struct folio *folio = page_folio(page);
52+ struct page *head = &folio->page;
5253 struct address_space *mapping;
5354 bool compound = PageCompound(page);
5455 /*
@@ -76,6 +77,7 @@ static void __dump_page(struct page *page) · 21 unmodified lines
7677 else
7778 mapping = (void *)(tmp & ~PAGE_MAPPING_FLAGS);
7879 head = page;
80+ folio = (struct folio *)page;
7981 compound = false;
8082 } else {
8183 mapping = page_mapping(page);
@@ -92,16 +94,10 @@ static void __dump_page(struct page *page) · 10 unmodified lines
9294 page, page_ref_count(head), mapcount, mapping,
9395 page_to_pgoff(page), page_to_pfn(page));
9496 if (compound) {
95− if (hpage_pincount_available(page)) {
96− pr_warn("head:%p order:%u compound_mapcount:%d compound_pincount:%d\n",
97− head, compound_order(head),
98− head_compound_mapcount(head),
99− head_compound_pincount(head));
100− } else {
101− pr_warn("head:%p order:%u compound_mapcount:%d\n",
102− head, compound_order(head),
103− head_compound_mapcount(head));
104− }
97+ pr_warn("head:%p order:%u compound_mapcount:%d compound_pincount:%d\n",
98+ head, compound_order(head),
99+ folio_entire_mapcount(folio),
100+ head_compound_pincount(head));
105101 }
106102
107103 #ifdef CONFIG_MEMCG
mm/filemap.c+41−18
@@ -842,26 +842,27 @@ noinline int __filemap_add_folio(struct address_space *mapping, · 841 unmodified lines
842842 {
843843 XA_STATE(xas, &mapping->i_pages, index);
844844 int huge = folio_test_hugetlb(folio);
845− int error;
846845 bool charged = false;
846+ long nr = 1;
847847
848848 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
849849 VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio);
850850 mapping_set_update(&xas, mapping);
851851
852− folio_get(folio);
853− folio->mapping = mapping;
854− folio->index = index;
855−
856852 if (!huge) {
857− error = mem_cgroup_charge(folio, NULL, gfp);
853+ int error = mem_cgroup_charge(folio, NULL, gfp);
858854 VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio);
859855 if (error)
860− goto error;
856+ return error;
861857 charged = true;
858+ xas_set_order(&xas, index, folio_order(folio));
859+ nr = folio_nr_pages(folio);
862860 }
863861
864862 gfp &= GFP_RECLAIM_MASK;
863+ folio_ref_add(folio, nr);
864+ folio->mapping = mapping;
865+ folio->index = xas.xa_index;
865866
866867 do {
867868 unsigned int order = xa_get_order(xas.xa, xas.xa_index);
@@ -885,6 +886,8 @@ noinline int __filemap_add_folio(struct address_space *mapping, · 17 unmodified lines
885886 /* entry may have been split before we acquired lock */
886887 order = xa_get_order(xas.xa, xas.xa_index);
887888 if (order > folio_order(folio)) {
889+ /* How to handle large swap entries? */
890+ BUG_ON(shmem_mapping(mapping));
888891 xas_split(&xas, old, order);
889892 xas_reset(&xas);
890893 }
@@ -894,29 +897,31 @@ noinline int __filemap_add_folio(struct address_space *mapping, · 3 unmodified lines
894897 if (xas_error(&xas))
895898 goto unlock;
896899
897− mapping->nrpages++;
900+ mapping->nrpages += nr;
898901
899902 /* hugetlb pages do not participate in page cache accounting */
900− if (!huge)
901− __lruvec_stat_add_folio(folio, NR_FILE_PAGES);
903+ if (!huge) {
904+ __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
905+ if (folio_test_pmd_mappable(folio))
906+ __lruvec_stat_mod_folio(folio,
907+ NR_FILE_THPS, nr);
908+ }
902909 unlock:
903910 xas_unlock_irq(&xas);
904911 } while (xas_nomem(&xas, gfp));
905912
906− if (xas_error(&xas)) {
907− error = xas_error(&xas);
908− if (charged)
909− mem_cgroup_uncharge(folio);
913+ if (xas_error(&xas))
910914 goto error;
911− }
912915
913916 trace_mm_filemap_add_to_page_cache(folio);
914917 return 0;
915918 error:
919+ if (charged)
920+ mem_cgroup_uncharge(folio);
916921 folio->mapping = NULL;
917922 /* Leave page->index set: truncation relies upon it */
918− folio_put(folio);
919− return error;
923+ folio_put_refs(folio, nr);
924+ return xas_error(&xas);
920925 }
921926 ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO);
922927
@@ -2997,6 +3002,24 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) · 2074 unmodified lines
29973002 struct file *fpin = NULL;
29983003 unsigned int mmap_miss;
29993004
3005+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3006+ /* Use the readahead code, even if readahead is disabled */
3007+ if (vmf->vma->vm_flags & VM_HUGEPAGE) {
3008+ fpin = maybe_unlock_mmap_for_io(vmf, fpin);
3009+ ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1);
3010+ ra->size = HPAGE_PMD_NR;
3011+ /*
3012+ * Fetch two PMD folios, so we get the chance to actually
3013+ * readahead, unless we've been told not to.
3014+ */
3015+ if (!(vmf->vma->vm_flags & VM_RAND_READ))
3016+ ra->size *= 2;
3017+ ra->async_size = HPAGE_PMD_NR;
3018+ page_cache_ra_order(&ractl, ra, HPAGE_PMD_ORDER);
3019+ return fpin;
3020+ }
3021+#endif
3022+
30003023 /* If we don't want any read-ahead, don't bother */
30013024 if (vmf->vma->vm_flags & VM_RAND_READ)
30023025 return fpin;
@@ -3029,7 +3052,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) · 26 unmodified lines
30293052 ra->size = ra->ra_pages;
30303053 ra->async_size = ra->ra_pages / 4;
30313054 ractl._index = ra->start;
3032− do_page_cache_ra(&ractl, ra->size, ra->async_size);
3055+ page_cache_ra_order(&ractl, ra, 0);
30333056 return fpin;
30343057 }
30353058
mm/folio-compat.c+13−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include <linux/migrate.h>
88 #include <linux/pagemap.h>
99 #include <linux/swap.h>
10+#include "internal.h"
1011
1112 struct address_space *page_mapping(struct page *page)
1213 {
@@ -151,3 +152,15 @@ int try_to_release_page(struct page *page, gfp_t gfp) · 138 unmodified lines
151152 return filemap_release_folio(page_folio(page), gfp);
152153 }
153154 EXPORT_SYMBOL(try_to_release_page);
155+
156+int isolate_lru_page(struct page *page)
157+{
158+ if (WARN_RATELIMIT(PageTail(page), "trying to isolate tail page"))
159+ return -EBUSY;
160+ return folio_isolate_lru((struct folio *)page);
161+}
162+
163+void putback_lru_page(struct page *page)
164+{
165+ folio_putback_lru(page_folio(page));
166+}
mm/gup.c+205−281
@@ -29,107 +29,71 @@ struct follow_page_context { · 28 unmodified lines
2929 unsigned int page_mask;
3030 };
3131
32−static void hpage_pincount_add(struct page *page, int refs)
33−{
34− VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
35− VM_BUG_ON_PAGE(page != compound_head(page), page);
36−
37− atomic_add(refs, compound_pincount_ptr(page));
38−}
39−
40−static void hpage_pincount_sub(struct page *page, int refs)
41−{
42− VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
43− VM_BUG_ON_PAGE(page != compound_head(page), page);
44−
45− atomic_sub(refs, compound_pincount_ptr(page));
46−}
47−
48−/* Equivalent to calling put_page() @refs times. */
49−static void put_page_refs(struct page *page, int refs)
50−{
51−#ifdef CONFIG_DEBUG_VM
52− if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))
53− return;
54−#endif
55−
56− /*
57− * Calling put_page() for each ref is unnecessarily slow. Only the last
58− * ref needs a put_page().
59− */
60− if (refs > 1)
61− page_ref_sub(page, refs - 1);
62− put_page(page);
63−}
64−
6532 /*
66− * Return the compound head page with ref appropriately incremented,
33+ * Return the folio with ref appropriately incremented,
6734 * or NULL if that failed.
6835 */
69−static inline struct page *try_get_compound_head(struct page *page, int refs)
36+static inline struct folio *try_get_folio(struct page *page, int refs)
7037 {
71− struct page *head = compound_head(page);
38+ struct folio *folio;
7239
73− if (WARN_ON_ONCE(page_ref_count(head) < 0))
40+retry:
41+ folio = page_folio(page);
42+ if (WARN_ON_ONCE(folio_ref_count(folio) < 0))
7443 return NULL;
75− if (unlikely(!page_cache_add_speculative(head, refs)))
44+ if (unlikely(!folio_ref_try_add_rcu(folio, refs)))
7645 return NULL;
7746
7847 /*
79− * At this point we have a stable reference to the head page; but it
80− * could be that between the compound_head() lookup and the refcount
81− * increment, the compound page was split, in which case we'd end up
82− * holding a reference on a page that has nothing to do with the page
48+ * At this point we have a stable reference to the folio; but it
49+ * could be that between calling page_folio() and the refcount
50+ * increment, the folio was split, in which case we'd end up
51+ * holding a reference on a folio that has nothing to do with the page
8352 * we were given anymore.
84− * So now that the head page is stable, recheck that the pages still
85− * belong together.
53+ * So now that the folio is stable, recheck that the page still
54+ * belongs to this folio.
8655 */
87− if (unlikely(compound_head(page) != head)) {
88− put_page_refs(head, refs);
89− return NULL;
56+ if (unlikely(page_folio(page) != folio)) {
57+ folio_put_refs(folio, refs);
58+ goto retry;
9059 }
9160
92− return head;
61+ return folio;
9362 }
9463
9564 /**
96− * try_grab_compound_head() - attempt to elevate a page's refcount, by a
97− * flags-dependent amount.
98− *
99− * Even though the name includes "compound_head", this function is still
100− * appropriate for callers that have a non-compound @page to get.
101− *
65+ * try_grab_folio() - Attempt to get or pin a folio.
10266 * @page: pointer to page to be grabbed
103− * @refs: the value to (effectively) add to the page's refcount
67+ * @refs: the value to (effectively) add to the folio's refcount
10468 * @flags: gup flags: these are the FOLL_* flag values.
10569 *
10670 * "grab" names in this file mean, "look at flags to decide whether to use
107− * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
71+ * FOLL_PIN or FOLL_GET behavior, when incrementing the folio's refcount.
10872 *
10973 * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
11074 * same time. (That's true throughout the get_user_pages*() and
11175 * pin_user_pages*() APIs.) Cases:
11276 *
113− * FOLL_GET: page's refcount will be incremented by @refs.
77+ * FOLL_GET: folio's refcount will be incremented by @refs.
11478 *
115− * FOLL_PIN on compound pages that are > two pages long: page's refcount will
116− * be incremented by @refs, and page[2].hpage_pinned_refcount will be
117− * incremented by @refs * GUP_PIN_COUNTING_BIAS.
79+ * FOLL_PIN on large folios: folio's refcount will be incremented by
80+ * @refs, and its compound_pincount will be incremented by @refs.
11881 *
119− * FOLL_PIN on normal pages, or compound pages that are two pages long:
120− * page's refcount will be incremented by @refs * GUP_PIN_COUNTING_BIAS.
82+ * FOLL_PIN on single-page folios: folio's refcount will be incremented by
83+ * @refs * GUP_PIN_COUNTING_BIAS.
12184 *
122− * Return: head page (with refcount appropriately incremented) for success, or
123− * NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's
124− * considered failure, and furthermore, a likely bug in the caller, so a warning
125− * is also emitted.
85+ * Return: The folio containing @page (with refcount appropriately
86+ * incremented) for success, or NULL upon failure. If neither FOLL_GET
87+ * nor FOLL_PIN was set, that's considered failure, and furthermore,
88+ * a likely bug in the caller, so a warning is also emitted.
12689 */
127−__maybe_unused struct page *try_grab_compound_head(struct page *page,
128− int refs, unsigned int flags)
90+struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags)
12991 {
13092 if (flags & FOLL_GET)
131− return try_get_compound_head(page, refs);
93+ return try_get_folio(page, refs);
13294 else if (flags & FOLL_PIN) {
95+ struct folio *folio;
96+
13397 /*
13498 * Can't do FOLL_LONGTERM + FOLL_PIN gup fast path if not in a
13599 * right zone, so fail and let the caller fall back to the slow
@@ -143,63 +107,57 @@ __maybe_unused struct page *try_grab_compound_head(struct page *page, · 7 unmodified lines
143107 * CAUTION: Don't use compound_head() on the page before this
144108 * point, the result won't be stable.
145109 */
146− page = try_get_compound_head(page, refs);
147− if (!page)
110+ folio = try_get_folio(page, refs);
111+ if (!folio)
148112 return NULL;
149113
150114 /*
151− * When pinning a compound page of order > 1 (which is what
152− * hpage_pincount_available() checks for), use an exact count to
153− * track it, via hpage_pincount_add/_sub().
115+ * When pinning a large folio, use an exact count to track it.
154116 *
155− * However, be sure to *also* increment the normal page refcount
156− * field at least once, so that the page really is pinned.
157− * That's why the refcount from the earlier
158− * try_get_compound_head() is left intact.
117+ * However, be sure to *also* increment the normal folio
118+ * refcount field at least once, so that the folio really
119+ * is pinned. That's why the refcount from the earlier
120+ * try_get_folio() is left intact.
159121 */
160− if (hpage_pincount_available(page))
161− hpage_pincount_add(page, refs);
122+ if (folio_test_large(folio))
123+ atomic_add(refs, folio_pincount_ptr(folio));
162124 else
163− page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));
164−
165− mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,
166− refs);
125+ folio_ref_add(folio,
126+ refs * (GUP_PIN_COUNTING_BIAS - 1));
127+ node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);
167128
168− return page;
129+ return folio;
169130 }
170131
171132 WARN_ON_ONCE(1);
172133 return NULL;
173134 }
174135
175−static void put_compound_head(struct page *page, int refs, unsigned int flags)
136+static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)
176137 {
177138 if (flags & FOLL_PIN) {
178− mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
179− refs);
180−
181− if (hpage_pincount_available(page))
182− hpage_pincount_sub(page, refs);
139+ node_stat_mod_folio(folio, NR_FOLL_PIN_RELEASED, refs);
140+ if (folio_test_large(folio))
141+ atomic_sub(refs, folio_pincount_ptr(folio));
183142 else
184143 refs *= GUP_PIN_COUNTING_BIAS;
185144 }
186145
187− put_page_refs(page, refs);
146+ folio_put_refs(folio, refs);
188147 }
189148
190149 /**
191150 * try_grab_page() - elevate a page's refcount by a flag-dependent amount
151+ * @page: pointer to page to be grabbed
152+ * @flags: gup flags: these are the FOLL_* flag values.
192153 *
193154 * This might not do anything at all, depending on the flags argument.
194155 *
195156 * "grab" names in this file mean, "look at flags to decide whether to use
196157 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
197158 *
198− * @page: pointer to page to be grabbed
199− * @flags: gup flags: these are the FOLL_* flag values.
200− *
201159 * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
202− * time. Cases: please see the try_grab_compound_head() documentation, with
160+ * time. Cases: please see the try_grab_folio() documentation, with
203161 * "refs=1".
204162 *
205163 * Return: true for success, or if no action was required (if neither FOLL_PIN
@@ -208,32 +166,28 @@ static void put_compound_head(struct page *page, int refs, unsigned int flags) · 2 unmodified lines
208166 */
209167 bool __must_check try_grab_page(struct page *page, unsigned int flags)
210168 {
169+ struct folio *folio = page_folio(page);
170+
211171 WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN));
172+ if (WARN_ON_ONCE(folio_ref_count(folio) <= 0))
173+ return false;
212174
213175 if (flags & FOLL_GET)
214− return try_get_page(page);
176+ folio_ref_inc(folio);
215177 else if (flags & FOLL_PIN) {
216− int refs = 1;
217−
218− page = compound_head(page);
219−
220− if (WARN_ON_ONCE(page_ref_count(page) <= 0))
221− return false;
222−
223− if (hpage_pincount_available(page))
224− hpage_pincount_add(page, 1);
225− else
226− refs = GUP_PIN_COUNTING_BIAS;
227−
228178 /*
229− * Similar to try_grab_compound_head(): even if using the
230− * hpage_pincount_add/_sub() routines, be sure to
231− * *also* increment the normal page refcount field at least
232− * once, so that the page really is pinned.
179+ * Similar to try_grab_folio(): be sure to *also*
180+ * increment the normal page refcount field at least once,
181+ * so that the page really is pinned.
233182 */
234− page_ref_add(page, refs);
183+ if (folio_test_large(folio)) {
184+ folio_ref_add(folio, 1);
185+ atomic_add(1, folio_pincount_ptr(folio));
186+ } else {
187+ folio_ref_add(folio, GUP_PIN_COUNTING_BIAS);
188+ }
235189
236− mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);
190+ node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, 1);
237191 }
238192
239193 return true;
@@ -250,62 +204,40 @@ bool __must_check try_grab_page(struct page *page, unsigned int flags) · 10 unmodified lines
250204 */
251205 void unpin_user_page(struct page *page)
252206 {
253− put_compound_head(compound_head(page), 1, FOLL_PIN);
207+ gup_put_folio(page_folio(page), 1, FOLL_PIN);
254208 }
255209 EXPORT_SYMBOL(unpin_user_page);
256210
257−static inline void compound_range_next(unsigned long i, unsigned long npages,
258− struct page **list, struct page **head,
259− unsigned int *ntails)
211+static inline struct folio *gup_folio_range_next(struct page *start,
212+ unsigned long npages, unsigned long i, unsigned int *ntails)
260213 {
261− struct page *next, *page;
214+ struct page *next = nth_page(start, i);
215+ struct folio *folio = page_folio(next);
262216 unsigned int nr = 1;
263217
264− if (i >= npages)
265− return;
266−
267− next = *list + i;
268− page = compound_head(next);
269− if (PageCompound(page) && compound_order(page) >= 1)
270− nr = min_t(unsigned int,
271− page + compound_nr(page) - next, npages - i);
218+ if (folio_test_large(folio))
219+ nr = min_t(unsigned int, npages - i,
220+ folio_nr_pages(folio) - folio_page_idx(folio, next));
272221
273− *head = page;
274222 *ntails = nr;
223+ return folio;
275224 }
276225
277−#define for_each_compound_range(__i, __list, __npages, __head, __ntails) \
278− for (__i = 0, \
279− compound_range_next(__i, __npages, __list, &(__head), &(__ntails)); \
280− __i < __npages; __i += __ntails, \
281− compound_range_next(__i, __npages, __list, &(__head), &(__ntails)))
282−
283−static inline void compound_next(unsigned long i, unsigned long npages,
284− struct page **list, struct page **head,
285− unsigned int *ntails)
226+static inline struct folio *gup_folio_next(struct page **list,
227+ unsigned long npages, unsigned long i, unsigned int *ntails)
286228 {
287− struct page *page;
229+ struct folio *folio = page_folio(list[i]);
288230 unsigned int nr;
289231
290− if (i >= npages)
291− return;
292−
293− page = compound_head(list[i]);
294232 for (nr = i + 1; nr < npages; nr++) {
295− if (compound_head(list[nr]) != page)
233+ if (page_folio(list[nr]) != folio)
296234 break;
297235 }
298236
299− *head = page;
300237 *ntails = nr - i;
238+ return folio;
301239 }
302240
303−#define for_each_compound_head(__i, __list, __npages, __head, __ntails) \
304− for (__i = 0, \
305− compound_next(__i, __npages, __list, &(__head), &(__ntails)); \
306− __i < __npages; __i += __ntails, \
307− compound_next(__i, __npages, __list, &(__head), &(__ntails)))
308−
309241 /**
310242 * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
311243 * @pages: array of pages to be maybe marked dirty, and definitely released.
@@ -331,16 +263,17 @@ static inline void compound_next(unsigned long i, unsigned long npages, · 19 unmodified lines
331263 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
332264 bool make_dirty)
333265 {
334− unsigned long index;
335− struct page *head;
336− unsigned int ntails;
266+ unsigned long i;
267+ struct folio *folio;
268+ unsigned int nr;
337269
338270 if (!make_dirty) {
339271 unpin_user_pages(pages, npages);
340272 return;
341273 }
342274
343− for_each_compound_head(index, pages, npages, head, ntails) {
275+ for (i = 0; i < npages; i += nr) {
276+ folio = gup_folio_next(pages, npages, i, &nr);
344277 /*
345278 * Checking PageDirty at this point may race with
346279 * clear_page_dirty_for_io(), but that's OK. Two key
@@ -361,9 +294,12 @@ void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, · 14 unmodified lines
361294 * written back, so it gets written back again in the
362295 * next writeback cycle. This is harmless.
363296 */
364− if (!PageDirty(head))
365− set_page_dirty_lock(head);
366− put_compound_head(head, ntails, FOLL_PIN);
297+ if (!folio_test_dirty(folio)) {
298+ folio_lock(folio);
299+ folio_mark_dirty(folio);
300+ folio_unlock(folio);
301+ }
302+ gup_put_folio(folio, nr, FOLL_PIN);
367303 }
368304 }
369305 EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
@@ -392,14 +328,18 @@ EXPORT_SYMBOL(unpin_user_pages_dirty_lock); · 22 unmodified lines
392328 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
393329 bool make_dirty)
394330 {
395− unsigned long index;
396− struct page *head;
397− unsigned int ntails;
331+ unsigned long i;
332+ struct folio *folio;
333+ unsigned int nr;
398334
399− for_each_compound_range(index, &page, npages, head, ntails) {
400− if (make_dirty && !PageDirty(head))
401− set_page_dirty_lock(head);
402− put_compound_head(head, ntails, FOLL_PIN);
335+ for (i = 0; i < npages; i += nr) {
336+ folio = gup_folio_range_next(page, npages, i, &nr);
337+ if (make_dirty && !folio_test_dirty(folio)) {
338+ folio_lock(folio);
339+ folio_mark_dirty(folio);
340+ folio_unlock(folio);
341+ }
342+ gup_put_folio(folio, nr, FOLL_PIN);
403343 }
404344 }
405345 EXPORT_SYMBOL(unpin_user_page_range_dirty_lock);
@@ -415,9 +355,9 @@ EXPORT_SYMBOL(unpin_user_page_range_dirty_lock); · 9 unmodified lines
415355 */
416356 void unpin_user_pages(struct page **pages, unsigned long npages)
417357 {
418− unsigned long index;
419− struct page *head;
420− unsigned int ntails;
358+ unsigned long i;
359+ struct folio *folio;
360+ unsigned int nr;
421361
422362 /*
423363 * If this WARN_ON() fires, then the system *might* be leaking pages (by
@@ -427,8 +367,10 @@ void unpin_user_pages(struct page **pages, unsigned long npages) · 3 unmodified lines
427367 if (WARN_ON(IS_ERR_VALUE(npages)))
428368 return;
429369
430− for_each_compound_head(index, pages, npages, head, ntails)
431− put_compound_head(head, ntails, FOLL_PIN);
370+ for (i = 0; i < npages; i += nr) {
371+ folio = gup_folio_next(pages, npages, i, &nr);
372+ gup_put_folio(folio, nr, FOLL_PIN);
373+ }
432374 }
433375 EXPORT_SYMBOL(unpin_user_pages);
434376
@@ -593,32 +535,6 @@ retry: · 158 unmodified lines
593535 */
594536 mark_page_accessed(page);
595537 }
596− if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
597− /* Do not mlock pte-mapped THP */
598− if (PageTransCompound(page))
599− goto out;
600−
601− /*
602− * The preliminary mapping check is mainly to avoid the
603− * pointless overhead of lock_page on the ZERO_PAGE
604− * which might bounce very badly if there is contention.
605− *
606− * If the page is already locked, we don't need to
607− * handle it now - vmscan will handle it later if and
608− * when it attempts to reclaim the page.
609− */
610− if (page->mapping && trylock_page(page)) {
611− lru_add_drain(); /* push cached pages to LRU */
612− /*
613− * Because we lock page here, and migration is
614− * blocked by the pte's page reference, and we
615− * know the page is still mapped, we don't even
616− * need to check for file-cache page truncation.
617− */
618− mlock_vma_page(page);
619− unlock_page(page);
620− }
621− }
622538 out:
623539 pte_unmap_unlock(ptep, ptl);
624540 return page;
@@ -941,9 +857,6 @@ static int faultin_page(struct vm_area_struct *vma, · 316 unmodified lines
941857 unsigned int fault_flags = 0;
942858 vm_fault_t ret;
943859
944− /* mlock all present pages, but do not fault in new pages */
945− if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
946− return -ENOENT;
947860 if (*flags & FOLL_NOFAULT)
948861 return -EFAULT;
949862 if (*flags & FOLL_WRITE)
@@ -1194,8 +1107,6 @@ retry: · 244 unmodified lines
11941107 case -ENOMEM:
11951108 case -EHWPOISON:
11961109 goto out;
1197− case -ENOENT:
1198− goto next_page;
11991110 }
12001111 BUG();
12011112 } else if (PTR_ERR(page) == -EEXIST) {
@@ -1500,9 +1411,14 @@ long populate_vma_page_range(struct vm_area_struct *vma, · 298 unmodified lines
15001411 VM_BUG_ON_VMA(end > vma->vm_end, vma);
15011412 mmap_assert_locked(mm);
15021413
1503− gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
1414+ /*
1415+ * Rightly or wrongly, the VM_LOCKONFAULT case has never used
1416+ * faultin_page() to break COW, so it has no work to do here.
1417+ */
15041418 if (vma->vm_flags & VM_LOCKONFAULT)
1505− gup_flags &= ~FOLL_POPULATE;
1419+ return nr_pages;
1420+
1421+ gup_flags = FOLL_TOUCH;
15061422 /*
15071423 * We want to touch writable mappings with a write fault in order
15081424 * to break COW, except for shared mappings because these don't COW
@@ -1569,10 +1485,9 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, · 60 unmodified lines
15691485 * in the page table.
15701486 * FOLL_HWPOISON: Return -EHWPOISON instead of -EFAULT when we hit
15711487 * a poisoned page.
1572− * FOLL_POPULATE: Always populate memory with VM_LOCKONFAULT.
15731488 * !FOLL_FORCE: Require proper access permissions.
15741489 */
1575− gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK | FOLL_HWPOISON;
1490+ gup_flags = FOLL_TOUCH | FOLL_HWPOISON;
15761491 if (write)
15771492 gup_flags |= FOLL_WRITE;
15781493
@@ -1852,72 +1767,80 @@ static long check_and_migrate_movable_pages(unsigned long nr_pages, · 273 unmodified lines
18521767 struct page **pages,
18531768 unsigned int gup_flags)
18541769 {
1855− unsigned long i;
1856− unsigned long isolation_error_count = 0;
1857− bool drain_allow = true;
1770+ unsigned long isolation_error_count = 0, i;
1771+ struct folio *prev_folio = NULL;
18581772 LIST_HEAD(movable_page_list);
1859− long ret = 0;
1860− struct page *prev_head = NULL;
1861− struct page *head;
1862− struct migration_target_control mtc = {
1863− .nid = NUMA_NO_NODE,
1864− .gfp_mask = GFP_USER | __GFP_NOWARN,
1865− };
1773+ bool drain_allow = true;
1774+ int ret = 0;
18661775
18671776 for (i = 0; i < nr_pages; i++) {
1868− head = compound_head(pages[i]);
1869− if (head == prev_head)
1777+ struct folio *folio = page_folio(pages[i]);
1778+
1779+ if (folio == prev_folio)
18701780 continue;
1871− prev_head = head;
1781+ prev_folio = folio;
1782+
1783+ if (folio_is_pinnable(folio))
1784+ continue;
1785+
18721786 /*
1873− * If we get a movable page, since we are going to be pinning
1874− * these entries, try to move them out if possible.
1787+ * Try to move out any movable page before pinning the range.
18751788 */
1876− if (!is_pinnable_page(head)) {
1877− if (PageHuge(head)) {
1878− if (!isolate_huge_page(head, &movable_page_list))
1879− isolation_error_count++;
1880− } else {
1881− if (!PageLRU(head) && drain_allow) {
1882− lru_add_drain_all();
1883− drain_allow = false;
1884− }
1789+ if (folio_test_hugetlb(folio)) {
1790+ if (!isolate_huge_page(&folio->page,
1791+ &movable_page_list))
1792+ isolation_error_count++;
1793+ continue;
1794+ }
18851795
1886− if (isolate_lru_page(head)) {
1887− isolation_error_count++;
1888− continue;
1889− }
1890− list_add_tail(&head->lru, &movable_page_list);
1891− mod_node_page_state(page_pgdat(head),
1892− NR_ISOLATED_ANON +
1893− page_is_file_lru(head),
1894− thp_nr_pages(head));
1895− }
1796+ if (!folio_test_lru(folio) && drain_allow) {
1797+ lru_add_drain_all();
1798+ drain_allow = false;
1799+ }
1800+
1801+ if (folio_isolate_lru(folio)) {
1802+ isolation_error_count++;
1803+ continue;
18961804 }
1805+ list_add_tail(&folio->lru, &movable_page_list);
1806+ node_stat_mod_folio(folio,
1807+ NR_ISOLATED_ANON + folio_is_file_lru(folio),
1808+ folio_nr_pages(folio));
18971809 }
18981810
1811+ if (!list_empty(&movable_page_list) || isolation_error_count)
1812+ goto unpin_pages;
1813+
18991814 /*
19001815 * If list is empty, and no isolation errors, means that all pages are
19011816 * in the correct zone.
19021817 */
1903− if (list_empty(&movable_page_list) && !isolation_error_count)
1904− return nr_pages;
1818+ return nr_pages;
19051819
1820+unpin_pages:
19061821 if (gup_flags & FOLL_PIN) {
19071822 unpin_user_pages(pages, nr_pages);
19081823 } else {
19091824 for (i = 0; i < nr_pages; i++)
19101825 put_page(pages[i]);
19111826 }
1827+
19121828 if (!list_empty(&movable_page_list)) {
1829+ struct migration_target_control mtc = {
1830+ .nid = NUMA_NO_NODE,
1831+ .gfp_mask = GFP_USER | __GFP_NOWARN,
1832+ };
1833+
19131834 ret = migrate_pages(&movable_page_list, alloc_migration_target,
19141835 NULL, (unsigned long)&mtc, MIGRATE_SYNC,
19151836 MR_LONGTERM_PIN, NULL);
1916− if (ret && !list_empty(&movable_page_list))
1917− putback_movable_pages(&movable_page_list);
1837+ if (ret > 0) /* number of pages not migrated */
1838+ ret = -ENOMEM;
19181839 }
19191840
1920− return ret > 0 ? -ENOMEM : ret;
1841+ if (ret && !list_empty(&movable_page_list))
1842+ putback_movable_pages(&movable_page_list);
1843+ return ret;
19211844 }
19221845 #else
19231846 static long check_and_migrate_movable_pages(unsigned long nr_pages,
@@ -2227,7 +2150,8 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, · 303 unmodified lines
22272150 ptem = ptep = pte_offset_map(&pmd, addr);
22282151 do {
22292152 pte_t pte = ptep_get_lockless(ptep);
2230− struct page *head, *page;
2153+ struct page *page;
2154+ struct folio *folio;
22312155
22322156 /*
22332157 * Similar to the PMD case below, NUMA hinting must take slow
@@ -2254,22 +2178,20 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, · 20 unmodified lines
22542178 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
22552179 page = pte_page(pte);
22562180
2257− head = try_grab_compound_head(page, 1, flags);
2258− if (!head)
2181+ folio = try_grab_folio(page, 1, flags);
2182+ if (!folio)
22592183 goto pte_unmap;
22602184
22612185 if (unlikely(page_is_secretmem(page))) {
2262− put_compound_head(head, 1, flags);
2186+ gup_put_folio(folio, 1, flags);
22632187 goto pte_unmap;
22642188 }
22652189
22662190 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2267− put_compound_head(head, 1, flags);
2191+ gup_put_folio(folio, 1, flags);
22682192 goto pte_unmap;
22692193 }
22702194
2271− VM_BUG_ON_PAGE(compound_head(page) != head, page);
2272−
22732195 /*
22742196 * We need to make the page accessible if and only if we are
22752197 * going to access its content (the FOLL_PIN case). Please
@@ -2279,14 +2201,13 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, · 3 unmodified lines
22792201 if (flags & FOLL_PIN) {
22802202 ret = arch_make_page_accessible(page);
22812203 if (ret) {
2282− unpin_user_page(page);
2204+ gup_put_folio(folio, 1, flags);
22832205 goto pte_unmap;
22842206 }
22852207 }
2286− SetPageReferenced(page);
2208+ folio_set_referenced(folio);
22872209 pages[*nr] = page;
22882210 (*nr)++;
2289−
22902211 } while (ptep++, addr += PAGE_SIZE, addr != end);
22912212
22922213 ret = 1;
@@ -2403,8 +2324,8 @@ static int record_subpages(struct page *page, unsigned long addr, · 110 unmodified lines
24032324 {
24042325 int nr;
24052326
2406− for (nr = 0; addr != end; addr += PAGE_SIZE)
2407− pages[nr++] = page++;
2327+ for (nr = 0; addr != end; nr++, addr += PAGE_SIZE)
2328+ pages[nr] = nth_page(page, nr);
24082329
24092330 return nr;
24102331 }
@@ -2422,7 +2343,8 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, · 11 unmodified lines
24222343 struct page **pages, int *nr)
24232344 {
24242345 unsigned long pte_end;
2425− struct page *head, *page;
2346+ struct page *page;
2347+ struct folio *folio;
24262348 pte_t pte;
24272349 int refs;
24282350
@@ -2438,21 +2360,20 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, · 9 unmodified lines
24382360 /* hugepages are never "special" */
24392361 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
24402362
2441− head = pte_page(pte);
2442− page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2363+ page = nth_page(pte_page(pte), (addr & (sz - 1)) >> PAGE_SHIFT);
24432364 refs = record_subpages(page, addr, end, pages + *nr);
24442365
2445− head = try_grab_compound_head(head, refs, flags);
2446− if (!head)
2366+ folio = try_grab_folio(page, refs, flags);
2367+ if (!folio)
24472368 return 0;
24482369
24492370 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2450− put_compound_head(head, refs, flags);
2371+ gup_put_folio(folio, refs, flags);
24512372 return 0;
24522373 }
24532374
24542375 *nr += refs;
2455− SetPageReferenced(head);
2376+ folio_set_referenced(folio);
24562377 return 1;
24572378 }
24582379
@@ -2486,7 +2407,8 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, · 27 unmodified lines
24862407 unsigned long end, unsigned int flags,
24872408 struct page **pages, int *nr)
24882409 {
2489− struct page *head, *page;
2410+ struct page *page;
2411+ struct folio *folio;
24902412 int refs;
24912413
24922414 if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
@@ -2499,20 +2421,20 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, · 6 unmodified lines
24992421 pages, nr);
25002422 }
25012423
2502− page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2424+ page = nth_page(pmd_page(orig), (addr & ~PMD_MASK) >> PAGE_SHIFT);
25032425 refs = record_subpages(page, addr, end, pages + *nr);
25042426
2505− head = try_grab_compound_head(pmd_page(orig), refs, flags);
2506− if (!head)
2427+ folio = try_grab_folio(page, refs, flags);
2428+ if (!folio)
25072429 return 0;
25082430
25092431 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2510− put_compound_head(head, refs, flags);
2432+ gup_put_folio(folio, refs, flags);
25112433 return 0;
25122434 }
25132435
25142436 *nr += refs;
2515− SetPageReferenced(head);
2437+ folio_set_referenced(folio);
25162438 return 1;
25172439 }
25182440
@@ -2520,7 +2442,8 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, · 1 unmodified line
25202442 unsigned long end, unsigned int flags,
25212443 struct page **pages, int *nr)
25222444 {
2523− struct page *head, *page;
2445+ struct page *page;
2446+ struct folio *folio;
25242447 int refs;
25252448
25262449 if (!pud_access_permitted(orig, flags & FOLL_WRITE))
@@ -2533,20 +2456,20 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, · 6 unmodified lines
25332456 pages, nr);
25342457 }
25352458
2536− page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2459+ page = nth_page(pud_page(orig), (addr & ~PUD_MASK) >> PAGE_SHIFT);
25372460 refs = record_subpages(page, addr, end, pages + *nr);
25382461
2539− head = try_grab_compound_head(pud_page(orig), refs, flags);
2540− if (!head)
2462+ folio = try_grab_folio(page, refs, flags);
2463+ if (!folio)
25412464 return 0;
25422465
25432466 if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2544− put_compound_head(head, refs, flags);
2467+ gup_put_folio(folio, refs, flags);
25452468 return 0;
25462469 }
25472470
25482471 *nr += refs;
2549− SetPageReferenced(head);
2472+ folio_set_referenced(folio);
25502473 return 1;
25512474 }
25522475
@@ -2555,27 +2478,28 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, · 2 unmodified lines
25552478 struct page **pages, int *nr)
25562479 {
25572480 int refs;
2558− struct page *head, *page;
2481+ struct page *page;
2482+ struct folio *folio;
25592483
25602484 if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
25612485 return 0;
25622486
25632487 BUILD_BUG_ON(pgd_devmap(orig));
25642488
2565− page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2489+ page = nth_page(pgd_page(orig), (addr & ~PGDIR_MASK) >> PAGE_SHIFT);
25662490 refs = record_subpages(page, addr, end, pages + *nr);
25672491
2568− head = try_grab_compound_head(pgd_page(orig), refs, flags);
2569− if (!head)
2492+ folio = try_grab_folio(page, refs, flags);
2493+ if (!folio)
25702494 return 0;
25712495
25722496 if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
2573− put_compound_head(head, refs, flags);
2497+ gup_put_folio(folio, refs, flags);
25742498 return 0;
25752499 }
25762500
25772501 *nr += refs;
2578− SetPageReferenced(head);
2502+ folio_set_referenced(folio);
25792503 return 1;
25802504 }
25812505
mm/huge_memory.c+62−116
@@ -583,13 +583,10 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, · 582 unmodified lines
583583 unsigned long ret;
584584 loff_t off = (loff_t)pgoff << PAGE_SHIFT;
585585
586− if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD))
587− goto out;
588−
589586 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);
590587 if (ret)
591588 return ret;
592−out:
589+
593590 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
594591 }
595592 EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
@@ -1381,39 +1378,6 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, · 785 unmodified lines
13811378 if (flags & FOLL_TOUCH)
13821379 touch_pmd(vma, addr, pmd, flags);
13831380
1384− if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
1385− /*
1386− * We don't mlock() pte-mapped THPs. This way we can avoid
1387− * leaking mlocked pages into non-VM_LOCKED VMAs.
1388− *
1389− * For anon THP:
1390− *
1391− * In most cases the pmd is the only mapping of the page as we
1392− * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
1393− * writable private mappings in populate_vma_page_range().
1394− *
1395− * The only scenario when we have the page shared here is if we
1396− * mlocking read-only mapping shared over fork(). We skip
1397− * mlocking such pages.
1398− *
1399− * For file THP:
1400− *
1401− * We can expect PageDoubleMap() to be stable under page lock:
1402− * for file pages we set it in page_add_file_rmap(), which
1403− * requires page to be locked.
1404− */
1405−
1406− if (PageAnon(page) && compound_mapcount(page) != 1)
1407− goto skip_mlock;
1408− if (PageDoubleMap(page) || !page->mapping)
1409− goto skip_mlock;
1410− if (!trylock_page(page))
1411− goto skip_mlock;
1412− if (page->mapping && !PageDoubleMap(page))
1413− mlock_vma_page(page);
1414− unlock_page(page);
1415− }
1416−skip_mlock:
14171381 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
14181382 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
14191383
@@ -1611,7 +1575,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · 191 unmodified lines
16111575
16121576 if (pmd_present(orig_pmd)) {
16131577 page = pmd_page(orig_pmd);
1614− page_remove_rmap(page, true);
1578+ page_remove_rmap(page, vma, true);
16151579 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
16161580 VM_BUG_ON_PAGE(!PageHead(page), page);
16171581 } else if (thp_migration_supported()) {
@@ -2007,7 +1971,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, · 389 unmodified lines
20071971 set_page_dirty(page);
20081972 if (!PageReferenced(page) && pmd_young(old_pmd))
20091973 SetPageReferenced(page);
2010− page_remove_rmap(page, true);
1974+ page_remove_rmap(page, vma, true);
20111975 put_page(page);
20121976 }
20131977 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);
@@ -2141,6 +2105,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, · 127 unmodified lines
21412105 }
21422106 }
21432107 unlock_page_memcg(page);
2108+
2109+ /* Above is effectively page_remove_rmap(page, vma, true) */
2110+ munlock_vma_page(page, vma, true);
21442111 }
21452112
21462113 smp_wmb(); /* make pte visible before pmd */
@@ -2148,18 +2115,18 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, · 1 unmodified line
21482115
21492116 if (freeze) {
21502117 for (i = 0; i < HPAGE_PMD_NR; i++) {
2151− page_remove_rmap(page + i, false);
2118+ page_remove_rmap(page + i, vma, false);
21522119 put_page(page + i);
21532120 }
21542121 }
21552122 }
21562123
21572124 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2158− unsigned long address, bool freeze, struct page *page)
2125+ unsigned long address, bool freeze, struct folio *folio)
21592126 {
21602127 spinlock_t *ptl;
21612128 struct mmu_notifier_range range;
2162− bool do_unlock_page = false;
2129+ bool do_unlock_folio = false;
21632130 pmd_t _pmd;
21642131
21652132 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
@@ -2169,20 +2136,20 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, · 3 unmodified lines
21692136 ptl = pmd_lock(vma->vm_mm, pmd);
21702137
21712138 /*
2172− * If caller asks to setup a migration entries, we need a page to check
2173− * pmd against. Otherwise we can end up replacing wrong page.
2139+ * If caller asks to setup a migration entry, we need a folio to check
2140+ * pmd against. Otherwise we can end up replacing wrong folio.
21742141 */
2175− VM_BUG_ON(freeze && !page);
2176− if (page) {
2177− VM_WARN_ON_ONCE(!PageLocked(page));
2178− if (page != pmd_page(*pmd))
2142+ VM_BUG_ON(freeze && !folio);
2143+ if (folio) {
2144+ VM_WARN_ON_ONCE(!folio_test_locked(folio));
2145+ if (folio != page_folio(pmd_page(*pmd)))
21792146 goto out;
21802147 }
21812148
21822149 repeat:
21832150 if (pmd_trans_huge(*pmd)) {
2184− if (!page) {
2185− page = pmd_page(*pmd);
2151+ if (!folio) {
2152+ folio = page_folio(pmd_page(*pmd));
21862153 /*
21872154 * An anonymous page must be locked, to ensure that a
21882155 * concurrent reuse_swap_page() sees stable mapcount;
@@ -2190,33 +2157,31 @@ repeat: · 1 unmodified line
21902157 * and page lock must not be taken when zap_pmd_range()
21912158 * calls __split_huge_pmd() while i_mmap_lock is held.
21922159 */
2193− if (PageAnon(page)) {
2194− if (unlikely(!trylock_page(page))) {
2195− get_page(page);
2160+ if (folio_test_anon(folio)) {
2161+ if (unlikely(!folio_trylock(folio))) {
2162+ folio_get(folio);
21962163 _pmd = *pmd;
21972164 spin_unlock(ptl);
2198− lock_page(page);
2165+ folio_lock(folio);
21992166 spin_lock(ptl);
22002167 if (unlikely(!pmd_same(*pmd, _pmd))) {
2201− unlock_page(page);
2202− put_page(page);
2203− page = NULL;
2168+ folio_unlock(folio);
2169+ folio_put(folio);
2170+ folio = NULL;
22042171 goto repeat;
22052172 }
2206− put_page(page);
2173+ folio_put(folio);
22072174 }
2208− do_unlock_page = true;
2175+ do_unlock_folio = true;
22092176 }
22102177 }
2211− if (PageMlocked(page))
2212− clear_page_mlock(page);
22132178 } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))
22142179 goto out;
22152180 __split_huge_pmd_locked(vma, pmd, range.start, freeze);
22162181 out:
22172182 spin_unlock(ptl);
2218− if (do_unlock_page)
2219− unlock_page(page);
2183+ if (do_unlock_folio)
2184+ folio_unlock(folio);
22202185 /*
22212186 * No need to double call mmu_notifier->invalidate_range() callback.
22222187 * They are 3 cases to consider inside __split_huge_pmd_locked():
@@ -2234,7 +2199,7 @@ out: · 11 unmodified lines
22342199 }
22352200
22362201 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
2237− bool freeze, struct page *page)
2202+ bool freeze, struct folio *folio)
22382203 {
22392204 pgd_t *pgd;
22402205 p4d_t *p4d;
@@ -2255,7 +2220,7 @@ void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, · 14 unmodified lines
22552220
22562221 pmd = pmd_offset(pud, address);
22572222
2258− __split_huge_pmd(vma, pmd, address, freeze, page);
2223+ __split_huge_pmd(vma, pmd, address, freeze, folio);
22592224 }
22602225
22612226 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)
@@ -2295,6 +2260,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, · 33 unmodified lines
22952260
22962261 static void unmap_page(struct page *page)
22972262 {
2263+ struct folio *folio = page_folio(page);
22982264 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
22992265 TTU_SYNC;
23002266
@@ -2305,26 +2271,27 @@ static void unmap_page(struct page *page) · 4 unmodified lines
23052271 * pages can simply be left unmapped, then faulted back on demand.
23062272 * If that is ever changed (perhaps for mlock), update remap_page().
23072273 */
2308− if (PageAnon(page))
2309− try_to_migrate(page, ttu_flags);
2274+ if (folio_test_anon(folio))
2275+ try_to_migrate(folio, ttu_flags);
23102276 else
2311− try_to_unmap(page, ttu_flags | TTU_IGNORE_MLOCK);
2277+ try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);
23122278
23132279 VM_WARN_ON_ONCE_PAGE(page_mapped(page), page);
23142280 }
23152281
2316−static void remap_page(struct page *page, unsigned int nr)
2282+static void remap_page(struct folio *folio, unsigned long nr)
23172283 {
2318− int i;
2284+ int i = 0;
23192285
23202286 /* If unmap_page() uses try_to_migrate() on file, remove this check */
2321− if (!PageAnon(page))
2287+ if (!folio_test_anon(folio))
23222288 return;
2323− if (PageTransHuge(page)) {
2324− remove_migration_ptes(page, page, true);
2325− } else {
2326− for (i = 0; i < nr; i++)
2327− remove_migration_ptes(page + i, page + i, true);
2289+ for (;;) {
2290+ remove_migration_ptes(folio, folio, true);
2291+ i += folio_nr_pages(folio);
2292+ if (i >= nr)
2293+ break;
2294+ folio = folio_next(folio);
23282295 }
23292296 }
23302297
@@ -2344,8 +2311,11 @@ static void lru_add_page_tail(struct page *head, struct page *tail, · 13 unmodified lines
23442311 } else {
23452312 /* head is still on lru (and we have it frozen) */
23462313 VM_WARN_ON(!PageLRU(head));
2314+ if (PageUnevictable(tail))
2315+ tail->mlock_count = 0;
2316+ else
2317+ list_add_tail(&tail->lru, &head->lru);
23472318 SetPageLRU(tail);
2348− list_add_tail(&tail->lru, &head->lru);
23492319 }
23502320 }
23512321
@@ -2481,7 +2451,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 129 unmodified lines
24812451 }
24822452 local_irq_enable();
24832453
2484− remap_page(head, nr);
2454+ remap_page(folio, nr);
24852455
24862456 if (PageSwapCache(head)) {
24872457 swp_entry_t entry = { .val = page_private(head) };
@@ -2506,30 +2476,6 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 18 unmodified lines
25062476 }
25072477 }
25082478
2509−int total_mapcount(struct page *page)
2510−{
2511− int i, compound, nr, ret;
2512−
2513− VM_BUG_ON_PAGE(PageTail(page), page);
2514−
2515− if (likely(!PageCompound(page)))
2516− return atomic_read(&page->_mapcount) + 1;
2517−
2518− compound = compound_mapcount(page);
2519− nr = compound_nr(page);
2520− if (PageHuge(page))
2521− return compound;
2522− ret = compound;
2523− for (i = 0; i < nr; i++)
2524− ret += atomic_read(&page[i]._mapcount) + 1;
2525− /* File pages has compound_mapcount included in _mapcount */
2526− if (!PageAnon(page))
2527− return ret - compound * nr;
2528− if (PageDoubleMap(page))
2529− ret -= nr;
2530− return ret;
2531−}
2532−
25332479 /*
25342480 * This calculates accurately how many mappings a transparent hugepage
25352481 * has (unlike page_mapcount() which isn't fully accurate). This full
@@ -2579,18 +2525,19 @@ int page_trans_huge_mapcount(struct page *page) · 43 unmodified lines
25792525 }
25802526
25812527 /* Racy check whether the huge page can be split */
2582−bool can_split_huge_page(struct page *page, int *pextra_pins)
2528+bool can_split_folio(struct folio *folio, int *pextra_pins)
25832529 {
25842530 int extra_pins;
25852531
25862532 /* Additional pins from page cache */
2587− if (PageAnon(page))
2588− extra_pins = PageSwapCache(page) ? thp_nr_pages(page) : 0;
2533+ if (folio_test_anon(folio))
2534+ extra_pins = folio_test_swapcache(folio) ?
2535+ folio_nr_pages(folio) : 0;
25892536 else
2590− extra_pins = thp_nr_pages(page);
2537+ extra_pins = folio_nr_pages(folio);
25912538 if (pextra_pins)
25922539 *pextra_pins = extra_pins;
2593− return total_mapcount(page) == page_count(page) - extra_pins - 1;
2540+ return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 1;
25942541 }
25952542
25962543 /*
@@ -2614,7 +2561,8 @@ bool can_split_huge_page(struct page *page, int *pextra_pins) · 17 unmodified lines
26142561 */
26152562 int split_huge_page_to_list(struct page *page, struct list_head *list)
26162563 {
2617− struct page *head = compound_head(page);
2564+ struct folio *folio = page_folio(page);
2565+ struct page *head = &folio->page;
26182566 struct deferred_split *ds_queue = get_deferred_split_queue(head);
26192567 XA_STATE(xas, &head->mapping->i_pages, head->index);
26202568 struct anon_vma *anon_vma = NULL;
@@ -2634,7 +2582,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) · 13 unmodified lines
26342582 * The caller does not necessarily hold an mmap_lock that would
26352583 * prevent the anon_vma disappearing so we first we take a
26362584 * reference to it and then lock the anon_vma for write. This
2637− * is similar to page_lock_anon_vma_read except the write lock
2585+ * is similar to folio_lock_anon_vma_read except the write lock
26382586 * is taken to serialise against parallel split or collapse
26392587 * operations.
26402588 */
@@ -2681,7 +2629,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) · 40 unmodified lines
26812629 * Racy check if we can split the page, before unmap_page() will
26822630 * split PMDs
26832631 */
2684− if (!can_split_huge_page(head, &extra_pins)) {
2632+ if (!can_split_folio(folio, &extra_pins)) {
26852633 ret = -EBUSY;
26862634 goto out_unlock;
26872635 }
@@ -2731,7 +2679,7 @@ fail: · 43 unmodified lines
27312679 if (mapping)
27322680 xas_unlock(&xas);
27332681 local_irq_enable();
2734− remap_page(head, thp_nr_pages(head));
2682+ remap_page(folio, folio_nr_pages(folio));
27352683 ret = -EBUSY;
27362684 }
27372685
@@ -2988,7 +2936,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start, · 250 unmodified lines
29882936 goto next;
29892937
29902938 total++;
2991− if (!can_split_huge_page(compound_head(page), NULL))
2939+ if (!can_split_folio(page_folio(page), NULL))
29922940 goto next;
29932941
29942942 if (!trylock_page(page))
@@ -3181,7 +3129,7 @@ void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, · 186 unmodified lines
31813129 if (pmd_soft_dirty(pmdval))
31823130 pmdswp = pmd_swp_mksoft_dirty(pmdswp);
31833131 set_pmd_at(mm, address, pvmw->pmd, pmdswp);
3184− page_remove_rmap(page, true);
3132+ page_remove_rmap(page, vma, true);
31853133 put_page(page);
31863134 }
31873135
@@ -3210,10 +3158,8 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) · 22 unmodified lines
32103158 if (PageAnon(new))
32113159 page_add_anon_rmap(new, vma, mmun_start, true);
32123160 else
3213− page_add_file_rmap(new, true);
3161+ page_add_file_rmap(new, vma, true);
32143162 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);
3215− if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new))
3216− mlock_vma_page(new);
32173163
32183164 /* No need to invalidate - it was non-present before */
32193165 update_mmu_cache_pmd(vma, address, pvmw->pmd);
mm/hugetlb.c+9−6
@@ -1321,7 +1321,9 @@ static void __destroy_compound_gigantic_page(struct page *page, · 1320 unmodified lines
13211321 }
13221322
13231323 set_compound_order(page, 0);
1324+#ifdef CONFIG_64BIT
13241325 page[1].compound_nr = 0;
1326+#endif
13251327 __ClearPageHead(page);
13261328 }
13271329
@@ -1813,7 +1815,9 @@ out_error: · 485 unmodified lines
18131815 for (; j < nr_pages; j++, p = mem_map_next(p, page, j))
18141816 __ClearPageReserved(p);
18151817 set_compound_order(page, 0);
1818+#ifdef CONFIG_64BIT
18161819 page[1].compound_nr = 0;
1820+#endif
18171821 __ClearPageHead(page);
18181822 return false;
18191823 }
@@ -5013,7 +5017,7 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct · 3193 unmodified lines
50135017 set_page_dirty(page);
50145018
50155019 hugetlb_count_sub(pages_per_huge_page(h), mm);
5016− page_remove_rmap(page, true);
5020+ page_remove_rmap(page, vma, true);
50175021
50185022 spin_unlock(ptl);
50195023 tlb_remove_page_size(tlb, page, huge_page_size(h));
@@ -5258,7 +5262,7 @@ retry_avoidcopy: · 238 unmodified lines
52585262 /* Break COW */
52595263 huge_ptep_clear_flush(vma, haddr, ptep);
52605264 mmu_notifier_invalidate_range(mm, range.start, range.end);
5261− page_remove_rmap(old_page, true);
5265+ page_remove_rmap(old_page, vma, true);
52625266 hugepage_add_new_anon_rmap(new_page, vma, haddr);
52635267 set_huge_pte_at(mm, haddr, ptep,
52645268 make_huge_pte(vma, new_page, 1));
@@ -6074,7 +6078,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, · 809 unmodified lines
60746078
60756079 if (pages) {
60766080 /*
6077− * try_grab_compound_head() should always succeed here,
6081+ * try_grab_folio() should always succeed here,
60786082 * because: a) we hold the ptl lock, and b) we've just
60796083 * checked that the huge page is present in the page
60806084 * tables. If the huge page is present, then the tail
@@ -6083,9 +6087,8 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, · 2 unmodified lines
60836087 * any way. So this page must be available at this
60846088 * point, unless the page refcount overflowed:
60856089 */
6086− if (WARN_ON_ONCE(!try_grab_compound_head(pages[i],
6087− refs,
6088− flags))) {
6090+ if (WARN_ON_ONCE(!try_grab_folio(pages[i], refs,
6091+ flags))) {
60896092 spin_unlock(ptl);
60906093 remainder = 0;
60916094 err = -ENOMEM;
mm/internal.h+79−38
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010 #include <linux/fs.h>
1111 #include <linux/mm.h>
1212 #include <linux/pagemap.h>
13+#include <linux/rmap.h>
1314 #include <linux/tracepoint-defs.h>
1415
1516 struct folio_batch;
@@ -66,24 +67,20 @@ static inline void wake_throttle_isolated(pg_data_t *pgdat) · 50 unmodified lines
6667 vm_fault_t do_swap_page(struct vm_fault *vmf);
6768 void folio_rotate_reclaimable(struct folio *folio);
6869 bool __folio_end_writeback(struct folio *folio);
70+void deactivate_file_folio(struct folio *folio);
6971
7072 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
7173 unsigned long floor, unsigned long ceiling);
7274 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);
7375
74−static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
75−{
76− return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP));
77−}
78−
7976 struct zap_details;
8077 void unmap_page_range(struct mmu_gather *tlb,
8178 struct vm_area_struct *vma,
8279 unsigned long addr, unsigned long end,
8380 struct zap_details *details);
8481
85−void do_page_cache_ra(struct readahead_control *, unsigned long nr_to_read,
86− unsigned long lookahead_size);
82+void page_cache_ra_order(struct readahead_control *, struct file_ra_state *,
83+ unsigned int order);
8784 void force_page_cache_ra(struct readahead_control *, unsigned long nr);
8885 static inline void force_page_cache_readahead(struct address_space *mapping,
8986 struct file *file, pgoff_t index, unsigned long nr_to_read)
@@ -100,6 +97,9 @@ void filemap_free_folio(struct address_space *mapping, struct folio *folio); · 10 unmodified lines
10097 int truncate_inode_folio(struct address_space *mapping, struct folio *folio);
10198 bool truncate_inode_partial_folio(struct folio *folio, loff_t start,
10299 loff_t end);
100+long invalidate_inode_page(struct page *page);
101+unsigned long invalidate_mapping_pagevec(struct address_space *mapping,
102+ pgoff_t start, pgoff_t end, unsigned long *nr_pagevec);
103103
104104 /**
105105 * folio_evictable - Test whether a folio is evictable.
@@ -163,8 +163,10 @@ pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr, · 57 unmodified lines
163163 /*
164164 * in mm/vmscan.c:
165165 */
166−extern int isolate_lru_page(struct page *page);
167−extern void putback_lru_page(struct page *page);
166+int isolate_lru_page(struct page *page);
167+int folio_isolate_lru(struct folio *folio);
168+void putback_lru_page(struct page *page);
169+void folio_putback_lru(struct folio *folio);
168170 extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason);
169171
170172 /*
@@ -396,6 +398,7 @@ static inline bool is_data_mapping(vm_flags_t flags) · 225 unmodified lines
396398 void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
397399 struct vm_area_struct *prev);
398400 void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma);
401+struct anon_vma *folio_anon_vma(struct folio *folio);
399402
400403 #ifdef CONFIG_MMU
401404 void unmap_mapping_folio(struct folio *folio);
@@ -404,32 +407,56 @@ extern long populate_vma_page_range(struct vm_area_struct *vma, · 2 unmodified lines
404407 extern long faultin_vma_page_range(struct vm_area_struct *vma,
405408 unsigned long start, unsigned long end,
406409 bool write, int *locked);
407−extern void munlock_vma_pages_range(struct vm_area_struct *vma,
408− unsigned long start, unsigned long end);
409−static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
410−{
411− munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
412−}
413−
414−/*
415− * must be called with vma's mmap_lock held for read or write, and page locked.
416− */
417−extern void mlock_vma_page(struct page *page);
418−extern unsigned int munlock_vma_page(struct page *page);
419−
420410 extern int mlock_future_check(struct mm_struct *mm, unsigned long flags,
421411 unsigned long len);
422−
423412 /*
424− * Clear the page's PageMlocked(). This can be useful in a situation where
425− * we want to unconditionally remove a page from the pagecache -- e.g.,
426− * on truncation or freeing.
413+ * mlock_vma_page() and munlock_vma_page():
414+ * should be called with vma's mmap_lock held for read or write,
415+ * under page table lock for the pte/pmd being added or removed.
427416 *
428− * It is legal to call this function for any page, mlocked or not.
429− * If called for a page that is still mapped by mlocked vmas, all we do
430− * is revert to lazy LRU behaviour -- semantics are not broken.
417+ * mlock is usually called at the end of page_add_*_rmap(),
418+ * munlock at the end of page_remove_rmap(); but new anon
419+ * pages are managed by lru_cache_add_inactive_or_unevictable()
420+ * calling mlock_new_page().
421+ *
422+ * @compound is used to include pmd mappings of THPs, but filter out
423+ * pte mappings of THPs, which cannot be consistently counted: a pte
424+ * mapping of the THP head cannot be distinguished by the page alone.
431425 */
432−extern void clear_page_mlock(struct page *page);
426+void mlock_folio(struct folio *folio);
427+static inline void mlock_vma_folio(struct folio *folio,
428+ struct vm_area_struct *vma, bool compound)
429+{
430+ /*
431+ * The VM_SPECIAL check here serves two purposes.
432+ * 1) VM_IO check prevents migration from double-counting during mlock.
433+ * 2) Although mmap_region() and mlock_fixup() take care that VM_LOCKED
434+ * is never left set on a VM_SPECIAL vma, there is an interval while
435+ * file->f_op->mmap() is using vm_insert_page(s), when VM_LOCKED may
436+ * still be set while VM_SPECIAL bits are added: so ignore it then.
437+ */
438+ if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED) &&
439+ (compound || !folio_test_large(folio)))
440+ mlock_folio(folio);
441+}
442+
443+static inline void mlock_vma_page(struct page *page,
444+ struct vm_area_struct *vma, bool compound)
445+{
446+ mlock_vma_folio(page_folio(page), vma, compound);
447+}
448+
449+void munlock_page(struct page *page);
450+static inline void munlock_vma_page(struct page *page,
451+ struct vm_area_struct *vma, bool compound)
452+{
453+ if (unlikely(vma->vm_flags & VM_LOCKED) &&
454+ (compound || !PageTransCompound(page)))
455+ munlock_page(page);
456+}
457+void mlock_new_page(struct page *page);
458+bool need_mlock_page_drain(int cpu);
459+void mlock_page_drain(int cpu);
433460
434461 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
435462
@@ -463,18 +490,20 @@ vma_address(struct page *page, struct vm_area_struct *vma) · 27 unmodified lines
463490 }
464491
465492 /*
466− * Then at what user virtual address will none of the page be found in vma?
493+ * Then at what user virtual address will none of the range be found in vma?
467494 * Assumes that vma_address() already returned a good starting address.
468− * If page is a compound head, the entire compound page is considered.
469495 */
470−static inline unsigned long
471−vma_address_end(struct page *page, struct vm_area_struct *vma)
496+static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw)
472497 {
498+ struct vm_area_struct *vma = pvmw->vma;
473499 pgoff_t pgoff;
474500 unsigned long address;
475501
476− VM_BUG_ON_PAGE(PageKsm(page), page); /* KSM page->index unusable */
477− pgoff = page_to_pgoff(page) + compound_nr(page);
502+ /* Common case, plus ->pgoff is invalid for KSM */
503+ if (pvmw->nr_pages == 1)
504+ return pvmw->address + PAGE_SIZE;
505+
506+ pgoff = pvmw->pgoff + pvmw->nr_pages;
478507 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
479508 /* Check for address beyond vma (or wrapped through 0?) */
480509 if (address < vma->vm_start || address > vma->vm_end)
@@ -504,8 +533,13 @@ static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, · 23 unmodified lines
504533 }
505534 #else /* !CONFIG_MMU */
506535 static inline void unmap_mapping_folio(struct folio *folio) { }
507−static inline void clear_page_mlock(struct page *page) { }
508−static inline void mlock_vma_page(struct page *page) { }
536+static inline void mlock_vma_page(struct page *page,
537+ struct vm_area_struct *vma, bool compound) { }
538+static inline void munlock_vma_page(struct page *page,
539+ struct vm_area_struct *vma, bool compound) { }
540+static inline void mlock_new_page(struct page *page) { }
541+static inline bool need_mlock_page_drain(int cpu) { return false; }
542+static inline void mlock_page_drain(int cpu) { }
509543 static inline void vunmap_range_noflush(unsigned long start, unsigned long end)
510544 {
511545 }
@@ -713,6 +747,13 @@ void vunmap_range_noflush(unsigned long start, unsigned long end); · 201 unmodified lines
713747 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
714748 unsigned long addr, int page_nid, int *flags);
715749
750+void free_zone_device_page(struct page *page);
751+
752+/*
753+ * mm/gup.c
754+ */
755+struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags);
756+
716757 DECLARE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
717758
718759 #endif /* __MM_INTERNAL_H */
mm/khugepaged.c+11−4
@@ -774,7 +774,7 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page, · 773 unmodified lines
774774 */
775775 spin_lock(ptl);
776776 ptep_clear(vma->vm_mm, address, _pte);
777− page_remove_rmap(src_page, false);
777+ page_remove_rmap(src_page, vma, false);
778778 spin_unlock(ptl);
779779 free_page_and_swap_cache(src_page);
780780 }
@@ -1513,7 +1513,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) · 732 unmodified lines
15131513 if (pte_none(*pte))
15141514 continue;
15151515 page = vm_normal_page(vma, addr, *pte);
1516− page_remove_rmap(page, false);
1516+ page_remove_rmap(page, vma, false);
15171517 }
15181518
15191519 pte_unmap_unlock(start_pte, ptl);
@@ -1834,13 +1834,13 @@ static void collapse_file(struct mm_struct *mm, · 314 unmodified lines
18341834 }
18351835
18361836 if (page_mapped(page))
1837− unmap_mapping_pages(mapping, index, 1, false);
1837+ try_to_unmap(page_folio(page),
1838+ TTU_IGNORE_MLOCK | TTU_BATCH_FLUSH);
18381839
18391840 xas_lock_irq(&xas);
18401841 xas_set(&xas, index);
18411842
18421843 VM_BUG_ON_PAGE(page != xas_load(&xas), page);
1843− VM_BUG_ON_PAGE(page_mapped(page), page);
18441844
18451845 /*
18461846 * The page is expected to have page_count() == 3:
@@ -1904,6 +1904,13 @@ xa_locked: · 57 unmodified lines
19041904 xas_unlock_irq(&xas);
19051905 xa_unlocked:
19061906
1907+ /*
1908+ * If collapse is successful, flush must be done now before copying.
1909+ * If collapse is unsuccessful, does flush actually need to be done?
1910+ * Do it anyway, to clear the state.
1911+ */
1912+ try_to_unmap_flush();
1913+
19071914 if (result == SCAN_SUCCEED) {
19081915 struct page *page, *tmp;
19091916
mm/ksm.c+10−22
@@ -1034,10 +1034,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, · 1033 unmodified lines
10341034 pte_t *orig_pte)
10351035 {
10361036 struct mm_struct *mm = vma->vm_mm;
1037− struct page_vma_mapped_walk pvmw = {
1038− .page = page,
1039− .vma = vma,
1040− };
1037+ DEFINE_PAGE_VMA_WALK(pvmw, page, vma, 0, 0);
10411038 int swapped;
10421039 int err = -EFAULT;
10431040 struct mmu_notifier_range range;
@@ -1177,7 +1174,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, · 133 unmodified lines
11771174 ptep_clear_flush(vma, addr, ptep);
11781175 set_pte_at_notify(mm, addr, ptep, newpte);
11791176
1180− page_remove_rmap(page, false);
1177+ page_remove_rmap(page, vma, false);
11811178 if (!page_mapped(page))
11821179 try_to_free_swap(page);
11831180 put_page(page);
@@ -1252,16 +1249,6 @@ static int try_to_merge_one_page(struct vm_area_struct *vma, · 68 unmodified lines
12521249 err = replace_page(vma, page, kpage, orig_pte);
12531250 }
12541251
1255− if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {
1256− munlock_vma_page(page);
1257− if (!PageMlocked(kpage)) {
1258− unlock_page(page);
1259− lock_page(kpage);
1260− mlock_vma_page(kpage);
1261− page = kpage; /* for final unlock */
1262− }
1263− }
1264−
12651252 out_unlock:
12661253 unlock_page(page);
12671254 out:
@@ -2567,7 +2554,8 @@ void __ksm_exit(struct mm_struct *mm) · 1299 unmodified lines
25672554 struct page *ksm_might_need_to_copy(struct page *page,
25682555 struct vm_area_struct *vma, unsigned long address)
25692556 {
2570− struct anon_vma *anon_vma = page_anon_vma(page);
2557+ struct folio *folio = page_folio(page);
2558+ struct anon_vma *anon_vma = folio_anon_vma(folio);
25712559 struct page *new_page;
25722560
25732561 if (PageKsm(page)) {
@@ -2603,21 +2591,21 @@ struct page *ksm_might_need_to_copy(struct page *page, · 29 unmodified lines
26032591 return new_page;
26042592 }
26052593
2606−void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)
2594+void rmap_walk_ksm(struct folio *folio, const struct rmap_walk_control *rwc)
26072595 {
26082596 struct stable_node *stable_node;
26092597 struct rmap_item *rmap_item;
26102598 int search_new_forks = 0;
26112599
2612− VM_BUG_ON_PAGE(!PageKsm(page), page);
2600+ VM_BUG_ON_FOLIO(!folio_test_ksm(folio), folio);
26132601
26142602 /*
26152603 * Rely on the page lock to protect against concurrent modifications
26162604 * to that page's node of the stable tree.
26172605 */
2618− VM_BUG_ON_PAGE(!PageLocked(page), page);
2606+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
26192607
2620− stable_node = page_stable_node(page);
2608+ stable_node = folio_stable_node(folio);
26212609 if (!stable_node)
26222610 return;
26232611 again:
@@ -2652,11 +2640,11 @@ again: · 28 unmodified lines
26522640 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
26532641 continue;
26542642
2655− if (!rwc->rmap_one(page, vma, addr, rwc->arg)) {
2643+ if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) {
26562644 anon_vma_unlock_read(anon_vma);
26572645 return;
26582646 }
2659− if (rwc->done && rwc->done(page)) {
2647+ if (rwc->done && rwc->done(folio)) {
26602648 anon_vma_unlock_read(anon_vma);
26612649 return;
26622650 }
mm/madvise.c+5−0
@@ -502,6 +502,11 @@ static void madvise_cold_page_range(struct mmu_gather *tlb, · 501 unmodified lines
502502 tlb_end_vma(tlb, vma);
503503 }
504504
505+static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
506+{
507+ return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP));
508+}
509+
505510 static long madvise_cold(struct vm_area_struct *vma,
506511 struct vm_area_struct **prev,
507512 unsigned long start_addr, unsigned long end_addr)
mm/memcontrol.c+16−21
@@ -53,6 +53,7 @@ · 52 unmodified lines
5353 #include <linux/fs.h>
5454 #include <linux/seq_file.h>
5555 #include <linux/vmpressure.h>
56+#include <linux/memremap.h>
5657 #include <linux/mm_inline.h>
5758 #include <linux/swap_cgroup.h>
5859 #include <linux/cpu.h>
@@ -1271,8 +1272,7 @@ struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, · 1212 unmodified lines
12711272 * @nr_pages: positive when adding or negative when removing
12721273 *
12731274 * This function must be called under lru_lock, just before a page is added
1274− * to or just after a page is removed from an lru list (that ordering being
1275− * so as to allow it to check that lru_size 0 is consistent with list_empty).
1275+ * to or just after a page is removed from an lru list.
12761276 */
12771277 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
12781278 int zid, int nr_pages)
@@ -5436,17 +5436,12 @@ static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, · 4157 unmodified lines
54365436 return NULL;
54375437
54385438 /*
5439− * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
5440− * a device and because they are not accessible by CPU they are store
5441− * as special swap entry in the CPU page table.
5439+ * Handle device private pages that are not accessible by the CPU, but
5440+ * stored as special swap entries in the page table.
54425441 */
54435442 if (is_device_private_entry(ent)) {
54445443 page = pfn_swap_entry_to_page(ent);
5445− /*
5446− * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
5447− * a refcount of 1 when free (unlike normal page)
5448− */
5449− if (!page_ref_add_unless(page, 1, 1))
5444+ if (!get_page_unless_zero(page))
54505445 return NULL;
54515446 return page;
54525447 }
@@ -7053,19 +7048,19 @@ static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) · 1600 unmodified lines
70537048
70547049 /**
70557050 * mem_cgroup_swapout - transfer a memsw charge to swap
7056− * @page: page whose memsw charge to transfer
7051+ * @folio: folio whose memsw charge to transfer
70577052 * @entry: swap entry to move the charge to
70587053 *
7059− * Transfer the memsw charge of @page to @entry.
7054+ * Transfer the memsw charge of @folio to @entry.
70607055 */
7061−void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
7056+void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
70627057 {
70637058 struct mem_cgroup *memcg, *swap_memcg;
70647059 unsigned int nr_entries;
70657060 unsigned short oldid;
70667061
7067− VM_BUG_ON_PAGE(PageLRU(page), page);
7068− VM_BUG_ON_PAGE(page_count(page), page);
7062+ VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
7063+ VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
70697064
70707065 if (mem_cgroup_disabled())
70717066 return;
@@ -7073,9 +7068,9 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) · 1 unmodified line
70737068 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
70747069 return;
70757070
7076− memcg = page_memcg(page);
7071+ memcg = folio_memcg(folio);
70777072
7078− VM_WARN_ON_ONCE_PAGE(!memcg, page);
7073+ VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
70797074 if (!memcg)
70807075 return;
70817076
@@ -7085,16 +7080,16 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) · 3 unmodified lines
70857080 * ancestor for the swap instead and transfer the memory+swap charge.
70867081 */
70877082 swap_memcg = mem_cgroup_id_get_online(memcg);
7088− nr_entries = thp_nr_pages(page);
7083+ nr_entries = folio_nr_pages(folio);
70897084 /* Get references for the tail pages, too */
70907085 if (nr_entries > 1)
70917086 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
70927087 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
70937088 nr_entries);
7094− VM_BUG_ON_PAGE(oldid, page);
7089+ VM_BUG_ON_FOLIO(oldid, folio);
70957090 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
70967091
7097− page->memcg_data = 0;
7092+ folio->memcg_data = 0;
70987093
70997094 if (!mem_cgroup_is_root(memcg))
71007095 page_counter_uncharge(&memcg->memory, nr_entries);
@@ -7114,7 +7109,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) · 13 unmodified lines
71147109 memcg_stats_lock();
71157110 mem_cgroup_charge_statistics(memcg, -nr_entries);
71167111 memcg_stats_unlock();
7117− memcg_check_events(memcg, page_to_nid(page));
7112+ memcg_check_events(memcg, folio_nid(folio));
71187113
71197114 css_put(&memcg->css);
71207115 }
mm/memory-failure.c+7−5
@@ -478,12 +478,13 @@ static struct task_struct *task_early_kill(struct task_struct *tsk, · 477 unmodified lines
478478 static void collect_procs_anon(struct page *page, struct list_head *to_kill,
479479 int force_early)
480480 {
481+ struct folio *folio = page_folio(page);
481482 struct vm_area_struct *vma;
482483 struct task_struct *tsk;
483484 struct anon_vma *av;
484485 pgoff_t pgoff;
485486
486− av = page_lock_anon_vma_read(page);
487+ av = folio_lock_anon_vma_read(folio);
487488 if (av == NULL) /* Not actually mapped anymore */
488489 return;
489490
@@ -1347,6 +1348,7 @@ static int get_hwpoison_page(struct page *p, unsigned long flags) · 857 unmodified lines
13471348 static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
13481349 int flags, struct page *hpage)
13491350 {
1351+ struct folio *folio = page_folio(hpage);
13501352 enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_SYNC;
13511353 struct address_space *mapping;
13521354 LIST_HEAD(tokill);
@@ -1421,12 +1423,12 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, · 68 unmodified lines
14211423 */
14221424 mapping = hugetlb_page_mapping_lock_write(hpage);
14231425 if (mapping) {
1424− try_to_unmap(hpage, ttu|TTU_RMAP_LOCKED);
1426+ try_to_unmap(folio, ttu|TTU_RMAP_LOCKED);
14251427 i_mmap_unlock_write(mapping);
14261428 } else
14271429 pr_info("Memory failure: %#lx: could not lock mapping for mapped huge page\n", pfn);
14281430 } else {
1429− try_to_unmap(hpage, ttu);
1431+ try_to_unmap(folio, ttu);
14301432 }
14311433
14321434 unmap_success = !page_mapped(hpage);
@@ -2169,7 +2171,7 @@ static bool isolate_page(struct page *page, struct list_head *pagelist) · 736 unmodified lines
21692171 */
21702172 static int __soft_offline_page(struct page *page)
21712173 {
2172− int ret = 0;
2174+ long ret = 0;
21732175 unsigned long pfn = page_to_pfn(page);
21742176 struct page *hpage = compound_head(page);
21752177 char const *msg_page[] = {"page", "hugepage"};
@@ -2216,7 +2218,7 @@ static int __soft_offline_page(struct page *page) · 40 unmodified lines
22162218 if (!list_empty(&pagelist))
22172219 putback_movable_pages(&pagelist);
22182220
2219− pr_info("soft offline: %#lx: %s migration failed %d, type %pGp\n",
2221+ pr_info("soft offline: %#lx: %s migration failed %ld, type %pGp\n",
22202222 pfn, msg_page[huge], ret, &page->flags);
22212223 if (ret > 0)
22222224 ret = -EBUSY;
mm/memory.c+15−28
@@ -735,9 +735,6 @@ static void restore_exclusive_pte(struct vm_area_struct *vma, · 734 unmodified lines
735735
736736 set_pte_at(vma->vm_mm, address, ptep, pte);
737737
738− if (vma->vm_flags & VM_LOCKED)
739− mlock_vma_page(page);
740−
741738 /*
742739 * No need to invalidate - it was non-present before. However
743740 * secondary CPUs may have mappings that need invalidating.
@@ -1389,7 +1386,7 @@ again: · 645 unmodified lines
13891386 mark_page_accessed(page);
13901387 }
13911388 rss[mm_counter(page)]--;
1392− page_remove_rmap(page, false);
1389+ page_remove_rmap(page, vma, false);
13931390 if (unlikely(page_mapcount(page) < 0))
13941391 print_bad_pte(vma, addr, ptent, page);
13951392 if (unlikely(__tlb_remove_page(tlb, page))) {
@@ -1408,7 +1405,7 @@ again: · 12 unmodified lines
14081405 continue;
14091406 rss[mm_counter(page)]--;
14101407 if (is_device_private_entry(entry))
1411− page_remove_rmap(page, false);
1408+ page_remove_rmap(page, vma, false);
14121409 put_page(page);
14131410 } else if (!non_swap_entry(entry)) {
14141411 /* Genuine swap entry, hence a private anon page */
@@ -1763,16 +1760,16 @@ static int validate_page_before_insert(struct page *page) · 348 unmodified lines
17631760 return 0;
17641761 }
17651762
1766−static int insert_page_into_pte_locked(struct mm_struct *mm, pte_t *pte,
1763+static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
17671764 unsigned long addr, struct page *page, pgprot_t prot)
17681765 {
17691766 if (!pte_none(*pte))
17701767 return -EBUSY;
17711768 /* Ok, finally just insert the thing.. */
17721769 get_page(page);
1773− inc_mm_counter_fast(mm, mm_counter_file(page));
1774− page_add_file_rmap(page, false);
1775− set_pte_at(mm, addr, pte, mk_pte(page, prot));
1770+ inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
1771+ page_add_file_rmap(page, vma, false);
1772+ set_pte_at(vma->vm_mm, addr, pte, mk_pte(page, prot));
17761773 return 0;
17771774 }
17781775
@@ -1786,7 +1783,6 @@ static int insert_page_into_pte_locked(struct mm_struct *mm, pte_t *pte, · 7 unmodified lines
17861783 static int insert_page(struct vm_area_struct *vma, unsigned long addr,
17871784 struct page *page, pgprot_t prot)
17881785 {
1789− struct mm_struct *mm = vma->vm_mm;
17901786 int retval;
17911787 pte_t *pte;
17921788 spinlock_t *ptl;
@@ -1795,17 +1791,17 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, · 2 unmodified lines
17951791 if (retval)
17961792 goto out;
17971793 retval = -ENOMEM;
1798− pte = get_locked_pte(mm, addr, &ptl);
1794+ pte = get_locked_pte(vma->vm_mm, addr, &ptl);
17991795 if (!pte)
18001796 goto out;
1801− retval = insert_page_into_pte_locked(mm, pte, addr, page, prot);
1797+ retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
18021798 pte_unmap_unlock(pte, ptl);
18031799 out:
18041800 return retval;
18051801 }
18061802
18071803 #ifdef pte_index
1808−static int insert_page_in_batch_locked(struct mm_struct *mm, pte_t *pte,
1804+static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
18091805 unsigned long addr, struct page *page, pgprot_t prot)
18101806 {
18111807 int err;
@@ -1815,7 +1811,7 @@ static int insert_page_in_batch_locked(struct mm_struct *mm, pte_t *pte, · 3 unmodified lines
18151811 err = validate_page_before_insert(page);
18161812 if (err)
18171813 return err;
1818− return insert_page_into_pte_locked(mm, pte, addr, page, prot);
1814+ return insert_page_into_pte_locked(vma, pte, addr, page, prot);
18191815 }
18201816
18211817 /* insert_pages() amortizes the cost of spinlock operations
@@ -1852,7 +1848,7 @@ more: · 30 unmodified lines
18521848
18531849 start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
18541850 for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
1855− int err = insert_page_in_batch_locked(mm, pte,
1851+ int err = insert_page_in_batch_locked(vma, pte,
18561852 addr, pages[curr_page_idx], prot);
18571853 if (unlikely(err)) {
18581854 pte_unmap_unlock(start_pte, pte_lock);
@@ -3108,7 +3104,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) · 1249 unmodified lines
31083104 * mapcount is visible. So transitively, TLBs to
31093105 * old page will be flushed before it can be reused.
31103106 */
3111− page_remove_rmap(old_page, false);
3107+ page_remove_rmap(old_page, vma, false);
31123108 }
31133109
31143110 /* Free the old page.. */
@@ -3128,16 +3124,6 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) · 13 unmodified lines
31283124 */
31293125 mmu_notifier_invalidate_range_only_end(&range);
31303126 if (old_page) {
3131− /*
3132− * Don't let another task, with possibly unlocked vma,
3133− * keep the mlocked page.
3134− */
3135− if (page_copied && (vma->vm_flags & VM_LOCKED)) {
3136− lock_page(old_page); /* LRU manipulation */
3137− if (PageMlocked(old_page))
3138− munlock_vma_page(old_page);
3139− unlock_page(old_page);
3140− }
31413127 if (page_copied)
31423128 free_swap_cache(old_page);
31433129 put_page(old_page);
@@ -3958,7 +3944,8 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) · 814 unmodified lines
39583944 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
39593945
39603946 add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
3961− page_add_file_rmap(page, true);
3947+ page_add_file_rmap(page, vma, true);
3948+
39623949 /*
39633950 * deposit and withdraw with pmd lock held
39643951 */
@@ -4007,7 +3994,7 @@ void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr) · 42 unmodified lines
40073994 lru_cache_add_inactive_or_unevictable(page, vma);
40083995 } else {
40093996 inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
4010− page_add_file_rmap(page, false);
3997+ page_add_file_rmap(page, vma, false);
40113998 }
40123999 set_pte_at(vma->vm_mm, addr, vmf->pte, entry);
40134000 }
mm/memory_hotplug.c+8−5
@@ -1617,10 +1617,13 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · 1616 unmodified lines
16171617 DEFAULT_RATELIMIT_BURST);
16181618
16191619 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1620+ struct folio *folio;
1621+
16201622 if (!pfn_valid(pfn))
16211623 continue;
16221624 page = pfn_to_page(pfn);
1623− head = compound_head(page);
1625+ folio = page_folio(page);
1626+ head = &folio->page;
16241627
16251628 if (PageHuge(page)) {
16261629 pfn = page_to_pfn(head) + compound_nr(head) - 1;
@@ -1637,10 +1640,10 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · 10 unmodified lines
16371640 * the unmap as the catch all safety net).
16381641 */
16391642 if (PageHWPoison(page)) {
1640− if (WARN_ON(PageLRU(page)))
1641− isolate_lru_page(page);
1642− if (page_mapped(page))
1643− try_to_unmap(page, TTU_IGNORE_MLOCK);
1643+ if (WARN_ON(folio_test_lru(folio)))
1644+ folio_isolate_lru(folio);
1645+ if (folio_mapped(folio))
1646+ try_to_unmap(folio, TTU_IGNORE_MLOCK);
16441647 continue;
16451648 }
16461649
mm/memremap.c+34−31
@@ -4,7 +4,7 @@ · 3 unmodified lines
44 #include <linux/io.h>
55 #include <linux/kasan.h>
66 #include <linux/memory_hotplug.h>
7−#include <linux/mm.h>
7+#include <linux/memremap.h>
88 #include <linux/pfn_t.h>
99 #include <linux/swap.h>
1010 #include <linux/mmzone.h>
@@ -12,6 +12,7 @@ · 1 unmodified line
1212 #include <linux/types.h>
1313 #include <linux/wait_bit.h>
1414 #include <linux/xarray.h>
15+#include "internal.h"
1516
1617 static DEFINE_XARRAY(pgmap_array);
1718
@@ -37,21 +38,19 @@ unsigned long memremap_compat_align(void) · 19 unmodified lines
3738 EXPORT_SYMBOL_GPL(memremap_compat_align);
3839 #endif
3940
40−#ifdef CONFIG_DEV_PAGEMAP_OPS
41+#ifdef CONFIG_FS_DAX
4142 DEFINE_STATIC_KEY_FALSE(devmap_managed_key);
4243 EXPORT_SYMBOL(devmap_managed_key);
4344
4445 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
4546 {
46− if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
47− pgmap->type == MEMORY_DEVICE_FS_DAX)
47+ if (pgmap->type == MEMORY_DEVICE_FS_DAX)
4848 static_branch_dec(&devmap_managed_key);
4949 }
5050
5151 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)
5252 {
53− if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
54− pgmap->type == MEMORY_DEVICE_FS_DAX)
53+ if (pgmap->type == MEMORY_DEVICE_FS_DAX)
5554 static_branch_inc(&devmap_managed_key);
5655 }
5756 #else
@@ -61,7 +60,7 @@ static void devmap_managed_enable_get(struct dev_pagemap *pgmap) · 3 unmodified lines
6160 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
6261 {
6362 }
64−#endif /* CONFIG_DEV_PAGEMAP_OPS */
63+#endif /* CONFIG_FS_DAX */
6564
6665 static void pgmap_array_delete(struct range *range)
6766 {
@@ -102,23 +101,12 @@ static unsigned long pfn_end(struct dev_pagemap *pgmap, int range_id) · 34 unmodified lines
102101 return (range->start + range_len(range)) >> PAGE_SHIFT;
103102 }
104103
105−static unsigned long pfn_next(struct dev_pagemap *pgmap, unsigned long pfn)
106−{
107− if (pfn % (1024 << pgmap->vmemmap_shift))
108− cond_resched();
109− return pfn + pgmap_vmemmap_nr(pgmap);
110−}
111−
112104 static unsigned long pfn_len(struct dev_pagemap *pgmap, unsigned long range_id)
113105 {
114106 return (pfn_end(pgmap, range_id) -
115107 pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift;
116108 }
117109
118−#define for_each_device_pfn(pfn, map, i) \
119− for (pfn = pfn_first(map, i); pfn < pfn_end(map, i); \
120− pfn = pfn_next(map, pfn))
121−
122110 static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
123111 {
124112 struct range *range = &pgmap->ranges[range_id];
@@ -147,13 +135,11 @@ static void pageunmap_range(struct dev_pagemap *pgmap, int range_id) · 22 unmodified lines
147135
148136 void memunmap_pages(struct dev_pagemap *pgmap)
149137 {
150− unsigned long pfn;
151138 int i;
152139
153140 percpu_ref_kill(&pgmap->ref);
154141 for (i = 0; i < pgmap->nr_range; i++)
155− for_each_device_pfn(pfn, pgmap, i)
156− put_page(pfn_to_page(pfn));
142+ percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i));
157143 wait_for_completion(&pgmap->done);
158144 percpu_ref_exit(&pgmap->ref);
159145
@@ -329,8 +315,7 @@ void *memremap_pages(struct dev_pagemap *pgmap, int nid) · 169 unmodified lines
329315 }
330316 break;
331317 case MEMORY_DEVICE_FS_DAX:
332− if (!IS_ENABLED(CONFIG_ZONE_DEVICE) ||
333− IS_ENABLED(CONFIG_FS_DAX_LIMITED)) {
318+ if (IS_ENABLED(CONFIG_FS_DAX_LIMITED)) {
334319 WARN(1, "File system DAX not supported\n");
335320 return ERR_PTR(-EINVAL);
336321 }
@@ -466,21 +451,17 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, · 129 unmodified lines
466451 }
467452 EXPORT_SYMBOL_GPL(get_dev_pagemap);
468453
469−#ifdef CONFIG_DEV_PAGEMAP_OPS
470−void free_devmap_managed_page(struct page *page)
454+void free_zone_device_page(struct page *page)
471455 {
472− /* notify page idle for dax */
473− if (!is_device_private_page(page)) {
474− wake_up_var(&page->_refcount);
456+ if (WARN_ON_ONCE(!page->pgmap->ops || !page->pgmap->ops->page_free))
475457 return;
476− }
477458
478459 __ClearPageWaiters(page);
479460
480461 mem_cgroup_uncharge(page_folio(page));
481462
482463 /*
483− * When a device_private page is freed, the page->mapping field
464+ * When a device managed page is freed, the page->mapping field
484465 * may still contain a (stale) mapping value. For example, the
485466 * lower bits of page->mapping may still identify the page as an
486467 * anonymous page. Ultimately, this entire field is just stale
@@ -502,5 +483,27 @@ void free_devmap_managed_page(struct page *page) · 15 unmodified lines
502483 */
503484 page->mapping = NULL;
504485 page->pgmap->ops->page_free(page);
486+
487+ /*
488+ * Reset the page count to 1 to prepare for handing out the page again.
489+ */
490+ set_page_count(page, 1);
491+}
492+
493+#ifdef CONFIG_FS_DAX
494+bool __put_devmap_managed_page(struct page *page)
495+{
496+ if (page->pgmap->type != MEMORY_DEVICE_FS_DAX)
497+ return false;
498+
499+ /*
500+ * fsdax page refcounts are 1-based, rather than 0-based: if
501+ * refcount is 1, then the page is free and the refcount is
502+ * stable because nobody holds a reference on the page.
503+ */
504+ if (page_ref_dec_return(page) == 1)
505+ wake_up_var(&page->_refcount);
506+ return true;
505507 }
506−#endif /* CONFIG_DEV_PAGEMAP_OPS */
508+EXPORT_SYMBOL(__put_devmap_managed_page);
509+#endif /* CONFIG_FS_DAX */
mm/migrate.c+56−814
@@ -38,12 +38,10 @@ · 37 unmodified lines
3838 #include <linux/hugetlb.h>
3939 #include <linux/hugetlb_cgroup.h>
4040 #include <linux/gfp.h>
41−#include <linux/pagewalk.h>
4241 #include <linux/pfn_t.h>
4342 #include <linux/memremap.h>
4443 #include <linux/userfaultfd_k.h>
4544 #include <linux/balloon_compaction.h>
46−#include <linux/mmu_notifier.h>
4745 #include <linux/page_idle.h>
4846 #include <linux/page_owner.h>
4947 #include <linux/sched/mm.h>
@@ -174,37 +172,33 @@ void putback_movable_pages(struct list_head *l) · 124 unmodified lines
174172 /*
175173 * Restore a potential migration pte to a working pte entry
176174 */
177−static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
178− unsigned long addr, void *old)
175+static bool remove_migration_pte(struct folio *folio,
176+ struct vm_area_struct *vma, unsigned long addr, void *old)
179177 {
180− struct page_vma_mapped_walk pvmw = {
181− .page = old,
182− .vma = vma,
183− .address = addr,
184− .flags = PVMW_SYNC | PVMW_MIGRATION,
185− };
186− struct page *new;
187− pte_t pte;
188− swp_entry_t entry;
178+ DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION);
189179
190− VM_BUG_ON_PAGE(PageTail(page), page);
191180 while (page_vma_mapped_walk(&pvmw)) {
192− if (PageKsm(page))
193− new = page;
194− else
195− new = page - pvmw.page->index +
196− linear_page_index(vma, pvmw.address);
181+ pte_t pte;
182+ swp_entry_t entry;
183+ struct page *new;
184+ unsigned long idx = 0;
185+
186+ /* pgoff is invalid for ksm pages, but they are never large */
187+ if (folio_test_large(folio) && !folio_test_hugetlb(folio))
188+ idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff;
189+ new = folio_page(folio, idx);
197190
198191 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
199192 /* PMD-mapped THP migration entry */
200193 if (!pvmw.pte) {
201− VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
194+ VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
195+ !folio_test_pmd_mappable(folio), folio);
202196 remove_migration_pmd(&pvmw, new);
203197 continue;
204198 }
205199 #endif
206200
207− get_page(new);
201+ folio_get(folio);
208202 pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
209203 if (pte_swp_soft_dirty(*pvmw.pte))
210204 pte = pte_mksoft_dirty(pte);
@@ -233,12 +227,12 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, · 22 unmodified lines
233227 }
234228
235229 #ifdef CONFIG_HUGETLB_PAGE
236− if (PageHuge(new)) {
230+ if (folio_test_hugetlb(folio)) {
237231 unsigned int shift = huge_page_shift(hstate_vma(vma));
238232
239233 pte = pte_mkhuge(pte);
240234 pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
241− if (PageAnon(new))
235+ if (folio_test_anon(folio))
242236 hugepage_add_anon_rmap(new, vma, pvmw.address);
243237 else
244238 page_dup_rmap(new, true);
@@ -246,17 +240,14 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, · 1 unmodified line
246240 } else
247241 #endif
248242 {
249− if (PageAnon(new))
243+ if (folio_test_anon(folio))
250244 page_add_anon_rmap(new, vma, pvmw.address, false);
251245 else
252− page_add_file_rmap(new, false);
246+ page_add_file_rmap(new, vma, false);
253247 set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
254248 }
255− if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
256− mlock_vma_page(new);
257−
258− if (PageTransHuge(page) && PageMlocked(page))
259− clear_page_mlock(page);
249+ if (vma->vm_flags & VM_LOCKED)
250+ mlock_page_drain(smp_processor_id());
260251
261252 /* No need to invalidate - it was non-present before */
262253 update_mmu_cache(vma, pvmw.address, pvmw.pte);
@@ -269,17 +260,17 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, · 6 unmodified lines
269260 * Get rid of all migration entries and replace them by
270261 * references to the indicated page.
271262 */
272−void remove_migration_ptes(struct page *old, struct page *new, bool locked)
263+void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked)
273264 {
274265 struct rmap_walk_control rwc = {
275266 .rmap_one = remove_migration_pte,
276− .arg = old,
267+ .arg = src,
277268 };
278269
279270 if (locked)
280− rmap_walk_locked(new, &rwc);
271+ rmap_walk_locked(dst, &rwc);
281272 else
282− rmap_walk(new, &rwc);
273+ rmap_walk(dst, &rwc);
283274 }
284275
285276 /*
@@ -342,14 +333,8 @@ static int expected_page_refs(struct address_space *mapping, struct page *page) · 56 unmodified lines
342333 {
343334 int expected_count = 1;
344335
345− /*
346− * Device private pages have an extra refcount as they are
347− * ZONE_DEVICE pages.
348− */
349− expected_count += is_device_private_page(page);
350336 if (mapping)
351337 expected_count += compound_nr(page) + page_has_private(page);
352−
353338 return expected_count;
354339 }
355340
@@ -772,6 +757,7 @@ int buffer_migrate_page_norefs(struct address_space *mapping, · 416 unmodified lines
772757 */
773758 static int writeout(struct address_space *mapping, struct page *page)
774759 {
760+ struct folio *folio = page_folio(page);
775761 struct writeback_control wbc = {
776762 .sync_mode = WB_SYNC_NONE,
777763 .nr_to_write = 1,
@@ -797,7 +783,7 @@ static int writeout(struct address_space *mapping, struct page *page) · 19 unmodified lines
797783 * At this point we know that the migration attempt cannot
798784 * be successful.
799785 */
800− remove_migration_ptes(page, page, false);
786+ remove_migration_ptes(folio, folio, false);
801787
802788 rc = mapping->a_ops->writepage(page, &wbc);
803789
@@ -927,6 +913,8 @@ out: · 123 unmodified lines
927913 static int __unmap_and_move(struct page *page, struct page *newpage,
928914 int force, enum migrate_mode mode)
929915 {
916+ struct folio *folio = page_folio(page);
917+ struct folio *dst = page_folio(newpage);
930918 int rc = -EAGAIN;
931919 bool page_was_mapped = false;
932920 struct anon_vma *anon_vma = NULL;
@@ -1030,16 +1018,31 @@ static int __unmap_and_move(struct page *page, struct page *newpage, · 97 unmodified lines
10301018 /* Establish migration ptes */
10311019 VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
10321020 page);
1033− try_to_migrate(page, 0);
1021+ try_to_migrate(folio, 0);
10341022 page_was_mapped = true;
10351023 }
10361024
10371025 if (!page_mapped(page))
10381026 rc = move_to_new_page(newpage, page, mode);
10391027
1028+ /*
1029+ * When successful, push newpage to LRU immediately: so that if it
1030+ * turns out to be an mlocked page, remove_migration_ptes() will
1031+ * automatically build up the correct newpage->mlock_count for it.
1032+ *
1033+ * We would like to do something similar for the old page, when
1034+ * unsuccessful, and other cases when a page has been temporarily
1035+ * isolated from the unevictable LRU: but this case is the easiest.
1036+ */
1037+ if (rc == MIGRATEPAGE_SUCCESS) {
1038+ lru_cache_add(newpage);
1039+ if (page_was_mapped)
1040+ lru_add_drain();
1041+ }
1042+
10401043 if (page_was_mapped)
1041− remove_migration_ptes(page,
1042− rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
1044+ remove_migration_ptes(folio,
1045+ rc == MIGRATEPAGE_SUCCESS ? dst : folio, false);
10431046
10441047 out_unlock_both:
10451048 unlock_page(newpage);
@@ -1050,20 +1053,12 @@ out_unlock: · 4 unmodified lines
10501053 unlock_page(page);
10511054 out:
10521055 /*
1053− * If migration is successful, decrease refcount of the newpage
1056+ * If migration is successful, decrease refcount of the newpage,
10541057 * which will not free the page because new page owner increased
1055− * refcounter. As well, if it is LRU page, add the page to LRU
1056− * list in here. Use the old state of the isolated source page to
1057− * determine if we migrated a LRU page. newpage was already unlocked
1058− * and possibly modified by its owner - don't rely on the page
1059− * state.
1058+ * refcounter.
10601059 */
1061− if (rc == MIGRATEPAGE_SUCCESS) {
1062− if (unlikely(!is_lru))
1063− put_page(newpage);
1064− else
1065− putback_lru_page(newpage);
1066− }
1060+ if (rc == MIGRATEPAGE_SUCCESS)
1061+ put_page(newpage);
10671062
10681063 return rc;
10691064 }
@@ -1173,6 +1168,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 103 unmodified lines
11731168 enum migrate_mode mode, int reason,
11741169 struct list_head *ret)
11751170 {
1171+ struct folio *dst, *src = page_folio(hpage);
11761172 int rc = -EAGAIN;
11771173 int page_was_mapped = 0;
11781174 struct page *new_hpage;
@@ -1200,6 +1196,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 21 unmodified lines
12001196 new_hpage = get_new_page(hpage, private);
12011197 if (!new_hpage)
12021198 return -ENOMEM;
1199+ dst = page_folio(new_hpage);
12031200
12041201 if (!trylock_page(hpage)) {
12051202 if (!force)
@@ -1249,7 +1246,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 43 unmodified lines
12491246 ttu |= TTU_RMAP_LOCKED;
12501247 }
12511248
1252− try_to_migrate(hpage, ttu);
1249+ try_to_migrate(src, ttu);
12531250 page_was_mapped = 1;
12541251
12551252 if (mapping_locked)
@@ -1260,8 +1257,8 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 4 unmodified lines
12601257 rc = move_to_new_page(new_hpage, hpage, mode);
12611258
12621259 if (page_was_mapped)
1263− remove_migration_ptes(hpage,
1264− rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1260+ remove_migration_ptes(src,
1261+ rc == MIGRATEPAGE_SUCCESS ? dst : src, false);
12651262
12661263 unlock_put_anon:
12671264 unlock_page(new_hpage);
@@ -2147,761 +2144,6 @@ out: · 879 unmodified lines
21472144 #endif /* CONFIG_NUMA_BALANCING */
21482145 #endif /* CONFIG_NUMA */
21492146
2150−#ifdef CONFIG_DEVICE_PRIVATE
2151−static int migrate_vma_collect_skip(unsigned long start,
2152− unsigned long end,
2153− struct mm_walk *walk)
2154−{
2155− struct migrate_vma *migrate = walk->private;
2156− unsigned long addr;
2157−
2158− for (addr = start; addr < end; addr += PAGE_SIZE) {
2159− migrate->dst[migrate->npages] = 0;
2160− migrate->src[migrate->npages++] = 0;
2161− }
2162−
2163− return 0;
2164−}
2165−
2166−static int migrate_vma_collect_hole(unsigned long start,
2167− unsigned long end,
2168− __always_unused int depth,
2169− struct mm_walk *walk)
2170−{
2171− struct migrate_vma *migrate = walk->private;
2172− unsigned long addr;
2173−
2174− /* Only allow populating anonymous memory. */
2175− if (!vma_is_anonymous(walk->vma))
2176− return migrate_vma_collect_skip(start, end, walk);
2177−
2178− for (addr = start; addr < end; addr += PAGE_SIZE) {
2179− migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
2180− migrate->dst[migrate->npages] = 0;
2181− migrate->npages++;
2182− migrate->cpages++;
2183− }
2184−
2185− return 0;
2186−}
2187−
2188−static int migrate_vma_collect_pmd(pmd_t *pmdp,
2189− unsigned long start,
2190− unsigned long end,
2191− struct mm_walk *walk)
2192−{
2193− struct migrate_vma *migrate = walk->private;
2194− struct vm_area_struct *vma = walk->vma;
2195− struct mm_struct *mm = vma->vm_mm;
2196− unsigned long addr = start, unmapped = 0;
2197− spinlock_t *ptl;
2198− pte_t *ptep;
2199−
2200−again:
2201− if (pmd_none(*pmdp))
2202− return migrate_vma_collect_hole(start, end, -1, walk);
2203−
2204− if (pmd_trans_huge(*pmdp)) {
2205− struct page *page;
2206−
2207− ptl = pmd_lock(mm, pmdp);
2208− if (unlikely(!pmd_trans_huge(*pmdp))) {
2209− spin_unlock(ptl);
2210− goto again;
2211− }
2212−
2213− page = pmd_page(*pmdp);
2214− if (is_huge_zero_page(page)) {
2215− spin_unlock(ptl);
2216− split_huge_pmd(vma, pmdp, addr);
2217− if (pmd_trans_unstable(pmdp))
2218− return migrate_vma_collect_skip(start, end,
2219− walk);
2220− } else {
2221− int ret;
2222−
2223− get_page(page);
2224− spin_unlock(ptl);
2225− if (unlikely(!trylock_page(page)))
2226− return migrate_vma_collect_skip(start, end,
2227− walk);
2228− ret = split_huge_page(page);
2229− unlock_page(page);
2230− put_page(page);
2231− if (ret)
2232− return migrate_vma_collect_skip(start, end,
2233− walk);
2234− if (pmd_none(*pmdp))
2235− return migrate_vma_collect_hole(start, end, -1,
2236− walk);
2237− }
2238− }
2239−
2240− if (unlikely(pmd_bad(*pmdp)))
2241− return migrate_vma_collect_skip(start, end, walk);
2242−
2243− ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2244− arch_enter_lazy_mmu_mode();
2245−
2246− for (; addr < end; addr += PAGE_SIZE, ptep++) {
2247− unsigned long mpfn = 0, pfn;
2248− struct page *page;
2249− swp_entry_t entry;
2250− pte_t pte;
2251−
2252− pte = *ptep;
2253−
2254− if (pte_none(pte)) {
2255− if (vma_is_anonymous(vma)) {
2256− mpfn = MIGRATE_PFN_MIGRATE;
2257− migrate->cpages++;
2258− }
2259− goto next;
2260− }
2261−
2262− if (!pte_present(pte)) {
2263− /*
2264− * Only care about unaddressable device page special
2265− * page table entry. Other special swap entries are not
2266− * migratable, and we ignore regular swapped page.
2267− */
2268− entry = pte_to_swp_entry(pte);
2269− if (!is_device_private_entry(entry))
2270− goto next;
2271−
2272− page = pfn_swap_entry_to_page(entry);
2273− if (!(migrate->flags &
2274− MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
2275− page->pgmap->owner != migrate->pgmap_owner)
2276− goto next;
2277−
2278− mpfn = migrate_pfn(page_to_pfn(page)) |
2279− MIGRATE_PFN_MIGRATE;
2280− if (is_writable_device_private_entry(entry))
2281− mpfn |= MIGRATE_PFN_WRITE;
2282− } else {
2283− if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
2284− goto next;
2285− pfn = pte_pfn(pte);
2286− if (is_zero_pfn(pfn)) {
2287− mpfn = MIGRATE_PFN_MIGRATE;
2288− migrate->cpages++;
2289− goto next;
2290− }
2291− page = vm_normal_page(migrate->vma, addr, pte);
2292− mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
2293− mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
2294− }
2295−
2296− /* FIXME support THP */
2297− if (!page || !page->mapping || PageTransCompound(page)) {
2298− mpfn = 0;
2299− goto next;
2300− }
2301−
2302− /*
2303− * By getting a reference on the page we pin it and that blocks
2304− * any kind of migration. Side effect is that it "freezes" the
2305− * pte.
2306− *
2307− * We drop this reference after isolating the page from the lru
2308− * for non device page (device page are not on the lru and thus
2309− * can't be dropped from it).
2310− */
2311− get_page(page);
2312−
2313− /*
2314− * Optimize for the common case where page is only mapped once
2315− * in one process. If we can lock the page, then we can safely
2316− * set up a special migration page table entry now.
2317− */
2318− if (trylock_page(page)) {
2319− pte_t swp_pte;
2320−
2321− migrate->cpages++;
2322− ptep_get_and_clear(mm, addr, ptep);
2323−
2324− /* Setup special migration page table entry */
2325− if (mpfn & MIGRATE_PFN_WRITE)
2326− entry = make_writable_migration_entry(
2327− page_to_pfn(page));
2328− else
2329− entry = make_readable_migration_entry(
2330− page_to_pfn(page));
2331− swp_pte = swp_entry_to_pte(entry);
2332− if (pte_present(pte)) {
2333− if (pte_soft_dirty(pte))
2334− swp_pte = pte_swp_mksoft_dirty(swp_pte);
2335− if (pte_uffd_wp(pte))
2336− swp_pte = pte_swp_mkuffd_wp(swp_pte);
2337− } else {
2338− if (pte_swp_soft_dirty(pte))
2339− swp_pte = pte_swp_mksoft_dirty(swp_pte);
2340− if (pte_swp_uffd_wp(pte))
2341− swp_pte = pte_swp_mkuffd_wp(swp_pte);
2342− }
2343− set_pte_at(mm, addr, ptep, swp_pte);
2344−
2345− /*
2346− * This is like regular unmap: we remove the rmap and
2347− * drop page refcount. Page won't be freed, as we took
2348− * a reference just above.
2349− */
2350− page_remove_rmap(page, false);
2351− put_page(page);
2352−
2353− if (pte_present(pte))
2354− unmapped++;
2355− } else {
2356− put_page(page);
2357− mpfn = 0;
2358− }
2359−
2360−next:
2361− migrate->dst[migrate->npages] = 0;
2362− migrate->src[migrate->npages++] = mpfn;
2363− }
2364− arch_leave_lazy_mmu_mode();
2365− pte_unmap_unlock(ptep - 1, ptl);
2366−
2367− /* Only flush the TLB if we actually modified any entries */
2368− if (unmapped)
2369− flush_tlb_range(walk->vma, start, end);
2370−
2371− return 0;
2372−}
2373−
2374−static const struct mm_walk_ops migrate_vma_walk_ops = {
2375− .pmd_entry = migrate_vma_collect_pmd,
2376− .pte_hole = migrate_vma_collect_hole,
2377−};
2378−
2379−/*
2380− * migrate_vma_collect() - collect pages over a range of virtual addresses
2381− * @migrate: migrate struct containing all migration information
2382− *
2383− * This will walk the CPU page table. For each virtual address backed by a
2384− * valid page, it updates the src array and takes a reference on the page, in
2385− * order to pin the page until we lock it and unmap it.
2386− */
2387−static void migrate_vma_collect(struct migrate_vma *migrate)
2388−{
2389− struct mmu_notifier_range range;
2390−
2391− /*
2392− * Note that the pgmap_owner is passed to the mmu notifier callback so
2393− * that the registered device driver can skip invalidating device
2394− * private page mappings that won't be migrated.
2395− */
2396− mmu_notifier_range_init_owner(&range, MMU_NOTIFY_MIGRATE, 0,
2397− migrate->vma, migrate->vma->vm_mm, migrate->start, migrate->end,
2398− migrate->pgmap_owner);
2399− mmu_notifier_invalidate_range_start(&range);
2400−
2401− walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
2402− &migrate_vma_walk_ops, migrate);
2403−
2404− mmu_notifier_invalidate_range_end(&range);
2405− migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
2406−}
2407−
2408−/*
2409− * migrate_vma_check_page() - check if page is pinned or not
2410− * @page: struct page to check
2411− *
2412− * Pinned pages cannot be migrated. This is the same test as in
2413− * folio_migrate_mapping(), except that here we allow migration of a
2414− * ZONE_DEVICE page.
2415− */
2416−static bool migrate_vma_check_page(struct page *page)
2417−{
2418− /*
2419− * One extra ref because caller holds an extra reference, either from
2420− * isolate_lru_page() for a regular page, or migrate_vma_collect() for
2421− * a device page.
2422− */
2423− int extra = 1;
2424−
2425− /*
2426− * FIXME support THP (transparent huge page), it is bit more complex to
2427− * check them than regular pages, because they can be mapped with a pmd
2428− * or with a pte (split pte mapping).
2429− */
2430− if (PageCompound(page))
2431− return false;
2432−
2433− /* Page from ZONE_DEVICE have one extra reference */
2434− if (is_zone_device_page(page))
2435− extra++;
2436−
2437− /* For file back page */
2438− if (page_mapping(page))
2439− extra += 1 + page_has_private(page);
2440−
2441− if ((page_count(page) - extra) > page_mapcount(page))
2442− return false;
2443−
2444− return true;
2445−}
2446−
2447−/*
2448− * migrate_vma_unmap() - replace page mapping with special migration pte entry
2449− * @migrate: migrate struct containing all migration information
2450− *
2451− * Isolate pages from the LRU and replace mappings (CPU page table pte) with a
2452− * special migration pte entry and check if it has been pinned. Pinned pages are
2453− * restored because we cannot migrate them.
2454− *
2455− * This is the last step before we call the device driver callback to allocate
2456− * destination memory and copy contents of original page over to new page.
2457− */
2458−static void migrate_vma_unmap(struct migrate_vma *migrate)
2459−{
2460− const unsigned long npages = migrate->npages;
2461− unsigned long i, restore = 0;
2462− bool allow_drain = true;
2463−
2464− lru_add_drain();
2465−
2466− for (i = 0; i < npages; i++) {
2467− struct page *page = migrate_pfn_to_page(migrate->src[i]);
2468−
2469− if (!page)
2470− continue;
2471−
2472− /* ZONE_DEVICE pages are not on LRU */
2473− if (!is_zone_device_page(page)) {
2474− if (!PageLRU(page) && allow_drain) {
2475− /* Drain CPU's pagevec */
2476− lru_add_drain_all();
2477− allow_drain = false;
2478− }
2479−
2480− if (isolate_lru_page(page)) {
2481− migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2482− migrate->cpages--;
2483− restore++;
2484− continue;
2485− }
2486−
2487− /* Drop the reference we took in collect */
2488− put_page(page);
2489− }
2490−
2491− if (page_mapped(page))
2492− try_to_migrate(page, 0);
2493−
2494− if (page_mapped(page) || !migrate_vma_check_page(page)) {
2495− if (!is_zone_device_page(page)) {
2496− get_page(page);
2497− putback_lru_page(page);
2498− }
2499−
2500− migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2501− migrate->cpages--;
2502− restore++;
2503− continue;
2504− }
2505− }
2506−
2507− for (i = 0; i < npages && restore; i++) {
2508− struct page *page = migrate_pfn_to_page(migrate->src[i]);
2509−
2510− if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2511− continue;
2512−
2513− remove_migration_ptes(page, page, false);
2514−
2515− migrate->src[i] = 0;
2516− unlock_page(page);
2517− put_page(page);
2518− restore--;
2519− }
2520−}
2521−
2522−/**
2523− * migrate_vma_setup() - prepare to migrate a range of memory
2524− * @args: contains the vma, start, and pfns arrays for the migration
2525− *
2526− * Returns: negative errno on failures, 0 when 0 or more pages were migrated
2527− * without an error.
2528− *
2529− * Prepare to migrate a range of memory virtual address range by collecting all
2530− * the pages backing each virtual address in the range, saving them inside the
2531− * src array. Then lock those pages and unmap them. Once the pages are locked
2532− * and unmapped, check whether each page is pinned or not. Pages that aren't
2533− * pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
2534− * corresponding src array entry. Then restores any pages that are pinned, by
2535− * remapping and unlocking those pages.
2536− *
2537− * The caller should then allocate destination memory and copy source memory to
2538− * it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
2539− * flag set). Once these are allocated and copied, the caller must update each
2540− * corresponding entry in the dst array with the pfn value of the destination
2541− * page and with MIGRATE_PFN_VALID. Destination pages must be locked via
2542− * lock_page().
2543− *
2544− * Note that the caller does not have to migrate all the pages that are marked
2545− * with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
2546− * device memory to system memory. If the caller cannot migrate a device page
2547− * back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
2548− * consequences for the userspace process, so it must be avoided if at all
2549− * possible.
2550− *
2551− * For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
2552− * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
2553− * allowing the caller to allocate device memory for those unbacked virtual
2554− * addresses. For this the caller simply has to allocate device memory and
2555− * properly set the destination entry like for regular migration. Note that
2556− * this can still fail, and thus inside the device driver you must check if the
2557− * migration was successful for those entries after calling migrate_vma_pages(),
2558− * just like for regular migration.
2559− *
2560− * After that, the callers must call migrate_vma_pages() to go over each entry
2561− * in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
2562− * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
2563− * then migrate_vma_pages() to migrate struct page information from the source
2564− * struct page to the destination struct page. If it fails to migrate the
2565− * struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
2566− * src array.
2567− *
2568− * At this point all successfully migrated pages have an entry in the src
2569− * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
2570− * array entry with MIGRATE_PFN_VALID flag set.
2571− *
2572− * Once migrate_vma_pages() returns the caller may inspect which pages were
2573− * successfully migrated, and which were not. Successfully migrated pages will
2574− * have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
2575− *
2576− * It is safe to update device page table after migrate_vma_pages() because
2577− * both destination and source page are still locked, and the mmap_lock is held
2578− * in read mode (hence no one can unmap the range being migrated).
2579− *
2580− * Once the caller is done cleaning up things and updating its page table (if it
2581− * chose to do so, this is not an obligation) it finally calls
2582− * migrate_vma_finalize() to update the CPU page table to point to new pages
2583− * for successfully migrated pages or otherwise restore the CPU page table to
2584− * point to the original source pages.
2585− */
2586−int migrate_vma_setup(struct migrate_vma *args)
2587−{
2588− long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
2589−
2590− args->start &= PAGE_MASK;
2591− args->end &= PAGE_MASK;
2592− if (!args->vma || is_vm_hugetlb_page(args->vma) ||
2593− (args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
2594− return -EINVAL;
2595− if (nr_pages <= 0)
2596− return -EINVAL;
2597− if (args->start < args->vma->vm_start ||
2598− args->start >= args->vma->vm_end)
2599− return -EINVAL;
2600− if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
2601− return -EINVAL;
2602− if (!args->src || !args->dst)
2603− return -EINVAL;
2604−
2605− memset(args->src, 0, sizeof(*args->src) * nr_pages);
2606− args->cpages = 0;
2607− args->npages = 0;
2608−
2609− migrate_vma_collect(args);
2610−
2611− if (args->cpages)
2612− migrate_vma_unmap(args);
2613−
2614− /*
2615− * At this point pages are locked and unmapped, and thus they have
2616− * stable content and can safely be copied to destination memory that
2617− * is allocated by the drivers.
2618− */
2619− return 0;
2620−
2621−}
2622−EXPORT_SYMBOL(migrate_vma_setup);
2623−
2624−/*
2625− * This code closely matches the code in:
2626− * __handle_mm_fault()
2627− * handle_pte_fault()
2628− * do_anonymous_page()
2629− * to map in an anonymous zero page but the struct page will be a ZONE_DEVICE
2630− * private page.
2631− */
2632−static void migrate_vma_insert_page(struct migrate_vma *migrate,
2633− unsigned long addr,
2634− struct page *page,
2635− unsigned long *src)
2636−{
2637− struct vm_area_struct *vma = migrate->vma;
2638− struct mm_struct *mm = vma->vm_mm;
2639− bool flush = false;
2640− spinlock_t *ptl;
2641− pte_t entry;
2642− pgd_t *pgdp;
2643− p4d_t *p4dp;
2644− pud_t *pudp;
2645− pmd_t *pmdp;
2646− pte_t *ptep;
2647−
2648− /* Only allow populating anonymous memory */
2649− if (!vma_is_anonymous(vma))
2650− goto abort;
2651−
2652− pgdp = pgd_offset(mm, addr);
2653− p4dp = p4d_alloc(mm, pgdp, addr);
2654− if (!p4dp)
2655− goto abort;
2656− pudp = pud_alloc(mm, p4dp, addr);
2657− if (!pudp)
2658− goto abort;
2659− pmdp = pmd_alloc(mm, pudp, addr);
2660− if (!pmdp)
2661− goto abort;
2662−
2663− if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
2664− goto abort;
2665−
2666− /*
2667− * Use pte_alloc() instead of pte_alloc_map(). We can't run
2668− * pte_offset_map() on pmds where a huge pmd might be created
2669− * from a different thread.
2670− *
2671− * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
2672− * parallel threads are excluded by other means.
2673− *
2674− * Here we only have mmap_read_lock(mm).
2675− */
2676− if (pte_alloc(mm, pmdp))
2677− goto abort;
2678−
2679− /* See the comment in pte_alloc_one_map() */
2680− if (unlikely(pmd_trans_unstable(pmdp)))
2681− goto abort;
2682−
2683− if (unlikely(anon_vma_prepare(vma)))
2684− goto abort;
2685− if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))
2686− goto abort;
2687−
2688− /*
2689− * The memory barrier inside __SetPageUptodate makes sure that
2690− * preceding stores to the page contents become visible before
2691− * the set_pte_at() write.
2692− */
2693− __SetPageUptodate(page);
2694−
2695− if (is_zone_device_page(page)) {
2696− if (is_device_private_page(page)) {
2697− swp_entry_t swp_entry;
2698−
2699− if (vma->vm_flags & VM_WRITE)
2700− swp_entry = make_writable_device_private_entry(
2701− page_to_pfn(page));
2702− else
2703− swp_entry = make_readable_device_private_entry(
2704− page_to_pfn(page));
2705− entry = swp_entry_to_pte(swp_entry);
2706− } else {
2707− /*
2708− * For now we only support migrating to un-addressable
2709− * device memory.
2710− */
2711− pr_warn_once("Unsupported ZONE_DEVICE page type.\n");
2712− goto abort;
2713− }
2714− } else {
2715− entry = mk_pte(page, vma->vm_page_prot);
2716− if (vma->vm_flags & VM_WRITE)
2717− entry = pte_mkwrite(pte_mkdirty(entry));
2718− }
2719−
2720− ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2721−
2722− if (check_stable_address_space(mm))
2723− goto unlock_abort;
2724−
2725− if (pte_present(*ptep)) {
2726− unsigned long pfn = pte_pfn(*ptep);
2727−
2728− if (!is_zero_pfn(pfn))
2729− goto unlock_abort;
2730− flush = true;
2731− } else if (!pte_none(*ptep))
2732− goto unlock_abort;
2733−
2734− /*
2735− * Check for userfaultfd but do not deliver the fault. Instead,
2736− * just back off.
2737− */
2738− if (userfaultfd_missing(vma))
2739− goto unlock_abort;
2740−
2741− inc_mm_counter(mm, MM_ANONPAGES);
2742− page_add_new_anon_rmap(page, vma, addr, false);
2743− if (!is_zone_device_page(page))
2744− lru_cache_add_inactive_or_unevictable(page, vma);
2745− get_page(page);
2746−
2747− if (flush) {
2748− flush_cache_page(vma, addr, pte_pfn(*ptep));
2749− ptep_clear_flush_notify(vma, addr, ptep);
2750− set_pte_at_notify(mm, addr, ptep, entry);
2751− update_mmu_cache(vma, addr, ptep);
2752− } else {
2753− /* No need to invalidate - it was non-present before */
2754− set_pte_at(mm, addr, ptep, entry);
2755− update_mmu_cache(vma, addr, ptep);
2756− }
2757−
2758− pte_unmap_unlock(ptep, ptl);
2759− *src = MIGRATE_PFN_MIGRATE;
2760− return;
2761−
2762−unlock_abort:
2763− pte_unmap_unlock(ptep, ptl);
2764−abort:
2765− *src &= ~MIGRATE_PFN_MIGRATE;
2766−}
2767−
2768−/**
2769− * migrate_vma_pages() - migrate meta-data from src page to dst page
2770− * @migrate: migrate struct containing all migration information
2771− *
2772− * This migrates struct page meta-data from source struct page to destination
2773− * struct page. This effectively finishes the migration from source page to the
2774− * destination page.
2775− */
2776−void migrate_vma_pages(struct migrate_vma *migrate)
2777−{
2778− const unsigned long npages = migrate->npages;
2779− const unsigned long start = migrate->start;
2780− struct mmu_notifier_range range;
2781− unsigned long addr, i;
2782− bool notified = false;
2783−
2784− for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
2785− struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2786− struct page *page = migrate_pfn_to_page(migrate->src[i]);
2787− struct address_space *mapping;
2788− int r;
2789−
2790− if (!newpage) {
2791− migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2792− continue;
2793− }
2794−
2795− if (!page) {
2796− if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2797− continue;
2798− if (!notified) {
2799− notified = true;
2800−
2801− mmu_notifier_range_init_owner(&range,
2802− MMU_NOTIFY_MIGRATE, 0, migrate->vma,
2803− migrate->vma->vm_mm, addr, migrate->end,
2804− migrate->pgmap_owner);
2805− mmu_notifier_invalidate_range_start(&range);
2806− }
2807− migrate_vma_insert_page(migrate, addr, newpage,
2808− &migrate->src[i]);
2809− continue;
2810− }
2811−
2812− mapping = page_mapping(page);
2813−
2814− if (is_zone_device_page(newpage)) {
2815− if (is_device_private_page(newpage)) {
2816− /*
2817− * For now only support private anonymous when
2818− * migrating to un-addressable device memory.
2819− */
2820− if (mapping) {
2821− migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2822− continue;
2823− }
2824− } else {
2825− /*
2826− * Other types of ZONE_DEVICE page are not
2827− * supported.
2828− */
2829− migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2830− continue;
2831− }
2832− }
2833−
2834− r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
2835− if (r != MIGRATEPAGE_SUCCESS)
2836− migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2837− }
2838−
2839− /*
2840− * No need to double call mmu_notifier->invalidate_range() callback as
2841− * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
2842− * did already call it.
2843− */
2844− if (notified)
2845− mmu_notifier_invalidate_range_only_end(&range);
2846−}
2847−EXPORT_SYMBOL(migrate_vma_pages);
2848−
2849−/**
2850− * migrate_vma_finalize() - restore CPU page table entry
2851− * @migrate: migrate struct containing all migration information
2852− *
2853− * This replaces the special migration pte entry with either a mapping to the
2854− * new page if migration was successful for that page, or to the original page
2855− * otherwise.
2856− *
2857− * This also unlocks the pages and puts them back on the lru, or drops the extra
2858− * refcount, for device pages.
2859− */
2860−void migrate_vma_finalize(struct migrate_vma *migrate)
2861−{
2862− const unsigned long npages = migrate->npages;
2863− unsigned long i;
2864−
2865− for (i = 0; i < npages; i++) {
2866− struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2867− struct page *page = migrate_pfn_to_page(migrate->src[i]);
2868−
2869− if (!page) {
2870− if (newpage) {
2871− unlock_page(newpage);
2872− put_page(newpage);
2873− }
2874− continue;
2875− }
2876−
2877− if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
2878− if (newpage) {
2879− unlock_page(newpage);
2880− put_page(newpage);
2881− }
2882− newpage = page;
2883− }
2884−
2885− remove_migration_ptes(page, newpage, false);
2886− unlock_page(page);
2887−
2888− if (is_zone_device_page(page))
2889− put_page(page);
2890− else
2891− putback_lru_page(page);
2892−
2893− if (newpage != page) {
2894− unlock_page(newpage);
2895− if (is_zone_device_page(newpage))
2896− put_page(newpage);
2897− else
2898− putback_lru_page(newpage);
2899− }
2900− }
2901−}
2902−EXPORT_SYMBOL(migrate_vma_finalize);
2903−#endif /* CONFIG_DEVICE_PRIVATE */
2904−
29052147 /*
29062148 * node_demotion[] example:
29072149 *
mm/migrate_device.c+773−0
@@ -0,0 +1,773 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Device Memory Migration functionality.
4+ *
5+ * Originally written by Jérôme Glisse.
6+ */
7+#include <linux/export.h>
8+#include <linux/memremap.h>
9+#include <linux/migrate.h>
10+#include <linux/mm_inline.h>
11+#include <linux/mmu_notifier.h>
12+#include <linux/oom.h>
13+#include <linux/pagewalk.h>
14+#include <linux/rmap.h>
15+#include <linux/swapops.h>
16+#include <asm/tlbflush.h>
17+#include "internal.h"
18+
19+static int migrate_vma_collect_skip(unsigned long start,
20+ unsigned long end,
21+ struct mm_walk *walk)
22+{
23+ struct migrate_vma *migrate = walk->private;
24+ unsigned long addr;
25+
26+ for (addr = start; addr < end; addr += PAGE_SIZE) {
27+ migrate->dst[migrate->npages] = 0;
28+ migrate->src[migrate->npages++] = 0;
29+ }
30+
31+ return 0;
32+}
33+
34+static int migrate_vma_collect_hole(unsigned long start,
35+ unsigned long end,
36+ __always_unused int depth,
37+ struct mm_walk *walk)
38+{
39+ struct migrate_vma *migrate = walk->private;
40+ unsigned long addr;
41+
42+ /* Only allow populating anonymous memory. */
43+ if (!vma_is_anonymous(walk->vma))
44+ return migrate_vma_collect_skip(start, end, walk);
45+
46+ for (addr = start; addr < end; addr += PAGE_SIZE) {
47+ migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
48+ migrate->dst[migrate->npages] = 0;
49+ migrate->npages++;
50+ migrate->cpages++;
51+ }
52+
53+ return 0;
54+}
55+
56+static int migrate_vma_collect_pmd(pmd_t *pmdp,
57+ unsigned long start,
58+ unsigned long end,
59+ struct mm_walk *walk)
60+{
61+ struct migrate_vma *migrate = walk->private;
62+ struct vm_area_struct *vma = walk->vma;
63+ struct mm_struct *mm = vma->vm_mm;
64+ unsigned long addr = start, unmapped = 0;
65+ spinlock_t *ptl;
66+ pte_t *ptep;
67+
68+again:
69+ if (pmd_none(*pmdp))
70+ return migrate_vma_collect_hole(start, end, -1, walk);
71+
72+ if (pmd_trans_huge(*pmdp)) {
73+ struct page *page;
74+
75+ ptl = pmd_lock(mm, pmdp);
76+ if (unlikely(!pmd_trans_huge(*pmdp))) {
77+ spin_unlock(ptl);
78+ goto again;
79+ }
80+
81+ page = pmd_page(*pmdp);
82+ if (is_huge_zero_page(page)) {
83+ spin_unlock(ptl);
84+ split_huge_pmd(vma, pmdp, addr);
85+ if (pmd_trans_unstable(pmdp))
86+ return migrate_vma_collect_skip(start, end,
87+ walk);
88+ } else {
89+ int ret;
90+
91+ get_page(page);
92+ spin_unlock(ptl);
93+ if (unlikely(!trylock_page(page)))
94+ return migrate_vma_collect_skip(start, end,
95+ walk);
96+ ret = split_huge_page(page);
97+ unlock_page(page);
98+ put_page(page);
99+ if (ret)
100+ return migrate_vma_collect_skip(start, end,
101+ walk);
102+ if (pmd_none(*pmdp))
103+ return migrate_vma_collect_hole(start, end, -1,
104+ walk);
105+ }
106+ }
107+
108+ if (unlikely(pmd_bad(*pmdp)))
109+ return migrate_vma_collect_skip(start, end, walk);
110+
111+ ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
112+ arch_enter_lazy_mmu_mode();
113+
114+ for (; addr < end; addr += PAGE_SIZE, ptep++) {
115+ unsigned long mpfn = 0, pfn;
116+ struct page *page;
117+ swp_entry_t entry;
118+ pte_t pte;
119+
120+ pte = *ptep;
121+
122+ if (pte_none(pte)) {
123+ if (vma_is_anonymous(vma)) {
124+ mpfn = MIGRATE_PFN_MIGRATE;
125+ migrate->cpages++;
126+ }
127+ goto next;
128+ }
129+
130+ if (!pte_present(pte)) {
131+ /*
132+ * Only care about unaddressable device page special
133+ * page table entry. Other special swap entries are not
134+ * migratable, and we ignore regular swapped page.
135+ */
136+ entry = pte_to_swp_entry(pte);
137+ if (!is_device_private_entry(entry))
138+ goto next;
139+
140+ page = pfn_swap_entry_to_page(entry);
141+ if (!(migrate->flags &
142+ MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
143+ page->pgmap->owner != migrate->pgmap_owner)
144+ goto next;
145+
146+ mpfn = migrate_pfn(page_to_pfn(page)) |
147+ MIGRATE_PFN_MIGRATE;
148+ if (is_writable_device_private_entry(entry))
149+ mpfn |= MIGRATE_PFN_WRITE;
150+ } else {
151+ if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
152+ goto next;
153+ pfn = pte_pfn(pte);
154+ if (is_zero_pfn(pfn)) {
155+ mpfn = MIGRATE_PFN_MIGRATE;
156+ migrate->cpages++;
157+ goto next;
158+ }
159+ page = vm_normal_page(migrate->vma, addr, pte);
160+ mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
161+ mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
162+ }
163+
164+ /* FIXME support THP */
165+ if (!page || !page->mapping || PageTransCompound(page)) {
166+ mpfn = 0;
167+ goto next;
168+ }
169+
170+ /*
171+ * By getting a reference on the page we pin it and that blocks
172+ * any kind of migration. Side effect is that it "freezes" the
173+ * pte.
174+ *
175+ * We drop this reference after isolating the page from the lru
176+ * for non device page (device page are not on the lru and thus
177+ * can't be dropped from it).
178+ */
179+ get_page(page);
180+
181+ /*
182+ * Optimize for the common case where page is only mapped once
183+ * in one process. If we can lock the page, then we can safely
184+ * set up a special migration page table entry now.
185+ */
186+ if (trylock_page(page)) {
187+ pte_t swp_pte;
188+
189+ migrate->cpages++;
190+ ptep_get_and_clear(mm, addr, ptep);
191+
192+ /* Setup special migration page table entry */
193+ if (mpfn & MIGRATE_PFN_WRITE)
194+ entry = make_writable_migration_entry(
195+ page_to_pfn(page));
196+ else
197+ entry = make_readable_migration_entry(
198+ page_to_pfn(page));
199+ swp_pte = swp_entry_to_pte(entry);
200+ if (pte_present(pte)) {
201+ if (pte_soft_dirty(pte))
202+ swp_pte = pte_swp_mksoft_dirty(swp_pte);
203+ if (pte_uffd_wp(pte))
204+ swp_pte = pte_swp_mkuffd_wp(swp_pte);
205+ } else {
206+ if (pte_swp_soft_dirty(pte))
207+ swp_pte = pte_swp_mksoft_dirty(swp_pte);
208+ if (pte_swp_uffd_wp(pte))
209+ swp_pte = pte_swp_mkuffd_wp(swp_pte);
210+ }
211+ set_pte_at(mm, addr, ptep, swp_pte);
212+
213+ /*
214+ * This is like regular unmap: we remove the rmap and
215+ * drop page refcount. Page won't be freed, as we took
216+ * a reference just above.
217+ */
218+ page_remove_rmap(page, vma, false);
219+ put_page(page);
220+
221+ if (pte_present(pte))
222+ unmapped++;
223+ } else {
224+ put_page(page);
225+ mpfn = 0;
226+ }
227+
228+next:
229+ migrate->dst[migrate->npages] = 0;
230+ migrate->src[migrate->npages++] = mpfn;
231+ }
232+ arch_leave_lazy_mmu_mode();
233+ pte_unmap_unlock(ptep - 1, ptl);
234+
235+ /* Only flush the TLB if we actually modified any entries */
236+ if (unmapped)
237+ flush_tlb_range(walk->vma, start, end);
238+
239+ return 0;
240+}
241+
242+static const struct mm_walk_ops migrate_vma_walk_ops = {
243+ .pmd_entry = migrate_vma_collect_pmd,
244+ .pte_hole = migrate_vma_collect_hole,
245+};
246+
247+/*
248+ * migrate_vma_collect() - collect pages over a range of virtual addresses
249+ * @migrate: migrate struct containing all migration information
250+ *
251+ * This will walk the CPU page table. For each virtual address backed by a
252+ * valid page, it updates the src array and takes a reference on the page, in
253+ * order to pin the page until we lock it and unmap it.
254+ */
255+static void migrate_vma_collect(struct migrate_vma *migrate)
256+{
257+ struct mmu_notifier_range range;
258+
259+ /*
260+ * Note that the pgmap_owner is passed to the mmu notifier callback so
261+ * that the registered device driver can skip invalidating device
262+ * private page mappings that won't be migrated.
263+ */
264+ mmu_notifier_range_init_owner(&range, MMU_NOTIFY_MIGRATE, 0,
265+ migrate->vma, migrate->vma->vm_mm, migrate->start, migrate->end,
266+ migrate->pgmap_owner);
267+ mmu_notifier_invalidate_range_start(&range);
268+
269+ walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
270+ &migrate_vma_walk_ops, migrate);
271+
272+ mmu_notifier_invalidate_range_end(&range);
273+ migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
274+}
275+
276+/*
277+ * migrate_vma_check_page() - check if page is pinned or not
278+ * @page: struct page to check
279+ *
280+ * Pinned pages cannot be migrated. This is the same test as in
281+ * folio_migrate_mapping(), except that here we allow migration of a
282+ * ZONE_DEVICE page.
283+ */
284+static bool migrate_vma_check_page(struct page *page)
285+{
286+ /*
287+ * One extra ref because caller holds an extra reference, either from
288+ * isolate_lru_page() for a regular page, or migrate_vma_collect() for
289+ * a device page.
290+ */
291+ int extra = 1;
292+
293+ /*
294+ * FIXME support THP (transparent huge page), it is bit more complex to
295+ * check them than regular pages, because they can be mapped with a pmd
296+ * or with a pte (split pte mapping).
297+ */
298+ if (PageCompound(page))
299+ return false;
300+
301+ /* Page from ZONE_DEVICE have one extra reference */
302+ if (is_zone_device_page(page))
303+ extra++;
304+
305+ /* For file back page */
306+ if (page_mapping(page))
307+ extra += 1 + page_has_private(page);
308+
309+ if ((page_count(page) - extra) > page_mapcount(page))
310+ return false;
311+
312+ return true;
313+}
314+
315+/*
316+ * migrate_vma_unmap() - replace page mapping with special migration pte entry
317+ * @migrate: migrate struct containing all migration information
318+ *
319+ * Isolate pages from the LRU and replace mappings (CPU page table pte) with a
320+ * special migration pte entry and check if it has been pinned. Pinned pages are
321+ * restored because we cannot migrate them.
322+ *
323+ * This is the last step before we call the device driver callback to allocate
324+ * destination memory and copy contents of original page over to new page.
325+ */
326+static void migrate_vma_unmap(struct migrate_vma *migrate)
327+{
328+ const unsigned long npages = migrate->npages;
329+ unsigned long i, restore = 0;
330+ bool allow_drain = true;
331+
332+ lru_add_drain();
333+
334+ for (i = 0; i < npages; i++) {
335+ struct page *page = migrate_pfn_to_page(migrate->src[i]);
336+ struct folio *folio;
337+
338+ if (!page)
339+ continue;
340+
341+ /* ZONE_DEVICE pages are not on LRU */
342+ if (!is_zone_device_page(page)) {
343+ if (!PageLRU(page) && allow_drain) {
344+ /* Drain CPU's pagevec */
345+ lru_add_drain_all();
346+ allow_drain = false;
347+ }
348+
349+ if (isolate_lru_page(page)) {
350+ migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
351+ migrate->cpages--;
352+ restore++;
353+ continue;
354+ }
355+
356+ /* Drop the reference we took in collect */
357+ put_page(page);
358+ }
359+
360+ folio = page_folio(page);
361+ if (folio_mapped(folio))
362+ try_to_migrate(folio, 0);
363+
364+ if (page_mapped(page) || !migrate_vma_check_page(page)) {
365+ if (!is_zone_device_page(page)) {
366+ get_page(page);
367+ putback_lru_page(page);
368+ }
369+
370+ migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
371+ migrate->cpages--;
372+ restore++;
373+ continue;
374+ }
375+ }
376+
377+ for (i = 0; i < npages && restore; i++) {
378+ struct page *page = migrate_pfn_to_page(migrate->src[i]);
379+ struct folio *folio;
380+
381+ if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
382+ continue;
383+
384+ folio = page_folio(page);
385+ remove_migration_ptes(folio, folio, false);
386+
387+ migrate->src[i] = 0;
388+ folio_unlock(folio);
389+ folio_put(folio);
390+ restore--;
391+ }
392+}
393+
394+/**
395+ * migrate_vma_setup() - prepare to migrate a range of memory
396+ * @args: contains the vma, start, and pfns arrays for the migration
397+ *
398+ * Returns: negative errno on failures, 0 when 0 or more pages were migrated
399+ * without an error.
400+ *
401+ * Prepare to migrate a range of memory virtual address range by collecting all
402+ * the pages backing each virtual address in the range, saving them inside the
403+ * src array. Then lock those pages and unmap them. Once the pages are locked
404+ * and unmapped, check whether each page is pinned or not. Pages that aren't
405+ * pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
406+ * corresponding src array entry. Then restores any pages that are pinned, by
407+ * remapping and unlocking those pages.
408+ *
409+ * The caller should then allocate destination memory and copy source memory to
410+ * it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
411+ * flag set). Once these are allocated and copied, the caller must update each
412+ * corresponding entry in the dst array with the pfn value of the destination
413+ * page and with MIGRATE_PFN_VALID. Destination pages must be locked via
414+ * lock_page().
415+ *
416+ * Note that the caller does not have to migrate all the pages that are marked
417+ * with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
418+ * device memory to system memory. If the caller cannot migrate a device page
419+ * back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
420+ * consequences for the userspace process, so it must be avoided if at all
421+ * possible.
422+ *
423+ * For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
424+ * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
425+ * allowing the caller to allocate device memory for those unbacked virtual
426+ * addresses. For this the caller simply has to allocate device memory and
427+ * properly set the destination entry like for regular migration. Note that
428+ * this can still fail, and thus inside the device driver you must check if the
429+ * migration was successful for those entries after calling migrate_vma_pages(),
430+ * just like for regular migration.
431+ *
432+ * After that, the callers must call migrate_vma_pages() to go over each entry
433+ * in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
434+ * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
435+ * then migrate_vma_pages() to migrate struct page information from the source
436+ * struct page to the destination struct page. If it fails to migrate the
437+ * struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
438+ * src array.
439+ *
440+ * At this point all successfully migrated pages have an entry in the src
441+ * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
442+ * array entry with MIGRATE_PFN_VALID flag set.
443+ *
444+ * Once migrate_vma_pages() returns the caller may inspect which pages were
445+ * successfully migrated, and which were not. Successfully migrated pages will
446+ * have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
447+ *
448+ * It is safe to update device page table after migrate_vma_pages() because
449+ * both destination and source page are still locked, and the mmap_lock is held
450+ * in read mode (hence no one can unmap the range being migrated).
451+ *
452+ * Once the caller is done cleaning up things and updating its page table (if it
453+ * chose to do so, this is not an obligation) it finally calls
454+ * migrate_vma_finalize() to update the CPU page table to point to new pages
455+ * for successfully migrated pages or otherwise restore the CPU page table to
456+ * point to the original source pages.
457+ */
458+int migrate_vma_setup(struct migrate_vma *args)
459+{
460+ long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
461+
462+ args->start &= PAGE_MASK;
463+ args->end &= PAGE_MASK;
464+ if (!args->vma || is_vm_hugetlb_page(args->vma) ||
465+ (args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
466+ return -EINVAL;
467+ if (nr_pages <= 0)
468+ return -EINVAL;
469+ if (args->start < args->vma->vm_start ||
470+ args->start >= args->vma->vm_end)
471+ return -EINVAL;
472+ if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
473+ return -EINVAL;
474+ if (!args->src || !args->dst)
475+ return -EINVAL;
476+
477+ memset(args->src, 0, sizeof(*args->src) * nr_pages);
478+ args->cpages = 0;
479+ args->npages = 0;
480+
481+ migrate_vma_collect(args);
482+
483+ if (args->cpages)
484+ migrate_vma_unmap(args);
485+
486+ /*
487+ * At this point pages are locked and unmapped, and thus they have
488+ * stable content and can safely be copied to destination memory that
489+ * is allocated by the drivers.
490+ */
491+ return 0;
492+
493+}
494+EXPORT_SYMBOL(migrate_vma_setup);
495+
496+/*
497+ * This code closely matches the code in:
498+ * __handle_mm_fault()
499+ * handle_pte_fault()
500+ * do_anonymous_page()
501+ * to map in an anonymous zero page but the struct page will be a ZONE_DEVICE
502+ * private page.
503+ */
504+static void migrate_vma_insert_page(struct migrate_vma *migrate,
505+ unsigned long addr,
506+ struct page *page,
507+ unsigned long *src)
508+{
509+ struct vm_area_struct *vma = migrate->vma;
510+ struct mm_struct *mm = vma->vm_mm;
511+ bool flush = false;
512+ spinlock_t *ptl;
513+ pte_t entry;
514+ pgd_t *pgdp;
515+ p4d_t *p4dp;
516+ pud_t *pudp;
517+ pmd_t *pmdp;
518+ pte_t *ptep;
519+
520+ /* Only allow populating anonymous memory */
521+ if (!vma_is_anonymous(vma))
522+ goto abort;
523+
524+ pgdp = pgd_offset(mm, addr);
525+ p4dp = p4d_alloc(mm, pgdp, addr);
526+ if (!p4dp)
527+ goto abort;
528+ pudp = pud_alloc(mm, p4dp, addr);
529+ if (!pudp)
530+ goto abort;
531+ pmdp = pmd_alloc(mm, pudp, addr);
532+ if (!pmdp)
533+ goto abort;
534+
535+ if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
536+ goto abort;
537+
538+ /*
539+ * Use pte_alloc() instead of pte_alloc_map(). We can't run
540+ * pte_offset_map() on pmds where a huge pmd might be created
541+ * from a different thread.
542+ *
543+ * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
544+ * parallel threads are excluded by other means.
545+ *
546+ * Here we only have mmap_read_lock(mm).
547+ */
548+ if (pte_alloc(mm, pmdp))
549+ goto abort;
550+
551+ /* See the comment in pte_alloc_one_map() */
552+ if (unlikely(pmd_trans_unstable(pmdp)))
553+ goto abort;
554+
555+ if (unlikely(anon_vma_prepare(vma)))
556+ goto abort;
557+ if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))
558+ goto abort;
559+
560+ /*
561+ * The memory barrier inside __SetPageUptodate makes sure that
562+ * preceding stores to the page contents become visible before
563+ * the set_pte_at() write.
564+ */
565+ __SetPageUptodate(page);
566+
567+ if (is_device_private_page(page)) {
568+ swp_entry_t swp_entry;
569+
570+ if (vma->vm_flags & VM_WRITE)
571+ swp_entry = make_writable_device_private_entry(
572+ page_to_pfn(page));
573+ else
574+ swp_entry = make_readable_device_private_entry(
575+ page_to_pfn(page));
576+ entry = swp_entry_to_pte(swp_entry);
577+ } else {
578+ /*
579+ * For now we only support migrating to un-addressable device
580+ * memory.
581+ */
582+ if (is_zone_device_page(page)) {
583+ pr_warn_once("Unsupported ZONE_DEVICE page type.\n");
584+ goto abort;
585+ }
586+ entry = mk_pte(page, vma->vm_page_prot);
587+ if (vma->vm_flags & VM_WRITE)
588+ entry = pte_mkwrite(pte_mkdirty(entry));
589+ }
590+
591+ ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
592+
593+ if (check_stable_address_space(mm))
594+ goto unlock_abort;
595+
596+ if (pte_present(*ptep)) {
597+ unsigned long pfn = pte_pfn(*ptep);
598+
599+ if (!is_zero_pfn(pfn))
600+ goto unlock_abort;
601+ flush = true;
602+ } else if (!pte_none(*ptep))
603+ goto unlock_abort;
604+
605+ /*
606+ * Check for userfaultfd but do not deliver the fault. Instead,
607+ * just back off.
608+ */
609+ if (userfaultfd_missing(vma))
610+ goto unlock_abort;
611+
612+ inc_mm_counter(mm, MM_ANONPAGES);
613+ page_add_new_anon_rmap(page, vma, addr, false);
614+ if (!is_zone_device_page(page))
615+ lru_cache_add_inactive_or_unevictable(page, vma);
616+ get_page(page);
617+
618+ if (flush) {
619+ flush_cache_page(vma, addr, pte_pfn(*ptep));
620+ ptep_clear_flush_notify(vma, addr, ptep);
621+ set_pte_at_notify(mm, addr, ptep, entry);
622+ update_mmu_cache(vma, addr, ptep);
623+ } else {
624+ /* No need to invalidate - it was non-present before */
625+ set_pte_at(mm, addr, ptep, entry);
626+ update_mmu_cache(vma, addr, ptep);
627+ }
628+
629+ pte_unmap_unlock(ptep, ptl);
630+ *src = MIGRATE_PFN_MIGRATE;
631+ return;
632+
633+unlock_abort:
634+ pte_unmap_unlock(ptep, ptl);
635+abort:
636+ *src &= ~MIGRATE_PFN_MIGRATE;
637+}
638+
639+/**
640+ * migrate_vma_pages() - migrate meta-data from src page to dst page
641+ * @migrate: migrate struct containing all migration information
642+ *
643+ * This migrates struct page meta-data from source struct page to destination
644+ * struct page. This effectively finishes the migration from source page to the
645+ * destination page.
646+ */
647+void migrate_vma_pages(struct migrate_vma *migrate)
648+{
649+ const unsigned long npages = migrate->npages;
650+ const unsigned long start = migrate->start;
651+ struct mmu_notifier_range range;
652+ unsigned long addr, i;
653+ bool notified = false;
654+
655+ for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
656+ struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
657+ struct page *page = migrate_pfn_to_page(migrate->src[i]);
658+ struct address_space *mapping;
659+ int r;
660+
661+ if (!newpage) {
662+ migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
663+ continue;
664+ }
665+
666+ if (!page) {
667+ if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))
668+ continue;
669+ if (!notified) {
670+ notified = true;
671+
672+ mmu_notifier_range_init_owner(&range,
673+ MMU_NOTIFY_MIGRATE, 0, migrate->vma,
674+ migrate->vma->vm_mm, addr, migrate->end,
675+ migrate->pgmap_owner);
676+ mmu_notifier_invalidate_range_start(&range);
677+ }
678+ migrate_vma_insert_page(migrate, addr, newpage,
679+ &migrate->src[i]);
680+ continue;
681+ }
682+
683+ mapping = page_mapping(page);
684+
685+ if (is_device_private_page(newpage)) {
686+ /*
687+ * For now only support private anonymous when migrating
688+ * to un-addressable device memory.
689+ */
690+ if (mapping) {
691+ migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
692+ continue;
693+ }
694+ } else if (is_zone_device_page(newpage)) {
695+ /*
696+ * Other types of ZONE_DEVICE page are not supported.
697+ */
698+ migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
699+ continue;
700+ }
701+
702+ r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
703+ if (r != MIGRATEPAGE_SUCCESS)
704+ migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
705+ }
706+
707+ /*
708+ * No need to double call mmu_notifier->invalidate_range() callback as
709+ * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
710+ * did already call it.
711+ */
712+ if (notified)
713+ mmu_notifier_invalidate_range_only_end(&range);
714+}
715+EXPORT_SYMBOL(migrate_vma_pages);
716+
717+/**
718+ * migrate_vma_finalize() - restore CPU page table entry
719+ * @migrate: migrate struct containing all migration information
720+ *
721+ * This replaces the special migration pte entry with either a mapping to the
722+ * new page if migration was successful for that page, or to the original page
723+ * otherwise.
724+ *
725+ * This also unlocks the pages and puts them back on the lru, or drops the extra
726+ * refcount, for device pages.
727+ */
728+void migrate_vma_finalize(struct migrate_vma *migrate)
729+{
730+ const unsigned long npages = migrate->npages;
731+ unsigned long i;
732+
733+ for (i = 0; i < npages; i++) {
734+ struct folio *dst, *src;
735+ struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
736+ struct page *page = migrate_pfn_to_page(migrate->src[i]);
737+
738+ if (!page) {
739+ if (newpage) {
740+ unlock_page(newpage);
741+ put_page(newpage);
742+ }
743+ continue;
744+ }
745+
746+ if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
747+ if (newpage) {
748+ unlock_page(newpage);
749+ put_page(newpage);
750+ }
751+ newpage = page;
752+ }
753+
754+ src = page_folio(page);
755+ dst = page_folio(newpage);
756+ remove_migration_ptes(src, dst, false);
757+ folio_unlock(src);
758+
759+ if (is_zone_device_page(page))
760+ put_page(page);
761+ else
762+ putback_lru_page(page);
763+
764+ if (newpage != page) {
765+ unlock_page(newpage);
766+ if (is_zone_device_page(newpage))
767+ put_page(newpage);
768+ else
769+ putback_lru_page(newpage);
770+ }
771+ }
772+}
773+EXPORT_SYMBOL(migrate_vma_finalize);
mm/mlock.c+266−372
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <linux/swapops.h>
1515 #include <linux/pagemap.h>
1616 #include <linux/pagevec.h>
17+#include <linux/pagewalk.h>
1718 #include <linux/mempolicy.h>
1819 #include <linux/syscalls.h>
1920 #include <linux/sched.h>
@@ -27,6 +28,8 @@ · 7 unmodified lines
2728
2829 #include "internal.h"
2930
31+static DEFINE_PER_CPU(struct pagevec, mlock_pvec);
32+
3033 bool can_do_mlock(void)
3134 {
3235 if (rlimit(RLIMIT_MEMLOCK) != 0)
@@ -46,441 +49,320 @@ EXPORT_SYMBOL(can_do_mlock); · 13 unmodified lines
4649 * be placed on the LRU "unevictable" list, rather than the [in]active lists.
4750 * The unevictable list is an LRU sibling list to the [in]active lists.
4851 * PageUnevictable is set to indicate the unevictable state.
49− *
50− * When lazy mlocking via vmscan, it is important to ensure that the
51− * vma's VM_LOCKED status is not concurrently being modified, otherwise we
52− * may have mlocked a page that is being munlocked. So lazy mlock must take
53− * the mmap_lock for read, and verify that the vma really is locked
54− * (see mm/rmap.c).
5552 */
5653
57−/*
58− * LRU accounting for clear_page_mlock()
59− */
60−void clear_page_mlock(struct page *page)
54+static struct lruvec *__mlock_page(struct page *page, struct lruvec *lruvec)
6155 {
62− int nr_pages;
56+ /* There is nothing more we can do while it's off LRU */
57+ if (!TestClearPageLRU(page))
58+ return lruvec;
6359
64− if (!TestClearPageMlocked(page))
65− return;
60+ lruvec = folio_lruvec_relock_irq(page_folio(page), lruvec);
6661
67− nr_pages = thp_nr_pages(page);
68− mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
69− count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages);
70− /*
71− * The previous TestClearPageMlocked() corresponds to the smp_mb()
72− * in __pagevec_lru_add_fn().
73− *
74− * See __pagevec_lru_add_fn for more explanation.
75− */
76− if (!isolate_lru_page(page)) {
77− putback_lru_page(page);
78− } else {
62+ if (unlikely(page_evictable(page))) {
7963 /*
80− * We lost the race. the page already moved to evictable list.
64+ * This is a little surprising, but quite possible:
65+ * PageMlocked must have got cleared already by another CPU.
66+ * Could this page be on the Unevictable LRU? I'm not sure,
67+ * but move it now if so.
8168 */
82− if (PageUnevictable(page))
83− count_vm_events(UNEVICTABLE_PGSTRANDED, nr_pages);
69+ if (PageUnevictable(page)) {
70+ del_page_from_lru_list(page, lruvec);
71+ ClearPageUnevictable(page);
72+ add_page_to_lru_list(page, lruvec);
73+ __count_vm_events(UNEVICTABLE_PGRESCUED,
74+ thp_nr_pages(page));
75+ }
76+ goto out;
8477 }
78+
79+ if (PageUnevictable(page)) {
80+ if (PageMlocked(page))
81+ page->mlock_count++;
82+ goto out;
83+ }
84+
85+ del_page_from_lru_list(page, lruvec);
86+ ClearPageActive(page);
87+ SetPageUnevictable(page);
88+ page->mlock_count = !!PageMlocked(page);
89+ add_page_to_lru_list(page, lruvec);
90+ __count_vm_events(UNEVICTABLE_PGCULLED, thp_nr_pages(page));
91+out:
92+ SetPageLRU(page);
93+ return lruvec;
8594 }
8695
87−/*
88− * Mark page as mlocked if not already.
89− * If page on LRU, isolate and putback to move to unevictable list.
90− */
91−void mlock_vma_page(struct page *page)
96+static struct lruvec *__mlock_new_page(struct page *page, struct lruvec *lruvec)
9297 {
93− /* Serialize with page migration */
94− BUG_ON(!PageLocked(page));
98+ VM_BUG_ON_PAGE(PageLRU(page), page);
9599
96− VM_BUG_ON_PAGE(PageTail(page), page);
97− VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page);
100+ lruvec = folio_lruvec_relock_irq(page_folio(page), lruvec);
98101
99− if (!TestSetPageMlocked(page)) {
100− int nr_pages = thp_nr_pages(page);
102+ /* As above, this is a little surprising, but possible */
103+ if (unlikely(page_evictable(page)))
104+ goto out;
101105
102− mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages);
103− count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
104− if (!isolate_lru_page(page))
105− putback_lru_page(page);
106− }
106+ SetPageUnevictable(page);
107+ page->mlock_count = !!PageMlocked(page);
108+ __count_vm_events(UNEVICTABLE_PGCULLED, thp_nr_pages(page));
109+out:
110+ add_page_to_lru_list(page, lruvec);
111+ SetPageLRU(page);
112+ return lruvec;
107113 }
108114
109−/*
110− * Finish munlock after successful page isolation
111− *
112− * Page must be locked. This is a wrapper for page_mlock()
113− * and putback_lru_page() with munlock accounting.
114− */
115−static void __munlock_isolated_page(struct page *page)
115+static struct lruvec *__munlock_page(struct page *page, struct lruvec *lruvec)
116116 {
117− /*
118− * Optimization: if the page was mapped just once, that's our mapping
119− * and we don't need to check all the other vmas.
120− */
121− if (page_mapcount(page) > 1)
122− page_mlock(page);
117+ int nr_pages = thp_nr_pages(page);
118+ bool isolated = false;
119+
120+ if (!TestClearPageLRU(page))
121+ goto munlock;
123122
124− /* Did try_to_unlock() succeed or punt? */
125− if (!PageMlocked(page))
126− count_vm_events(UNEVICTABLE_PGMUNLOCKED, thp_nr_pages(page));
123+ isolated = true;
124+ lruvec = folio_lruvec_relock_irq(page_folio(page), lruvec);
127125
128− putback_lru_page(page);
126+ if (PageUnevictable(page)) {
127+ /* Then mlock_count is maintained, but might undercount */
128+ if (page->mlock_count)
129+ page->mlock_count--;
130+ if (page->mlock_count)
131+ goto out;
132+ }
133+ /* else assume that was the last mlock: reclaim will fix it if not */
134+
135+munlock:
136+ if (TestClearPageMlocked(page)) {
137+ __mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
138+ if (isolated || !PageUnevictable(page))
139+ __count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages);
140+ else
141+ __count_vm_events(UNEVICTABLE_PGSTRANDED, nr_pages);
142+ }
143+
144+ /* page_evictable() has to be checked *after* clearing Mlocked */
145+ if (isolated && PageUnevictable(page) && page_evictable(page)) {
146+ del_page_from_lru_list(page, lruvec);
147+ ClearPageUnevictable(page);
148+ add_page_to_lru_list(page, lruvec);
149+ __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
150+ }
151+out:
152+ if (isolated)
153+ SetPageLRU(page);
154+ return lruvec;
129155 }
130156
131157 /*
132− * Accounting for page isolation fail during munlock
133− *
134− * Performs accounting when page isolation fails in munlock. There is nothing
135− * else to do because it means some other task has already removed the page
136− * from the LRU. putback_lru_page() will take care of removing the page from
137− * the unevictable list, if necessary. vmscan [page_referenced()] will move
138− * the page back to the unevictable list if some other vma has it mlocked.
158+ * Flags held in the low bits of a struct page pointer on the mlock_pvec.
139159 */
140−static void __munlock_isolation_failed(struct page *page)
160+#define LRU_PAGE 0x1
161+#define NEW_PAGE 0x2
162+static inline struct page *mlock_lru(struct page *page)
141163 {
142− int nr_pages = thp_nr_pages(page);
164+ return (struct page *)((unsigned long)page + LRU_PAGE);
165+}
143166
144− if (PageUnevictable(page))
145− __count_vm_events(UNEVICTABLE_PGSTRANDED, nr_pages);
146− else
147− __count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages);
167+static inline struct page *mlock_new(struct page *page)
168+{
169+ return (struct page *)((unsigned long)page + NEW_PAGE);
148170 }
149171
150−/**
151− * munlock_vma_page - munlock a vma page
152− * @page: page to be unlocked, either a normal page or THP page head
153− *
154− * returns the size of the page as a page mask (0 for normal page,
155− * HPAGE_PMD_NR - 1 for THP head page)
156− *
157− * called from munlock()/munmap() path with page supposedly on the LRU.
158− * When we munlock a page, because the vma where we found the page is being
159− * munlock()ed or munmap()ed, we want to check whether other vmas hold the
160− * page locked so that we can leave it on the unevictable lru list and not
161− * bother vmscan with it. However, to walk the page's rmap list in
162− * page_mlock() we must isolate the page from the LRU. If some other
163− * task has removed the page from the LRU, we won't be able to do that.
164− * So we clear the PageMlocked as we might not get another chance. If we
165− * can't isolate the page, we leave it for putback_lru_page() and vmscan
166− * [page_referenced()/try_to_unmap()] to deal with.
172+/*
173+ * mlock_pagevec() is derived from pagevec_lru_move_fn():
174+ * perhaps that can make use of such page pointer flags in future,
175+ * but for now just keep it for mlock. We could use three separate
176+ * pagevecs instead, but one feels better (munlocking a full pagevec
177+ * does not need to drain mlocking pagevecs first).
167178 */
168−unsigned int munlock_vma_page(struct page *page)
179+static void mlock_pagevec(struct pagevec *pvec)
169180 {
170− int nr_pages;
171−
172− /* For page_mlock() and to serialize with page migration */
173− BUG_ON(!PageLocked(page));
174− VM_BUG_ON_PAGE(PageTail(page), page);
181+ struct lruvec *lruvec = NULL;
182+ unsigned long mlock;
183+ struct page *page;
184+ int i;
175185
176− if (!TestClearPageMlocked(page)) {
177− /* Potentially, PTE-mapped THP: do not skip the rest PTEs */
178− return 0;
186+ for (i = 0; i < pagevec_count(pvec); i++) {
187+ page = pvec->pages[i];
188+ mlock = (unsigned long)page & (LRU_PAGE | NEW_PAGE);
189+ page = (struct page *)((unsigned long)page - mlock);
190+ pvec->pages[i] = page;
191+
192+ if (mlock & LRU_PAGE)
193+ lruvec = __mlock_page(page, lruvec);
194+ else if (mlock & NEW_PAGE)
195+ lruvec = __mlock_new_page(page, lruvec);
196+ else
197+ lruvec = __munlock_page(page, lruvec);
179198 }
180199
181− nr_pages = thp_nr_pages(page);
182− mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
200+ if (lruvec)
201+ unlock_page_lruvec_irq(lruvec);
202+ release_pages(pvec->pages, pvec->nr);
203+ pagevec_reinit(pvec);
204+}
183205
184− if (!isolate_lru_page(page))
185− __munlock_isolated_page(page);
186− else
187− __munlock_isolation_failed(page);
206+void mlock_page_drain(int cpu)
207+{
208+ struct pagevec *pvec;
188209
189− return nr_pages - 1;
210+ pvec = &per_cpu(mlock_pvec, cpu);
211+ if (pagevec_count(pvec))
212+ mlock_pagevec(pvec);
190213 }
191214
192−/*
193− * convert get_user_pages() return value to posix mlock() error
194− */
195−static int __mlock_posix_error_return(long retval)
215+bool need_mlock_page_drain(int cpu)
196216 {
197− if (retval == -EFAULT)
198− retval = -ENOMEM;
199− else if (retval == -ENOMEM)
200− retval = -EAGAIN;
201− return retval;
217+ return pagevec_count(&per_cpu(mlock_pvec, cpu));
202218 }
203219
204−/*
205− * Prepare page for fast batched LRU putback via putback_lru_evictable_pagevec()
206− *
207− * The fast path is available only for evictable pages with single mapping.
208− * Then we can bypass the per-cpu pvec and get better performance.
209− * when mapcount > 1 we need page_mlock() which can fail.
210− * when !page_evictable(), we need the full redo logic of putback_lru_page to
211− * avoid leaving evictable page in unevictable list.
212− *
213− * In case of success, @page is added to @pvec and @pgrescued is incremented
214− * in case that the page was previously unevictable. @page is also unlocked.
220+/**
221+ * mlock_folio - mlock a folio already on (or temporarily off) LRU
222+ * @folio: folio to be mlocked.
215223 */
216−static bool __putback_lru_fast_prepare(struct page *page, struct pagevec *pvec,
217− int *pgrescued)
224+void mlock_folio(struct folio *folio)
218225 {
219− VM_BUG_ON_PAGE(PageLRU(page), page);
220− VM_BUG_ON_PAGE(!PageLocked(page), page);
226+ struct pagevec *pvec = &get_cpu_var(mlock_pvec);
221227
222− if (page_mapcount(page) <= 1 && page_evictable(page)) {
223− pagevec_add(pvec, page);
224− if (TestClearPageUnevictable(page))
225− (*pgrescued)++;
226− unlock_page(page);
227− return true;
228+ if (!folio_test_set_mlocked(folio)) {
229+ int nr_pages = folio_nr_pages(folio);
230+
231+ zone_stat_mod_folio(folio, NR_MLOCK, nr_pages);
232+ __count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
228233 }
229234
230− return false;
235+ folio_get(folio);
236+ if (!pagevec_add(pvec, mlock_lru(&folio->page)) ||
237+ folio_test_large(folio) || lru_cache_disabled())
238+ mlock_pagevec(pvec);
239+ put_cpu_var(mlock_pvec);
231240 }
232241
233−/*
234− * Putback multiple evictable pages to the LRU
235− *
236− * Batched putback of evictable pages that bypasses the per-cpu pvec. Some of
237− * the pages might have meanwhile become unevictable but that is OK.
242+/**
243+ * mlock_new_page - mlock a newly allocated page not yet on LRU
244+ * @page: page to be mlocked, either a normal page or a THP head.
238245 */
239−static void __putback_lru_fast(struct pagevec *pvec, int pgrescued)
246+void mlock_new_page(struct page *page)
240247 {
241− count_vm_events(UNEVICTABLE_PGMUNLOCKED, pagevec_count(pvec));
242− /*
243− *__pagevec_lru_add() calls release_pages() so we don't call
244− * put_page() explicitly
245− */
246− __pagevec_lru_add(pvec);
247− count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
248+ struct pagevec *pvec = &get_cpu_var(mlock_pvec);
249+ int nr_pages = thp_nr_pages(page);
250+
251+ SetPageMlocked(page);
252+ mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages);
253+ __count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
254+
255+ get_page(page);
256+ if (!pagevec_add(pvec, mlock_new(page)) ||
257+ PageHead(page) || lru_cache_disabled())
258+ mlock_pagevec(pvec);
259+ put_cpu_var(mlock_pvec);
248260 }
249261
250−/*
251− * Munlock a batch of pages from the same zone
252− *
253− * The work is split to two main phases. First phase clears the Mlocked flag
254− * and attempts to isolate the pages, all under a single zone lru lock.
255− * The second phase finishes the munlock only for pages where isolation
256− * succeeded.
257− *
258− * Note that the pagevec may be modified during the process.
262+/**
263+ * munlock_page - munlock a page
264+ * @page: page to be munlocked, either a normal page or a THP head.
259265 */
260−static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone)
266+void munlock_page(struct page *page)
261267 {
262− int i;
263− int nr = pagevec_count(pvec);
264− int delta_munlocked = -nr;
265− struct pagevec pvec_putback;
266− struct lruvec *lruvec = NULL;
267− int pgrescued = 0;
268−
269− pagevec_init(&pvec_putback);
270−
271− /* Phase 1: page isolation */
272− for (i = 0; i < nr; i++) {
273− struct page *page = pvec->pages[i];
274− struct folio *folio = page_folio(page);
275−
276− if (TestClearPageMlocked(page)) {
277− /*
278− * We already have pin from follow_page_mask()
279− * so we can spare the get_page() here.
280− */
281− if (TestClearPageLRU(page)) {
282− lruvec = folio_lruvec_relock_irq(folio, lruvec);
283− del_page_from_lru_list(page, lruvec);
284− continue;
285− } else
286− __munlock_isolation_failed(page);
287− } else {
288− delta_munlocked++;
289− }
290−
291− /*
292− * We won't be munlocking this page in the next phase
293− * but we still need to release the follow_page_mask()
294− * pin. We cannot do it under lru_lock however. If it's
295− * the last pin, __page_cache_release() would deadlock.
296− */
297− pagevec_add(&pvec_putback, pvec->pages[i]);
298− pvec->pages[i] = NULL;
299− }
300− if (lruvec) {
301− __mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
302− unlock_page_lruvec_irq(lruvec);
303− } else if (delta_munlocked) {
304− mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
305− }
306−
307− /* Now we can release pins of pages that we are not munlocking */
308− pagevec_release(&pvec_putback);
309−
310− /* Phase 2: page munlock */
311− for (i = 0; i < nr; i++) {
312− struct page *page = pvec->pages[i];
313−
314− if (page) {
315− lock_page(page);
316− if (!__putback_lru_fast_prepare(page, &pvec_putback,
317− &pgrescued)) {
318− /*
319− * Slow path. We don't want to lose the last
320− * pin before unlock_page()
321− */
322− get_page(page); /* for putback_lru_page() */
323− __munlock_isolated_page(page);
324− unlock_page(page);
325− put_page(page); /* from follow_page_mask() */
326− }
327− }
328− }
268+ struct pagevec *pvec = &get_cpu_var(mlock_pvec);
329269
330270 /*
331− * Phase 3: page putback for pages that qualified for the fast path
332− * This will also call put_page() to return pin from follow_page_mask()
271+ * TestClearPageMlocked(page) must be left to __munlock_page(),
272+ * which will check whether the page is multiply mlocked.
333273 */
334− if (pagevec_count(&pvec_putback))
335− __putback_lru_fast(&pvec_putback, pgrescued);
274+
275+ get_page(page);
276+ if (!pagevec_add(pvec, page) ||
277+ PageHead(page) || lru_cache_disabled())
278+ mlock_pagevec(pvec);
279+ put_cpu_var(mlock_pvec);
336280 }
337281
338−/*
339− * Fill up pagevec for __munlock_pagevec using pte walk
340− *
341− * The function expects that the struct page corresponding to @start address is
342− * a non-TPH page already pinned and in the @pvec, and that it belongs to @zone.
343− *
344− * The rest of @pvec is filled by subsequent pages within the same pmd and same
345− * zone, as long as the pte's are present and vm_normal_page() succeeds. These
346− * pages also get pinned.
347− *
348− * Returns the address of the next page that should be scanned. This equals
349− * @start + PAGE_SIZE when no page could be added by the pte walk.
350− */
351−static unsigned long __munlock_pagevec_fill(struct pagevec *pvec,
352− struct vm_area_struct *vma, struct zone *zone,
353− unsigned long start, unsigned long end)
282+static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
283+ unsigned long end, struct mm_walk *walk)
284+
354285 {
355− pte_t *pte;
286+ struct vm_area_struct *vma = walk->vma;
356287 spinlock_t *ptl;
288+ pte_t *start_pte, *pte;
289+ struct page *page;
357290
358− /*
359− * Initialize pte walk starting at the already pinned page where we
360− * are sure that there is a pte, as it was pinned under the same
361− * mmap_lock write op.
362− */
363− pte = get_locked_pte(vma->vm_mm, start, &ptl);
364− /* Make sure we do not cross the page table boundary */
365− end = pgd_addr_end(start, end);
366− end = p4d_addr_end(start, end);
367− end = pud_addr_end(start, end);
368− end = pmd_addr_end(start, end);
369−
370− /* The page next to the pinned page is the first we will try to get */
371− start += PAGE_SIZE;
372− while (start < end) {
373− struct page *page = NULL;
374− pte++;
375− if (pte_present(*pte))
376− page = vm_normal_page(vma, start, *pte);
377− /*
378− * Break if page could not be obtained or the page's node+zone does not
379− * match
380− */
381− if (!page || page_zone(page) != zone)
382− break;
291+ ptl = pmd_trans_huge_lock(pmd, vma);
292+ if (ptl) {
293+ if (!pmd_present(*pmd))
294+ goto out;
295+ if (is_huge_zero_pmd(*pmd))
296+ goto out;
297+ page = pmd_page(*pmd);
298+ if (vma->vm_flags & VM_LOCKED)
299+ mlock_folio(page_folio(page));
300+ else
301+ munlock_page(page);
302+ goto out;
303+ }
383304
384− /*
385− * Do not use pagevec for PTE-mapped THP,
386− * munlock_vma_pages_range() will handle them.
387− */
305+ start_pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
306+ for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) {
307+ if (!pte_present(*pte))
308+ continue;
309+ page = vm_normal_page(vma, addr, *pte);
310+ if (!page)
311+ continue;
388312 if (PageTransCompound(page))
389− break;
390−
391− get_page(page);
392− /*
393− * Increase the address that will be returned *before* the
394− * eventual break due to pvec becoming full by adding the page
395− */
396− start += PAGE_SIZE;
397− if (pagevec_add(pvec, page) == 0)
398− break;
313+ continue;
314+ if (vma->vm_flags & VM_LOCKED)
315+ mlock_folio(page_folio(page));
316+ else
317+ munlock_page(page);
399318 }
400− pte_unmap_unlock(pte, ptl);
401− return start;
319+ pte_unmap(start_pte);
320+out:
321+ spin_unlock(ptl);
322+ cond_resched();
323+ return 0;
402324 }
403325
404326 /*
405− * munlock_vma_pages_range() - munlock all pages in the vma range.'
406− * @vma - vma containing range to be munlock()ed.
327+ * mlock_vma_pages_range() - mlock any pages already in the range,
328+ * or munlock all pages in the range.
329+ * @vma - vma containing range to be mlock()ed or munlock()ed
407330 * @start - start address in @vma of the range
408− * @end - end of range in @vma.
331+ * @end - end of range in @vma
332+ * @newflags - the new set of flags for @vma.
409333 *
410− * For mremap(), munmap() and exit().
411− *
412− * Called with @vma VM_LOCKED.
413− *
414− * Returns with VM_LOCKED cleared. Callers must be prepared to
415− * deal with this.
416− *
417− * We don't save and restore VM_LOCKED here because pages are
418− * still on lru. In unmap path, pages might be scanned by reclaim
419− * and re-mlocked by page_mlock/try_to_unmap before we unmap and
420− * free them. This will result in freeing mlocked pages.
334+ * Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED;
335+ * called for munlock() and munlockall(), to clear VM_LOCKED from @vma.
421336 */
422−void munlock_vma_pages_range(struct vm_area_struct *vma,
423− unsigned long start, unsigned long end)
337+static void mlock_vma_pages_range(struct vm_area_struct *vma,
338+ unsigned long start, unsigned long end, vm_flags_t newflags)
424339 {
425− vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
340+ static const struct mm_walk_ops mlock_walk_ops = {
341+ .pmd_entry = mlock_pte_range,
342+ };
426343
427− while (start < end) {
428− struct page *page;
429− unsigned int page_mask = 0;
430− unsigned long page_increm;
431− struct pagevec pvec;
432− struct zone *zone;
344+ /*
345+ * There is a slight chance that concurrent page migration,
346+ * or page reclaim finding a page of this now-VM_LOCKED vma,
347+ * will call mlock_vma_page() and raise page's mlock_count:
348+ * double counting, leaving the page unevictable indefinitely.
349+ * Communicate this danger to mlock_vma_page() with VM_IO,
350+ * which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas.
351+ * mmap_lock is held in write mode here, so this weird
352+ * combination should not be visible to other mmap_lock users;
353+ * but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED.
354+ */
355+ if (newflags & VM_LOCKED)
356+ newflags |= VM_IO;
357+ WRITE_ONCE(vma->vm_flags, newflags);
433358
434− pagevec_init(&pvec);
435− /*
436− * Although FOLL_DUMP is intended for get_dump_page(),
437− * it just so happens that its special treatment of the
438− * ZERO_PAGE (returning an error instead of doing get_page)
439− * suits munlock very well (and if somehow an abnormal page
440− * has sneaked into the range, we won't oops here: great).
441− */
442− page = follow_page(vma, start, FOLL_GET | FOLL_DUMP);
443−
444− if (page && !IS_ERR(page)) {
445− if (PageTransTail(page)) {
446− VM_BUG_ON_PAGE(PageMlocked(page), page);
447− put_page(page); /* follow_page_mask() */
448− } else if (PageTransHuge(page)) {
449− lock_page(page);
450− /*
451− * Any THP page found by follow_page_mask() may
452− * have gotten split before reaching
453− * munlock_vma_page(), so we need to compute
454− * the page_mask here instead.
455− */
456− page_mask = munlock_vma_page(page);
457− unlock_page(page);
458− put_page(page); /* follow_page_mask() */
459− } else {
460− /*
461− * Non-huge pages are handled in batches via
462− * pagevec. The pin from follow_page_mask()
463− * prevents them from collapsing by THP.
464− */
465− pagevec_add(&pvec, page);
466− zone = page_zone(page);
467−
468− /*
469− * Try to fill the rest of pagevec using fast
470− * pte walk. This will also update start to
471− * the next page to process. Then munlock the
472− * pagevec.
473− */
474− start = __munlock_pagevec_fill(&pvec, vma,
475− zone, start, end);
476− __munlock_pagevec(&pvec, zone);
477− goto next;
478− }
479− }
480− page_increm = 1 + page_mask;
481− start += page_increm * PAGE_SIZE;
482−next:
483− cond_resched();
359+ lru_add_drain();
360+ walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL);
361+ lru_add_drain();
362+
363+ if (newflags & VM_IO) {
364+ newflags &= ~VM_IO;
365+ WRITE_ONCE(vma->vm_flags, newflags);
484366 }
485367 }
486368
@@ -500,10 +382,9 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, · 13 unmodified lines
500382 pgoff_t pgoff;
501383 int nr_pages;
502384 int ret = 0;
503− int lock = !!(newflags & VM_LOCKED);
504− vm_flags_t old_flags = vma->vm_flags;
385+ vm_flags_t oldflags = vma->vm_flags;
505386
506− if (newflags == vma->vm_flags || (vma->vm_flags & VM_SPECIAL) ||
387+ if (newflags == oldflags || (oldflags & VM_SPECIAL) ||
507388 is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm) ||
508389 vma_is_dax(vma) || vma_is_secretmem(vma))
509390 /* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */
@@ -535,9 +416,9 @@ success: · 25 unmodified lines
535416 * Keep track of amount of locked VM.
536417 */
537418 nr_pages = (end - start) >> PAGE_SHIFT;
538− if (!lock)
419+ if (!(newflags & VM_LOCKED))
539420 nr_pages = -nr_pages;
540− else if (old_flags & VM_LOCKED)
421+ else if (oldflags & VM_LOCKED)
541422 nr_pages = 0;
542423 mm->locked_vm += nr_pages;
543424
@@ -547,11 +428,12 @@ success: · 3 unmodified lines
547428 * set VM_LOCKED, populate_vma_page_range will bring it back.
548429 */
549430
550− if (lock)
431+ if ((newflags & VM_LOCKED) && (oldflags & VM_LOCKED)) {
432+ /* No work to do, and mlocking twice would be wrong */
551433 vma->vm_flags = newflags;
552− else
553− munlock_vma_pages_range(vma, start, end);
554−
434+ } else {
435+ mlock_vma_pages_range(vma, start, end, newflags);
436+ }
555437 out:
556438 *prev = vma;
557439 return ret;
@@ -645,6 +527,18 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, · 87 unmodified lines
645527 return count >> PAGE_SHIFT;
646528 }
647529
530+/*
531+ * convert get_user_pages() return value to posix mlock() error
532+ */
533+static int __mlock_posix_error_return(long retval)
534+{
535+ if (retval == -EFAULT)
536+ retval = -ENOMEM;
537+ else if (retval == -ENOMEM)
538+ retval = -EAGAIN;
539+ return retval;
540+}
541+
648542 static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
649543 {
650544 unsigned long locked;
mm/mmap.c+2−30
@@ -2672,6 +2672,8 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, · 2671 unmodified lines
26722672 vma->vm_prev = NULL;
26732673 do {
26742674 vma_rb_erase(vma, &mm->mm_rb);
2675+ if (vma->vm_flags & VM_LOCKED)
2676+ mm->locked_vm -= vma_pages(vma);
26752677 mm->map_count--;
26762678 tail_vma = vma;
26772679 vma = vma->vm_next;
@@ -2776,22 +2778,6 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, · 98 unmodified lines
27762778 return __split_vma(mm, vma, addr, new_below);
27772779 }
27782780
2779−static inline void
2780−unlock_range(struct vm_area_struct *start, unsigned long limit)
2781−{
2782− struct mm_struct *mm = start->vm_mm;
2783− struct vm_area_struct *tmp = start;
2784−
2785− while (tmp && tmp->vm_start < limit) {
2786− if (tmp->vm_flags & VM_LOCKED) {
2787− mm->locked_vm -= vma_pages(tmp);
2788− munlock_vma_pages_all(tmp);
2789− }
2790−
2791− tmp = tmp->vm_next;
2792− }
2793−}
2794−
27952781 /* Munmap is split into 2 main parts -- this part which finds
27962782 * what needs doing, and the areas themselves, which do the
27972783 * work. This now handles partial unmappings.
@@ -2872,12 +2858,6 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, · 74 unmodified lines
28722858 return error;
28732859 }
28742860
2875− /*
2876− * unlock any mlock()ed ranges before detaching vmas
2877− */
2878− if (mm->locked_vm)
2879− unlock_range(vma, end);
2880−
28812861 /* Detach vmas from rbtree */
28822862 if (!detach_vmas_to_be_unmapped(mm, vma, prev, end))
28832863 downgrade = false;
@@ -3145,20 +3125,12 @@ void exit_mmap(struct mm_struct *mm) · 261 unmodified lines
31453125 * Nothing can be holding mm->mmap_lock here and the above call
31463126 * to mmu_notifier_release(mm) ensures mmu notifier callbacks in
31473127 * __oom_reap_task_mm() will not block.
3148− *
3149− * This needs to be done before calling unlock_range(),
3150− * which clears VM_LOCKED, otherwise the oom reaper cannot
3151− * reliably test it.
31523128 */
31533129 (void)__oom_reap_task_mm(mm);
3154−
31553130 set_bit(MMF_OOM_SKIP, &mm->flags);
31563131 }
31573132
31583133 mmap_write_lock(mm);
3159− if (mm->locked_vm)
3160− unlock_range(mm->mmap, ULONG_MAX);
3161−
31623134 arch_exit_mmap(mm);
31633135
31643136 vma = mm->mmap;
mm/mmzone.c+7−0
@@ -81,6 +81,13 @@ void lruvec_init(struct lruvec *lruvec) · 80 unmodified lines
8181
8282 for_each_lru(lru)
8383 INIT_LIST_HEAD(&lruvec->lists[lru]);
84+ /*
85+ * The "Unevictable LRU" is imaginary: though its size is maintained,
86+ * it is never scanned, and unevictable pages are not threaded on it
87+ * (so that their lru fields can be reused to hold mlock_count).
88+ * Poison its list head, so that any operations on it would crash.
89+ */
90+ list_del(&lruvec->lists[LRU_UNEVICTABLE]);
8491 }
8592
8693 #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS)
mm/oom_kill.c+1−1
@@ -523,7 +523,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) · 522 unmodified lines
523523 set_bit(MMF_UNSTABLE, &mm->flags);
524524
525525 for (vma = mm->mmap ; vma; vma = vma->vm_next) {
526− if (!can_madv_lru_vma(vma))
526+ if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP))
527527 continue;
528528
529529 /*
mm/page_alloc.c+1−2
@@ -734,8 +734,7 @@ static void prep_compound_head(struct page *page, unsigned int order) · 733 unmodified lines
734734 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
735735 set_compound_order(page, order);
736736 atomic_set(compound_mapcount_ptr(page), -1);
737− if (hpage_pincount_available(page))
738− atomic_set(compound_pincount_ptr(page), 0);
737+ atomic_set(compound_pincount_ptr(page), 0);
739738 }
740739
741740 static void prep_compound_tail(struct page *head, int tail_idx)
mm/page_idle.c+14−16
@@ -13,6 +13,8 @@ · 12 unmodified lines
1313 #include <linux/page_ext.h>
1414 #include <linux/page_idle.h>
1515
16+#include "internal.h"
17+
1618 #define BITMAP_CHUNK_SIZE sizeof(u64)
1719 #define BITMAP_CHUNK_BITS (BITMAP_CHUNK_SIZE * BITS_PER_BYTE)
1820
@@ -44,15 +46,11 @@ static struct page *page_idle_get_page(unsigned long pfn) · 25 unmodified lines
4446 return page;
4547 }
4648
47−static bool page_idle_clear_pte_refs_one(struct page *page,
49+static bool page_idle_clear_pte_refs_one(struct folio *folio,
4850 struct vm_area_struct *vma,
4951 unsigned long addr, void *arg)
5052 {
51− struct page_vma_mapped_walk pvmw = {
52− .page = page,
53− .vma = vma,
54− .address = addr,
55− };
53+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
5654 bool referenced = false;
5755
5856 while (page_vma_mapped_walk(&pvmw)) {
@@ -74,41 +72,41 @@ static bool page_idle_clear_pte_refs_one(struct page *page, · 15 unmodified lines
7472 }
7573
7674 if (referenced) {
77− clear_page_idle(page);
75+ folio_clear_idle(folio);
7876 /*
7977 * We cleared the referenced bit in a mapping to this page. To
8078 * avoid interference with page reclaim, mark it young so that
81− * page_referenced() will return > 0.
79+ * folio_referenced() will return > 0.
8280 */
83− set_page_young(page);
81+ folio_set_young(folio);
8482 }
8583 return true;
8684 }
8785
8886 static void page_idle_clear_pte_refs(struct page *page)
8987 {
88+ struct folio *folio = page_folio(page);
9089 /*
9190 * Since rwc.arg is unused, rwc is effectively immutable, so we
9291 * can make it static const to save some cycles and stack.
9392 */
9493 static const struct rmap_walk_control rwc = {
9594 .rmap_one = page_idle_clear_pte_refs_one,
96− .anon_lock = page_lock_anon_vma_read,
95+ .anon_lock = folio_lock_anon_vma_read,
9796 };
9897 bool need_lock;
9998
100− if (!page_mapped(page) ||
101− !page_rmapping(page))
99+ if (!folio_mapped(folio) || !folio_raw_mapping(folio))
102100 return;
103101
104− need_lock = !PageAnon(page) || PageKsm(page);
105− if (need_lock && !trylock_page(page))
102+ need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
103+ if (need_lock && !folio_trylock(folio))
106104 return;
107105
108− rmap_walk(page, (struct rmap_walk_control *)&rwc);
106+ rmap_walk(folio, &rwc);
109107
110108 if (need_lock)
111− unlock_page(page);
109+ folio_unlock(folio);
112110 }
113111
114112 static ssize_t page_idle_bitmap_read(struct file *file, struct kobject *kobj,
mm/page_vma_mapped.c+26−32
@@ -53,18 +53,6 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw) · 52 unmodified lines
5353 return true;
5454 }
5555
56−static inline bool pfn_is_match(struct page *page, unsigned long pfn)
57−{
58− unsigned long page_pfn = page_to_pfn(page);
59−
60− /* normal page and hugetlbfs page */
61− if (!PageTransCompound(page) || PageHuge(page))
62− return page_pfn == pfn;
63−
64− /* THP can be referenced by any subpage */
65− return pfn >= page_pfn && pfn - page_pfn < thp_nr_pages(page);
66−}
67−
6856 /**
6957 * check_pte - check if @pvmw->page is mapped at the @pvmw->pte
7058 * @pvmw: page_vma_mapped_walk struct, includes a pair pte and page for checking
@@ -116,7 +104,17 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw) · 45 unmodified lines
116104 pfn = pte_pfn(*pvmw->pte);
117105 }
118106
119− return pfn_is_match(pvmw->page, pfn);
107+ return (pfn - pvmw->pfn) < pvmw->nr_pages;
108+}
109+
110+/* Returns true if the two ranges overlap. Careful to not overflow. */
111+static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw)
112+{
113+ if ((pfn + HPAGE_PMD_NR - 1) < pvmw->pfn)
114+ return false;
115+ if (pfn > pvmw->pfn + pvmw->nr_pages - 1)
116+ return false;
117+ return true;
120118 }
121119
122120 static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size)
@@ -127,7 +125,7 @@ static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size) · 4 unmodified lines
127125 }
128126
129127 /**
130− * page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at
128+ * page_vma_mapped_walk - check if @pvmw->pfn is mapped in @pvmw->vma at
131129 * @pvmw->address
132130 * @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
133131 * must be set. pmd, pte and ptl must be NULL.
@@ -152,8 +150,8 @@ static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size) · 18 unmodified lines
152150 */
153151 bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
154152 {
155− struct mm_struct *mm = pvmw->vma->vm_mm;
156− struct page *page = pvmw->page;
153+ struct vm_area_struct *vma = pvmw->vma;
154+ struct mm_struct *mm = vma->vm_mm;
157155 unsigned long end;
158156 pgd_t *pgd;
159157 p4d_t *p4d;
@@ -164,32 +162,26 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) · 4 unmodified lines
164162 if (pvmw->pmd && !pvmw->pte)
165163 return not_found(pvmw);
166164
167− if (unlikely(PageHuge(page))) {
165+ if (unlikely(is_vm_hugetlb_page(vma))) {
166+ unsigned long size = pvmw->nr_pages * PAGE_SIZE;
168167 /* The only possible mapping was handled on last iteration */
169168 if (pvmw->pte)
170169 return not_found(pvmw);
171170
172171 /* when pud is not present, pte will be NULL */
173− pvmw->pte = huge_pte_offset(mm, pvmw->address, page_size(page));
172+ pvmw->pte = huge_pte_offset(mm, pvmw->address, size);
174173 if (!pvmw->pte)
175174 return false;
176175
177− pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte);
176+ pvmw->ptl = huge_pte_lockptr(size_to_hstate(size), mm,
177+ pvmw->pte);
178178 spin_lock(pvmw->ptl);
179179 if (!check_pte(pvmw))
180180 return not_found(pvmw);
181181 return true;
182182 }
183183
184− /*
185− * Seek to next pte only makes sense for THP.
186− * But more important than that optimization, is to filter out
187− * any PageKsm page: whose page->index misleads vma_address()
188− * and vma_address_end() to disaster.
189− */
190− end = PageTransCompound(page) ?
191− vma_address_end(page, pvmw->vma) :
192− pvmw->address + PAGE_SIZE;
184+ end = vma_address_end(pvmw);
193185 if (pvmw->pte)
194186 goto next_pte;
195187 restart:
@@ -224,7 +216,7 @@ restart: · 28 unmodified lines
224216 if (likely(pmd_trans_huge(pmde))) {
225217 if (pvmw->flags & PVMW_MIGRATION)
226218 return not_found(pvmw);
227− if (pmd_page(pmde) != page)
219+ if (!check_pmd(pmd_pfn(pmde), pvmw))
228220 return not_found(pvmw);
229221 return true;
230222 }
@@ -236,7 +228,7 @@ restart: · 5 unmodified lines
236228 return not_found(pvmw);
237229 entry = pmd_to_swp_entry(pmde);
238230 if (!is_migration_entry(entry) ||
239− pfn_swap_entry_to_page(entry) != page)
231+ !check_pmd(swp_offset(entry), pvmw))
240232 return not_found(pvmw);
241233 return true;
242234 }
@@ -250,7 +242,8 @@ restart: · 7 unmodified lines
250242 * cleared *pmd but not decremented compound_mapcount().
251243 */
252244 if ((pvmw->flags & PVMW_SYNC) &&
253− PageTransCompound(page)) {
245+ transparent_hugepage_active(vma) &&
246+ (pvmw->nr_pages >= HPAGE_PMD_NR)) {
254247 spinlock_t *ptl = pmd_lock(mm, pvmw->pmd);
255248
256249 spin_unlock(ptl);
@@ -307,7 +300,8 @@ next_pte: · 50 unmodified lines
307300 int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
308301 {
309302 struct page_vma_mapped_walk pvmw = {
310− .page = page,
303+ .pfn = page_to_pfn(page),
304+ .nr_pages = 1,
311305 .vma = vma,
312306 .flags = PVMW_SYNC,
313307 };
mm/readahead.c+100−8
@@ -262,7 +262,7 @@ static void read_pages(struct readahead_control *rac, struct list_head *pages, · 261 unmodified lines
262262
263263 blk_finish_plug(&plug);
264264
265− BUG_ON(!list_empty(pages));
265+ BUG_ON(pages && !list_empty(pages));
266266 BUG_ON(readahead_count(rac));
267267
268268 out:
@@ -361,7 +361,7 @@ EXPORT_SYMBOL_GPL(page_cache_ra_unbounded); · 92 unmodified lines
361361 * behaviour which would occur if page allocations are causing VM writeback.
362362 * We really don't want to intermingle reads and writes like that.
363363 */
364−void do_page_cache_ra(struct readahead_control *ractl,
364+static void do_page_cache_ra(struct readahead_control *ractl,
365365 unsigned long nr_to_read, unsigned long lookahead_size)
366366 {
367367 struct inode *inode = ractl->mapping->host;
@@ -545,11 +545,103 @@ static int try_context_readahead(struct address_space *mapping, · 177 unmodified lines
545545 return 1;
546546 }
547547
548+/*
549+ * There are some parts of the kernel which assume that PMD entries
550+ * are exactly HPAGE_PMD_ORDER. Those should be fixed, but until then,
551+ * limit the maximum allocation order to PMD size. I'm not aware of any
552+ * assumptions about maximum order if THP are disabled, but 8 seems like
553+ * a good order (that's 1MB if you're using 4kB pages)
554+ */
555+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
556+#define MAX_PAGECACHE_ORDER HPAGE_PMD_ORDER
557+#else
558+#define MAX_PAGECACHE_ORDER 8
559+#endif
560+
561+static inline int ra_alloc_folio(struct readahead_control *ractl, pgoff_t index,
562+ pgoff_t mark, unsigned int order, gfp_t gfp)
563+{
564+ int err;
565+ struct folio *folio = filemap_alloc_folio(gfp, order);
566+
567+ if (!folio)
568+ return -ENOMEM;
569+ if (mark - index < (1UL << order))
570+ folio_set_readahead(folio);
571+ err = filemap_add_folio(ractl->mapping, folio, index, gfp);
572+ if (err)
573+ folio_put(folio);
574+ else
575+ ractl->_nr_pages += 1UL << order;
576+ return err;
577+}
578+
579+void page_cache_ra_order(struct readahead_control *ractl,
580+ struct file_ra_state *ra, unsigned int new_order)
581+{
582+ struct address_space *mapping = ractl->mapping;
583+ pgoff_t index = readahead_index(ractl);
584+ pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;
585+ pgoff_t mark = index + ra->size - ra->async_size;
586+ int err = 0;
587+ gfp_t gfp = readahead_gfp_mask(mapping);
588+
589+ if (!mapping_large_folio_support(mapping) || ra->size < 4)
590+ goto fallback;
591+
592+ limit = min(limit, index + ra->size - 1);
593+
594+ if (new_order < MAX_PAGECACHE_ORDER) {
595+ new_order += 2;
596+ if (new_order > MAX_PAGECACHE_ORDER)
597+ new_order = MAX_PAGECACHE_ORDER;
598+ while ((1 << new_order) > ra->size)
599+ new_order--;
600+ }
601+
602+ while (index <= limit) {
603+ unsigned int order = new_order;
604+
605+ /* Align with smaller pages if needed */
606+ if (index & ((1UL << order) - 1)) {
607+ order = __ffs(index);
608+ if (order == 1)
609+ order = 0;
610+ }
611+ /* Don't allocate pages past EOF */
612+ while (index + (1UL << order) - 1 > limit) {
613+ if (--order == 1)
614+ order = 0;
615+ }
616+ err = ra_alloc_folio(ractl, index, mark, order, gfp);
617+ if (err)
618+ break;
619+ index += 1UL << order;
620+ }
621+
622+ if (index > limit) {
623+ ra->size += index - limit - 1;
624+ ra->async_size += index - limit - 1;
625+ }
626+
627+ read_pages(ractl, NULL, false);
628+
629+ /*
630+ * If there were already pages in the page cache, then we may have
631+ * left some gaps. Let the regular readahead code take care of this
632+ * situation.
633+ */
634+ if (!err)
635+ return;
636+fallback:
637+ do_page_cache_ra(ractl, ra->size, ra->async_size);
638+}
639+
548640 /*
549641 * A minimal readahead algorithm for trivial sequential/random reads.
550642 */
551643 static void ondemand_readahead(struct readahead_control *ractl,
552− bool hit_readahead_marker, unsigned long req_size)
644+ struct folio *folio, unsigned long req_size)
553645 {
554646 struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);
555647 struct file_ra_state *ra = ractl->ra;
@@ -584,12 +676,12 @@ static void ondemand_readahead(struct readahead_control *ractl, · 28 unmodified lines
584676 }
585677
586678 /*
587− * Hit a marked page without valid readahead state.
679+ * Hit a marked folio without valid readahead state.
588680 * E.g. interleaved reads.
589681 * Query the pagecache for async_size, which normally equals to
590682 * readahead size. Ramp it up and use it as the new readahead size.
591683 */
592− if (hit_readahead_marker) {
684+ if (folio) {
593685 pgoff_t start;
594686
595687 rcu_read_lock();
@@ -662,7 +754,7 @@ readit: · 66 unmodified lines
662754 }
663755
664756 ractl->_index = ra->start;
665− do_page_cache_ra(ractl, ra->size, ra->async_size);
757+ page_cache_ra_order(ractl, ra, folio ? folio_order(folio) : 0);
666758 }
667759
668760 void page_cache_sync_ra(struct readahead_control *ractl,
@@ -690,7 +782,7 @@ void page_cache_sync_ra(struct readahead_control *ractl, · 21 unmodified lines
690782 }
691783
692784 /* do read-ahead */
693− ondemand_readahead(ractl, false, req_count);
785+ ondemand_readahead(ractl, NULL, req_count);
694786 }
695787 EXPORT_SYMBOL_GPL(page_cache_sync_ra);
696788
@@ -713,7 +805,7 @@ void page_cache_async_ra(struct readahead_control *ractl, · 16 unmodified lines
713805 return;
714806
715807 /* do read-ahead */
716− ondemand_readahead(ractl, true, req_count);
808+ ondemand_readahead(ractl, folio, req_count);
717809 }
718810 EXPORT_SYMBOL_GPL(page_cache_async_ra);
719811
mm/rmap.c+239−339
@@ -107,15 +107,15 @@ static inline void anon_vma_free(struct anon_vma *anon_vma) · 106 unmodified lines
107107 VM_BUG_ON(atomic_read(&anon_vma->refcount));
108108
109109 /*
110− * Synchronize against page_lock_anon_vma_read() such that
110+ * Synchronize against folio_lock_anon_vma_read() such that
111111 * we can safely hold the lock without the anon_vma getting
112112 * freed.
113113 *
114114 * Relies on the full mb implied by the atomic_dec_and_test() from
115115 * put_anon_vma() against the acquire barrier implied by
116− * down_read_trylock() from page_lock_anon_vma_read(). This orders:
116+ * down_read_trylock() from folio_lock_anon_vma_read(). This orders:
117117 *
118− * page_lock_anon_vma_read() VS put_anon_vma()
118+ * folio_lock_anon_vma_read() VS put_anon_vma()
119119 * down_read_trylock() atomic_dec_and_test()
120120 * LOCK MB
121121 * atomic_read() rwsem_is_locked()
@@ -168,7 +168,7 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, · 46 unmodified lines
168168 * allocate a new one.
169169 *
170170 * Anon-vma allocations are very subtle, because we may have
171− * optimistically looked up an anon_vma in page_lock_anon_vma_read()
171+ * optimistically looked up an anon_vma in folio_lock_anon_vma_read()
172172 * and that may actually touch the rwsem even in the newly
173173 * allocated vma (it depends on RCU to make sure that the
174174 * anon_vma isn't actually destroyed).
@@ -526,28 +526,28 @@ out: · 351 unmodified lines
526526 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
527527 * reference like with page_get_anon_vma() and then block on the mutex.
528528 */
529−struct anon_vma *page_lock_anon_vma_read(struct page *page)
529+struct anon_vma *folio_lock_anon_vma_read(struct folio *folio)
530530 {
531531 struct anon_vma *anon_vma = NULL;
532532 struct anon_vma *root_anon_vma;
533533 unsigned long anon_mapping;
534534
535535 rcu_read_lock();
536− anon_mapping = (unsigned long)READ_ONCE(page->mapping);
536+ anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
537537 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
538538 goto out;
539− if (!page_mapped(page))
539+ if (!folio_mapped(folio))
540540 goto out;
541541
542542 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
543543 root_anon_vma = READ_ONCE(anon_vma->root);
544544 if (down_read_trylock(&root_anon_vma->rwsem)) {
545545 /*
546− * If the page is still mapped, then this anon_vma is still
546+ * If the folio is still mapped, then this anon_vma is still
547547 * its anon_vma, and holding the mutex ensures that it will
548548 * not go away, see anon_vma_free().
549549 */
550− if (!page_mapped(page)) {
550+ if (!folio_mapped(folio)) {
551551 up_read(&root_anon_vma->rwsem);
552552 anon_vma = NULL;
553553 }
@@ -560,7 +560,7 @@ struct anon_vma *page_lock_anon_vma_read(struct page *page) · 6 unmodified lines
560560 goto out;
561561 }
562562
563− if (!page_mapped(page)) {
563+ if (!folio_mapped(folio)) {
564564 rcu_read_unlock();
565565 put_anon_vma(anon_vma);
566566 return NULL;
@@ -737,8 +737,9 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) · 170 unmodified lines
737737 */
738738 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
739739 {
740− if (PageAnon(page)) {
741− struct anon_vma *page__anon_vma = page_anon_vma(page);
740+ struct folio *folio = page_folio(page);
741+ if (folio_test_anon(folio)) {
742+ struct anon_vma *page__anon_vma = folio_anon_vma(folio);
742743 /*
743744 * Note: swapoff's unuse_vma() is more efficient with this
744745 * check, and needs it to match anon_vma when KSM is active.
@@ -748,7 +749,7 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) · 3 unmodified lines
748749 return -EFAULT;
749750 } else if (!vma->vm_file) {
750751 return -EFAULT;
751− } else if (vma->vm_file->f_mapping != compound_head(page)->mapping) {
752+ } else if (vma->vm_file->f_mapping != folio->mapping) {
752753 return -EFAULT;
753754 }
754755
@@ -789,30 +790,29 @@ out: · 34 unmodified lines
789790 return pmd;
790791 }
791792
792−struct page_referenced_arg {
793+struct folio_referenced_arg {
793794 int mapcount;
794795 int referenced;
795796 unsigned long vm_flags;
796797 struct mem_cgroup *memcg;
797798 };
798799 /*
799− * arg: page_referenced_arg will be passed
800+ * arg: folio_referenced_arg will be passed
800801 */
801−static bool page_referenced_one(struct page *page, struct vm_area_struct *vma,
802− unsigned long address, void *arg)
802+static bool folio_referenced_one(struct folio *folio,
803+ struct vm_area_struct *vma, unsigned long address, void *arg)
803804 {
804− struct page_referenced_arg *pra = arg;
805− struct page_vma_mapped_walk pvmw = {
806− .page = page,
807− .vma = vma,
808− .address = address,
809− };
805+ struct folio_referenced_arg *pra = arg;
806+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
810807 int referenced = 0;
811808
812809 while (page_vma_mapped_walk(&pvmw)) {
813810 address = pvmw.address;
814811
815− if (vma->vm_flags & VM_LOCKED) {
812+ if ((vma->vm_flags & VM_LOCKED) &&
813+ (!folio_test_large(folio) || !pvmw.pte)) {
814+ /* Restore the mlock which got missed */
815+ mlock_vma_folio(folio, vma, !pvmw.pte);
816816 page_vma_mapped_walk_done(&pvmw);
817817 pra->vm_flags |= VM_LOCKED;
818818 return false; /* To break the loop */
@@ -824,10 +824,10 @@ static bool page_referenced_one(struct page *page, struct vm_area_struct *vma, · 5 unmodified lines
824824 /*
825825 * Don't treat a reference through
826826 * a sequentially read mapping as such.
827− * If the page has been used in another mapping,
827+ * If the folio has been used in another mapping,
828828 * we will catch it; if this other mapping is
829829 * already gone, the unmap path will have set
830− * PG_referenced or activated the page.
830+ * the referenced flag or activated the folio.
831831 */
832832 if (likely(!(vma->vm_flags & VM_SEQ_READ)))
833833 referenced++;
@@ -837,7 +837,7 @@ static bool page_referenced_one(struct page *page, struct vm_area_struct *vma, · 3 unmodified lines
837837 pvmw.pmd))
838838 referenced++;
839839 } else {
840− /* unexpected pmd-mapped page? */
840+ /* unexpected pmd-mapped folio? */
841841 WARN_ON_ONCE(1);
842842 }
843843
@@ -845,13 +845,13 @@ static bool page_referenced_one(struct page *page, struct vm_area_struct *vma, · 1 unmodified line
845845 }
846846
847847 if (referenced)
848− clear_page_idle(page);
849− if (test_and_clear_page_young(page))
848+ folio_clear_idle(folio);
849+ if (folio_test_clear_young(folio))
850850 referenced++;
851851
852852 if (referenced) {
853853 pra->referenced++;
854− pra->vm_flags |= vma->vm_flags;
854+ pra->vm_flags |= vma->vm_flags & ~VM_LOCKED;
855855 }
856856
857857 if (!pra->mapcount)
@@ -860,9 +860,9 @@ static bool page_referenced_one(struct page *page, struct vm_area_struct *vma, · 2 unmodified lines
860860 return true;
861861 }
862862
863−static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg)
863+static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
864864 {
865− struct page_referenced_arg *pra = arg;
865+ struct folio_referenced_arg *pra = arg;
866866 struct mem_cgroup *memcg = pra->memcg;
867867
868868 if (!mm_match_cgroup(vma->vm_mm, memcg))
@@ -872,40 +872,39 @@ static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg) · 3 unmodified lines
872872 }
873873
874874 /**
875− * page_referenced - test if the page was referenced
876− * @page: the page to test
877− * @is_locked: caller holds lock on the page
875+ * folio_referenced() - Test if the folio was referenced.
876+ * @folio: The folio to test.
877+ * @is_locked: Caller holds lock on the folio.
878878 * @memcg: target memory cgroup
879− * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
879+ * @vm_flags: A combination of all the vma->vm_flags which referenced the folio.
880880 *
881− * Quick test_and_clear_referenced for all mappings to a page,
882− * returns the number of ptes which referenced the page.
881+ * Quick test_and_clear_referenced for all mappings of a folio,
882+ *
883+ * Return: The number of mappings which referenced the folio.
883884 */
884−int page_referenced(struct page *page,
885− int is_locked,
886− struct mem_cgroup *memcg,
887− unsigned long *vm_flags)
885+int folio_referenced(struct folio *folio, int is_locked,
886+ struct mem_cgroup *memcg, unsigned long *vm_flags)
888887 {
889888 int we_locked = 0;
890− struct page_referenced_arg pra = {
891− .mapcount = total_mapcount(page),
889+ struct folio_referenced_arg pra = {
890+ .mapcount = folio_mapcount(folio),
892891 .memcg = memcg,
893892 };
894893 struct rmap_walk_control rwc = {
895− .rmap_one = page_referenced_one,
894+ .rmap_one = folio_referenced_one,
896895 .arg = (void *)&pra,
897− .anon_lock = page_lock_anon_vma_read,
896+ .anon_lock = folio_lock_anon_vma_read,
898897 };
899898
900899 *vm_flags = 0;
901900 if (!pra.mapcount)
902901 return 0;
903902
904− if (!page_rmapping(page))
903+ if (!folio_raw_mapping(folio))
905904 return 0;
906905
907− if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
908− we_locked = trylock_page(page);
906+ if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) {
907+ we_locked = folio_trylock(folio);
909908 if (!we_locked)
910909 return 1;
911910 }
@@ -916,37 +915,32 @@ int page_referenced(struct page *page, · 4 unmodified lines
916915 * cgroups
917916 */
918917 if (memcg) {
919− rwc.invalid_vma = invalid_page_referenced_vma;
918+ rwc.invalid_vma = invalid_folio_referenced_vma;
920919 }
921920
922− rmap_walk(page, &rwc);
921+ rmap_walk(folio, &rwc);
923922 *vm_flags = pra.vm_flags;
924923
925924 if (we_locked)
926− unlock_page(page);
925+ folio_unlock(folio);
927926
928927 return pra.referenced;
929928 }
930929
931−static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
930+static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma,
932931 unsigned long address, void *arg)
933932 {
934− struct page_vma_mapped_walk pvmw = {
935− .page = page,
936− .vma = vma,
937− .address = address,
938− .flags = PVMW_SYNC,
939− };
933+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
940934 struct mmu_notifier_range range;
941935 int *cleaned = arg;
942936
943937 /*
944938 * We have to assume the worse case ie pmd for invalidation. Note that
945− * the page can not be free from this function.
939+ * the folio can not be freed from this function.
946940 */
947941 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
948942 0, vma, vma->vm_mm, address,
949− vma_address_end(page, vma));
943+ vma_address_end(&pvmw));
950944 mmu_notifier_invalidate_range_start(&range);
951945
952946 while (page_vma_mapped_walk(&pvmw)) {
@@ -974,14 +968,14 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, · 21 unmodified lines
974968 if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
975969 continue;
976970
977− flush_cache_page(vma, address, page_to_pfn(page));
971+ flush_cache_page(vma, address, folio_pfn(folio));
978972 entry = pmdp_invalidate(vma, address, pmd);
979973 entry = pmd_wrprotect(entry);
980974 entry = pmd_mkclean(entry);
981975 set_pmd_at(vma->vm_mm, address, pmd, entry);
982976 ret = 1;
983977 #else
984− /* unexpected pmd-mapped page? */
978+ /* unexpected pmd-mapped folio? */
985979 WARN_ON_ONCE(1);
986980 #endif
987981 }
@@ -1029,7 +1023,7 @@ int folio_mkclean(struct folio *folio) · 41 unmodified lines
10291023 if (!mapping)
10301024 return 0;
10311025
1032− rmap_walk(&folio->page, &rwc);
1026+ rmap_walk(folio, &rwc);
10331027
10341028 return cleaned;
10351029 }
@@ -1057,8 +1051,8 @@ void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma) · 21 unmodified lines
10571051 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
10581052 /*
10591053 * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
1060− * simultaneously, so a concurrent reader (eg page_referenced()'s
1061− * PageAnon()) will not see one without the other.
1054+ * simultaneously, so a concurrent reader (eg folio_referenced()'s
1055+ * folio_test_anon()) will not see one without the other.
10621056 */
10631057 WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);
10641058 }
@@ -1108,6 +1102,7 @@ static void __page_set_anon_rmap(struct page *page, · 43 unmodified lines
11081102 static void __page_check_anon_rmap(struct page *page,
11091103 struct vm_area_struct *vma, unsigned long address)
11101104 {
1105+ struct folio *folio = page_folio(page);
11111106 /*
11121107 * The page's anon-rmap details (mapping and index) are guaranteed to
11131108 * be set up correctly at this point.
@@ -1119,7 +1114,8 @@ static void __page_check_anon_rmap(struct page *page, · 5 unmodified lines
11191114 * are initially only visible via the pagetables, and the pte is locked
11201115 * over the call to page_add_new_anon_rmap.
11211116 */
1122− VM_BUG_ON_PAGE(page_anon_vma(page)->root != vma->anon_vma->root, page);
1117+ VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
1118+ folio);
11231119 VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),
11241120 page);
11251121 }
@@ -1181,17 +1177,17 @@ void do_page_add_anon_rmap(struct page *page, · 55 unmodified lines
11811177 __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr);
11821178 }
11831179
1184− if (unlikely(PageKsm(page))) {
1180+ if (unlikely(PageKsm(page)))
11851181 unlock_page_memcg(page);
1186− return;
1187− }
11881182
11891183 /* address might be in next vma when migration races vma_adjust */
1190− if (first)
1184+ else if (first)
11911185 __page_set_anon_rmap(page, vma, address,
11921186 flags & RMAP_EXCLUSIVE);
11931187 else
11941188 __page_check_anon_rmap(page, vma, address);
1189+
1190+ mlock_vma_page(page, vma, compound);
11951191 }
11961192
11971193 /**
@@ -1216,8 +1212,7 @@ void page_add_new_anon_rmap(struct page *page, · 18 unmodified lines
12161212 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
12171213 /* increment count (starts at -1) */
12181214 atomic_set(compound_mapcount_ptr(page), 0);
1219− if (hpage_pincount_available(page))
1220− atomic_set(compound_pincount_ptr(page), 0);
1215+ atomic_set(compound_pincount_ptr(page), 0);
12211216
12221217 __mod_lruvec_page_state(page, NR_ANON_THPS, nr);
12231218 } else {
@@ -1232,12 +1227,14 @@ void page_add_new_anon_rmap(struct page *page, · 8 unmodified lines
12321227
12331228 /**
12341229 * page_add_file_rmap - add pte mapping to a file page
1235− * @page: the page to add the mapping to
1236− * @compound: charge the page as compound or small page
1230+ * @page: the page to add the mapping to
1231+ * @vma: the vm area in which the mapping is added
1232+ * @compound: charge the page as compound or small page
12371233 *
12381234 * The caller needs to hold the pte lock.
12391235 */
1240−void page_add_file_rmap(struct page *page, bool compound)
1236+void page_add_file_rmap(struct page *page,
1237+ struct vm_area_struct *vma, bool compound)
12411238 {
12421239 int i, nr = 1;
12431240
@@ -1271,13 +1268,8 @@ void page_add_file_rmap(struct page *page, bool compound) · 27 unmodified lines
12711268 nr_pages);
12721269 } else {
12731270 if (PageTransCompound(page) && page_mapping(page)) {
1274− struct page *head = compound_head(page);
1275−
12761271 VM_WARN_ON_ONCE(!PageLocked(page));
1277−
1278− SetPageDoubleMap(head);
1279− if (PageMlocked(page))
1280− clear_page_mlock(head);
1272+ SetPageDoubleMap(compound_head(page));
12811273 }
12821274 if (!atomic_inc_and_test(&page->_mapcount))
12831275 goto out;
@@ -1285,6 +1277,8 @@ void page_add_file_rmap(struct page *page, bool compound) · 1 unmodified line
12851277 __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr);
12861278 out:
12871279 unlock_page_memcg(page);
1280+
1281+ mlock_vma_page(page, vma, compound);
12881282 }
12891283
12901284 static void page_remove_file_rmap(struct page *page, bool compound)
@@ -1327,9 +1321,6 @@ static void page_remove_file_rmap(struct page *page, bool compound) · 36 unmodified lines
13271321 * pte lock(a spinlock) is held, which implies preemption disabled.
13281322 */
13291323 __mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr);
1330−
1331− if (unlikely(PageMlocked(page)))
1332− clear_page_mlock(page);
13331324 }
13341325
13351326 static void page_remove_anon_compound_rmap(struct page *page)
@@ -1369,9 +1360,6 @@ static void page_remove_anon_compound_rmap(struct page *page) · 33 unmodified lines
13691360 nr = thp_nr_pages(page);
13701361 }
13711362
1372− if (unlikely(PageMlocked(page)))
1373− clear_page_mlock(page);
1374−
13751363 if (nr)
13761364 __mod_lruvec_page_state(page, NR_ANON_MAPPED, -nr);
13771365 }
@@ -1379,11 +1367,13 @@ static void page_remove_anon_compound_rmap(struct page *page) · 1 unmodified line
13791367 /**
13801368 * page_remove_rmap - take down pte mapping from a page
13811369 * @page: page to remove mapping from
1370+ * @vma: the vm area from which the mapping is removed
13821371 * @compound: uncharge the page as compound or small page
13831372 *
13841373 * The caller needs to hold the pte lock.
13851374 */
1386−void page_remove_rmap(struct page *page, bool compound)
1375+void page_remove_rmap(struct page *page,
1376+ struct vm_area_struct *vma, bool compound)
13871377 {
13881378 lock_page_memcg(page);
13891379
@@ -1408,9 +1398,6 @@ void page_remove_rmap(struct page *page, bool compound) · 18 unmodified lines
14081398 */
14091399 __dec_lruvec_page_state(page, NR_ANON_MAPPED);
14101400
1411− if (unlikely(PageMlocked(page)))
1412− clear_page_mlock(page);
1413−
14141401 if (PageTransCompound(page))
14151402 deferred_split_huge_page(compound_head(page));
14161403
@@ -1425,20 +1412,18 @@ void page_remove_rmap(struct page *page, bool compound) · 8 unmodified lines
14251412 */
14261413 out:
14271414 unlock_page_memcg(page);
1415+
1416+ munlock_vma_page(page, vma, compound);
14281417 }
14291418
14301419 /*
14311420 * @arg: enum ttu_flags will be passed to this argument
14321421 */
1433−static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
1422+static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
14341423 unsigned long address, void *arg)
14351424 {
14361425 struct mm_struct *mm = vma->vm_mm;
1437− struct page_vma_mapped_walk pvmw = {
1438− .page = page,
1439− .vma = vma,
1440− .address = address,
1441− };
1426+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
14421427 pte_t pteval;
14431428 struct page *subpage;
14441429 bool ret = true;
@@ -1455,21 +1440,20 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 10 unmodified lines
14551440 pvmw.flags = PVMW_SYNC;
14561441
14571442 if (flags & TTU_SPLIT_HUGE_PMD)
1458− split_huge_pmd_address(vma, address, false, page);
1443+ split_huge_pmd_address(vma, address, false, folio);
14591444
14601445 /*
14611446 * For THP, we have to assume the worse case ie pmd for invalidation.
14621447 * For hugetlb, it could be much worse if we need to do pud
14631448 * invalidation in the case of pmd sharing.
14641449 *
1465− * Note that the page can not be free in this function as call of
1466− * try_to_unmap() must hold a reference on the page.
1450+ * Note that the folio can not be freed in this function as call of
1451+ * try_to_unmap() must hold a reference on the folio.
14671452 */
1468− range.end = PageKsm(page) ?
1469− address + PAGE_SIZE : vma_address_end(page, vma);
1453+ range.end = vma_address_end(&pvmw);
14701454 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
14711455 address, range.end);
1472− if (PageHuge(page)) {
1456+ if (folio_test_hugetlb(folio)) {
14731457 /*
14741458 * If sharing is possible, start and end will be adjusted
14751459 * accordingly.
@@ -1480,32 +1464,26 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 4 unmodified lines
14801464 mmu_notifier_invalidate_range_start(&range);
14811465
14821466 while (page_vma_mapped_walk(&pvmw)) {
1467+ /* Unexpected PMD-mapped THP? */
1468+ VM_BUG_ON_FOLIO(!pvmw.pte, folio);
1469+
14831470 /*
1484− * If the page is mlock()d, we cannot swap it out.
1471+ * If the folio is in an mlock()d vma, we must not swap it out.
14851472 */
14861473 if (!(flags & TTU_IGNORE_MLOCK) &&
14871474 (vma->vm_flags & VM_LOCKED)) {
1488− /*
1489− * PTE-mapped THP are never marked as mlocked: so do
1490− * not set it on a DoubleMap THP, nor on an Anon THP
1491− * (which may still be PTE-mapped after DoubleMap was
1492− * cleared). But stop unmapping even in those cases.
1493− */
1494− if (!PageTransCompound(page) || (PageHead(page) &&
1495− !PageDoubleMap(page) && !PageAnon(page)))
1496− mlock_vma_page(page);
1475+ /* Restore the mlock which got missed */
1476+ mlock_vma_folio(folio, vma, false);
14971477 page_vma_mapped_walk_done(&pvmw);
14981478 ret = false;
14991479 break;
15001480 }
15011481
1502− /* Unexpected PMD-mapped THP? */
1503− VM_BUG_ON_PAGE(!pvmw.pte, page);
1504−
1505− subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
1482+ subpage = folio_page(folio,
1483+ pte_pfn(*pvmw.pte) - folio_pfn(folio));
15061484 address = pvmw.address;
15071485
1508− if (PageHuge(page) && !PageAnon(page)) {
1486+ if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) {
15091487 /*
15101488 * To call huge_pmd_unshare, i_mmap_rwsem must be
15111489 * held in write mode. Caller needs to explicitly
@@ -1544,7 +1522,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 32 unmodified lines
15441522 if (should_defer_flush(mm, flags)) {
15451523 /*
15461524 * We clear the PTE but do not flush so potentially
1547− * a remote CPU could still be writing to the page.
1525+ * a remote CPU could still be writing to the folio.
15481526 * If the entry was previously clean then the
15491527 * architecture must guarantee that a clear->dirty
15501528 * transition on a cached TLB entry is written through
@@ -1557,22 +1535,22 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 6 unmodified lines
15571535 pteval = ptep_clear_flush(vma, address, pvmw.pte);
15581536 }
15591537
1560− /* Move the dirty bit to the page. Now the pte is gone. */
1538+ /* Set the dirty flag on the folio now the pte is gone. */
15611539 if (pte_dirty(pteval))
1562− set_page_dirty(page);
1540+ folio_mark_dirty(folio);
15631541
15641542 /* Update high watermark before we lower rss */
15651543 update_hiwater_rss(mm);
15661544
15671545 if (PageHWPoison(subpage) && !(flags & TTU_IGNORE_HWPOISON)) {
15681546 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
1569− if (PageHuge(page)) {
1570− hugetlb_count_sub(compound_nr(page), mm);
1547+ if (folio_test_hugetlb(folio)) {
1548+ hugetlb_count_sub(folio_nr_pages(folio), mm);
15711549 set_huge_swap_pte_at(mm, address,
15721550 pvmw.pte, pteval,
15731551 vma_mmu_pagesize(vma));
15741552 } else {
1575− dec_mm_counter(mm, mm_counter(page));
1553+ dec_mm_counter(mm, mm_counter(&folio->page));
15761554 set_pte_at(mm, address, pvmw.pte, pteval);
15771555 }
15781556
@@ -1587,18 +1565,19 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 8 unmodified lines
15871565 * migration) will not expect userfaults on already
15881566 * copied pages.
15891567 */
1590− dec_mm_counter(mm, mm_counter(page));
1568+ dec_mm_counter(mm, mm_counter(&folio->page));
15911569 /* We have to invalidate as we cleared the pte */
15921570 mmu_notifier_invalidate_range(mm, address,
15931571 address + PAGE_SIZE);
1594− } else if (PageAnon(page)) {
1572+ } else if (folio_test_anon(folio)) {
15951573 swp_entry_t entry = { .val = page_private(subpage) };
15961574 pte_t swp_pte;
15971575 /*
15981576 * Store the swap location in the pte.
15991577 * See handle_pte_fault() ...
16001578 */
1601− if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) {
1579+ if (unlikely(folio_test_swapbacked(folio) !=
1580+ folio_test_swapcache(folio))) {
16021581 WARN_ON_ONCE(1);
16031582 ret = false;
16041583 /* We have to invalidate as we cleared the pte */
@@ -1609,8 +1588,8 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 4 unmodified lines
16091588 }
16101589
16111590 /* MADV_FREE page check */
1612− if (!PageSwapBacked(page)) {
1613− if (!PageDirty(page)) {
1591+ if (!folio_test_swapbacked(folio)) {
1592+ if (!folio_test_dirty(folio)) {
16141593 /* Invalidate as we cleared the pte */
16151594 mmu_notifier_invalidate_range(mm,
16161595 address, address + PAGE_SIZE);
@@ -1619,11 +1598,11 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 2 unmodified lines
16191598 }
16201599
16211600 /*
1622− * If the page was redirtied, it cannot be
1601+ * If the folio was redirtied, it cannot be
16231602 * discarded. Remap the page to page table.
16241603 */
16251604 set_pte_at(mm, address, pvmw.pte, pteval);
1626− SetPageSwapBacked(page);
1605+ folio_set_swapbacked(folio);
16271606 ret = false;
16281607 page_vma_mapped_walk_done(&pvmw);
16291608 break;
@@ -1660,16 +1639,17 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 30 unmodified lines
16601639 address + PAGE_SIZE);
16611640 } else {
16621641 /*
1663− * This is a locked file-backed page, thus it cannot
1664− * be removed from the page cache and replaced by a new
1665− * page before mmu_notifier_invalidate_range_end, so no
1666− * concurrent thread might update its page table to
1667− * point at new page while a device still is using this
1668− * page.
1642+ * This is a locked file-backed folio,
1643+ * so it cannot be removed from the page
1644+ * cache and replaced by a new folio before
1645+ * mmu_notifier_invalidate_range_end, so no
1646+ * concurrent thread might update its page table
1647+ * to point at a new folio while a device is
1648+ * still using this folio.
16691649 *
16701650 * See Documentation/vm/mmu_notifier.rst
16711651 */
1672− dec_mm_counter(mm, mm_counter_file(page));
1652+ dec_mm_counter(mm, mm_counter_file(&folio->page));
16731653 }
16741654 discard:
16751655 /*
@@ -1679,8 +1659,10 @@ discard: · 3 unmodified lines
16791659 *
16801660 * See Documentation/vm/mmu_notifier.rst
16811661 */
1682− page_remove_rmap(subpage, PageHuge(page));
1683− put_page(page);
1662+ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
1663+ if (vma->vm_flags & VM_LOCKED)
1664+ mlock_page_drain(smp_processor_id());
1665+ folio_put(folio);
16841666 }
16851667
16861668 mmu_notifier_invalidate_range_end(&range);
@@ -1693,35 +1675,35 @@ static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) · 6 unmodified lines
16931675 return vma_is_temporary_stack(vma);
16941676 }
16951677
1696−static int page_not_mapped(struct page *page)
1678+static int page_not_mapped(struct folio *folio)
16971679 {
1698− return !page_mapped(page);
1680+ return !folio_mapped(folio);
16991681 }
17001682
17011683 /**
1702− * try_to_unmap - try to remove all page table mappings to a page
1703− * @page: the page to get unmapped
1684+ * try_to_unmap - Try to remove all page table mappings to a folio.
1685+ * @folio: The folio to unmap.
17041686 * @flags: action and flags
17051687 *
17061688 * Tries to remove all the page table entries which are mapping this
1707− * page, used in the pageout path. Caller must hold the page lock.
1689+ * folio. It is the caller's responsibility to check if the folio is
1690+ * still mapped if needed (use TTU_SYNC to prevent accounting races).
17081691 *
1709− * It is the caller's responsibility to check if the page is still
1710− * mapped when needed (use TTU_SYNC to prevent accounting races).
1692+ * Context: Caller must hold the folio lock.
17111693 */
1712−void try_to_unmap(struct page *page, enum ttu_flags flags)
1694+void try_to_unmap(struct folio *folio, enum ttu_flags flags)
17131695 {
17141696 struct rmap_walk_control rwc = {
17151697 .rmap_one = try_to_unmap_one,
17161698 .arg = (void *)flags,
17171699 .done = page_not_mapped,
1718− .anon_lock = page_lock_anon_vma_read,
1700+ .anon_lock = folio_lock_anon_vma_read,
17191701 };
17201702
17211703 if (flags & TTU_RMAP_LOCKED)
1722− rmap_walk_locked(page, &rwc);
1704+ rmap_walk_locked(folio, &rwc);
17231705 else
1724− rmap_walk(page, &rwc);
1706+ rmap_walk(folio, &rwc);
17251707 }
17261708
17271709 /*
@@ -1730,15 +1712,11 @@ void try_to_unmap(struct page *page, enum ttu_flags flags) · 2 unmodified lines
17301712 * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
17311713 * containing migration entries.
17321714 */
1733−static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
1715+static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
17341716 unsigned long address, void *arg)
17351717 {
17361718 struct mm_struct *mm = vma->vm_mm;
1737− struct page_vma_mapped_walk pvmw = {
1738− .page = page,
1739− .vma = vma,
1740− .address = address,
1741− };
1719+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
17421720 pte_t pteval;
17431721 struct page *subpage;
17441722 bool ret = true;
@@ -1759,7 +1737,7 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma, · 14 unmodified lines
17591737 * TTU_SPLIT_HUGE_PMD and it wants to freeze.
17601738 */
17611739 if (flags & TTU_SPLIT_HUGE_PMD)
1762− split_huge_pmd_address(vma, address, true, page);
1740+ split_huge_pmd_address(vma, address, true, folio);
17631741
17641742 /*
17651743 * For THP, we have to assume the worse case ie pmd for invalidation.
@@ -1769,11 +1747,10 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma, · 3 unmodified lines
17691747 * Note that the page can not be free in this function as call of
17701748 * try_to_unmap() must hold a reference on the page.
17711749 */
1772− range.end = PageKsm(page) ?
1773− address + PAGE_SIZE : vma_address_end(page, vma);
1750+ range.end = vma_address_end(&pvmw);
17741751 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
17751752 address, range.end);
1776− if (PageHuge(page)) {
1753+ if (folio_test_hugetlb(folio)) {
17771754 /*
17781755 * If sharing is possible, start and end will be adjusted
17791756 * accordingly.
@@ -1787,21 +1764,24 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma, · 7 unmodified lines
17871764 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
17881765 /* PMD-mapped THP migration entry */
17891766 if (!pvmw.pte) {
1790− VM_BUG_ON_PAGE(PageHuge(page) ||
1791− !PageTransCompound(page), page);
1767+ subpage = folio_page(folio,
1768+ pmd_pfn(*pvmw.pmd) - folio_pfn(folio));
1769+ VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
1770+ !folio_test_pmd_mappable(folio), folio);
17921771
1793− set_pmd_migration_entry(&pvmw, page);
1772+ set_pmd_migration_entry(&pvmw, subpage);
17941773 continue;
17951774 }
17961775 #endif
17971776
17981777 /* Unexpected PMD-mapped THP? */
1799− VM_BUG_ON_PAGE(!pvmw.pte, page);
1778+ VM_BUG_ON_FOLIO(!pvmw.pte, folio);
18001779
1801− subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
1780+ subpage = folio_page(folio,
1781+ pte_pfn(*pvmw.pte) - folio_pfn(folio));
18021782 address = pvmw.address;
18031783
1804− if (PageHuge(page) && !PageAnon(page)) {
1784+ if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) {
18051785 /*
18061786 * To call huge_pmd_unshare, i_mmap_rwsem must be
18071787 * held in write mode. Caller needs to explicitly
@@ -1839,15 +1819,15 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma, · 31 unmodified lines
18391819 flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
18401820 pteval = ptep_clear_flush(vma, address, pvmw.pte);
18411821
1842− /* Move the dirty bit to the page. Now the pte is gone. */
1822+ /* Set the dirty flag on the folio now the pte is gone. */
18431823 if (pte_dirty(pteval))
1844− set_page_dirty(page);
1824+ folio_mark_dirty(folio);
18451825
18461826 /* Update high watermark before we lower rss */
18471827 update_hiwater_rss(mm);
18481828
1849− if (is_zone_device_page(page)) {
1850− unsigned long pfn = page_to_pfn(page);
1829+ if (folio_is_zone_device(folio)) {
1830+ unsigned long pfn = folio_pfn(folio);
18511831 swp_entry_t entry;
18521832 pte_t swp_pte;
18531833
@@ -1883,16 +1863,16 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma, · 29 unmodified lines
18831863 * changed when hugepage migrations to device private
18841864 * memory are supported.
18851865 */
1886− subpage = page;
1866+ subpage = &folio->page;
18871867 } else if (PageHWPoison(subpage)) {
18881868 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
1889− if (PageHuge(page)) {
1890− hugetlb_count_sub(compound_nr(page), mm);
1869+ if (folio_test_hugetlb(folio)) {
1870+ hugetlb_count_sub(folio_nr_pages(folio), mm);
18911871 set_huge_swap_pte_at(mm, address,
18921872 pvmw.pte, pteval,
18931873 vma_mmu_pagesize(vma));
18941874 } else {
1895− dec_mm_counter(mm, mm_counter(page));
1875+ dec_mm_counter(mm, mm_counter(&folio->page));
18961876 set_pte_at(mm, address, pvmw.pte, pteval);
18971877 }
18981878
@@ -1907,7 +1887,7 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma, · 8 unmodified lines
19071887 * migration) will not expect userfaults on already
19081888 * copied pages.
19091889 */
1910− dec_mm_counter(mm, mm_counter(page));
1890+ dec_mm_counter(mm, mm_counter(&folio->page));
19111891 /* We have to invalidate as we cleared the pte */
19121892 mmu_notifier_invalidate_range(mm, address,
19131893 address + PAGE_SIZE);
@@ -1953,8 +1933,10 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma, · 39 unmodified lines
19531933 *
19541934 * See Documentation/vm/mmu_notifier.rst
19551935 */
1956− page_remove_rmap(subpage, PageHuge(page));
1957− put_page(page);
1936+ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
1937+ if (vma->vm_flags & VM_LOCKED)
1938+ mlock_page_drain(smp_processor_id());
1939+ folio_put(folio);
19581940 }
19591941
19601942 mmu_notifier_invalidate_range_end(&range);
@@ -1964,19 +1946,19 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma, · 3 unmodified lines
19641946
19651947 /**
19661948 * try_to_migrate - try to replace all page table mappings with swap entries
1967− * @page: the page to replace page table entries for
1949+ * @folio: the folio to replace page table entries for
19681950 * @flags: action and flags
19691951 *
1970− * Tries to remove all the page table entries which are mapping this page and
1971− * replace them with special swap entries. Caller must hold the page lock.
1952+ * Tries to remove all the page table entries which are mapping this folio and
1953+ * replace them with special swap entries. Caller must hold the folio lock.
19721954 */
1973−void try_to_migrate(struct page *page, enum ttu_flags flags)
1955+void try_to_migrate(struct folio *folio, enum ttu_flags flags)
19741956 {
19751957 struct rmap_walk_control rwc = {
19761958 .rmap_one = try_to_migrate_one,
19771959 .arg = (void *)flags,
19781960 .done = page_not_mapped,
1979− .anon_lock = page_lock_anon_vma_read,
1961+ .anon_lock = folio_lock_anon_vma_read,
19801962 };
19811963
19821964 /*
@@ -1987,7 +1969,7 @@ void try_to_migrate(struct page *page, enum ttu_flags flags) · 4 unmodified lines
19871969 TTU_SYNC)))
19881970 return;
19891971
1990− if (is_zone_device_page(page) && !is_device_private_page(page))
1972+ if (folio_is_zone_device(folio) && !folio_is_device_private(folio))
19911973 return;
19921974
19931975 /*
@@ -1998,83 +1980,13 @@ void try_to_migrate(struct page *page, enum ttu_flags flags) · 4 unmodified lines
19981980 * locking requirements of exec(), migration skips
19991981 * temporary VMAs until after exec() completes.
20001982 */
2001− if (!PageKsm(page) && PageAnon(page))
1983+ if (!folio_test_ksm(folio) && folio_test_anon(folio))
20021984 rwc.invalid_vma = invalid_migration_vma;
20031985
20041986 if (flags & TTU_RMAP_LOCKED)
2005− rmap_walk_locked(page, &rwc);
1987+ rmap_walk_locked(folio, &rwc);
20061988 else
2007− rmap_walk(page, &rwc);
2008−}
2009−
2010−/*
2011− * Walks the vma's mapping a page and mlocks the page if any locked vma's are
2012− * found. Once one is found the page is locked and the scan can be terminated.
2013− */
2014−static bool page_mlock_one(struct page *page, struct vm_area_struct *vma,
2015− unsigned long address, void *unused)
2016−{
2017− struct page_vma_mapped_walk pvmw = {
2018− .page = page,
2019− .vma = vma,
2020− .address = address,
2021− };
2022−
2023− /* An un-locked vma doesn't have any pages to lock, continue the scan */
2024− if (!(vma->vm_flags & VM_LOCKED))
2025− return true;
2026−
2027− while (page_vma_mapped_walk(&pvmw)) {
2028− /*
2029− * Need to recheck under the ptl to serialise with
2030− * __munlock_pagevec_fill() after VM_LOCKED is cleared in
2031− * munlock_vma_pages_range().
2032− */
2033− if (vma->vm_flags & VM_LOCKED) {
2034− /*
2035− * PTE-mapped THP are never marked as mlocked; but
2036− * this function is never called on a DoubleMap THP,
2037− * nor on an Anon THP (which may still be PTE-mapped
2038− * after DoubleMap was cleared).
2039− */
2040− mlock_vma_page(page);
2041− /*
2042− * No need to scan further once the page is marked
2043− * as mlocked.
2044− */
2045− page_vma_mapped_walk_done(&pvmw);
2046− return false;
2047− }
2048− }
2049−
2050− return true;
2051−}
2052−
2053−/**
2054− * page_mlock - try to mlock a page
2055− * @page: the page to be mlocked
2056− *
2057− * Called from munlock code. Checks all of the VMAs mapping the page and mlocks
2058− * the page if any are found. The page will be returned with PG_mlocked cleared
2059− * if it is not mapped by any locked vmas.
2060− */
2061−void page_mlock(struct page *page)
2062−{
2063− struct rmap_walk_control rwc = {
2064− .rmap_one = page_mlock_one,
2065− .done = page_not_mapped,
2066− .anon_lock = page_lock_anon_vma_read,
2067−
2068− };
2069−
2070− VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page);
2071− VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page);
2072−
2073− /* Anon THP are only marked as mlocked when singly mapped */
2074− if (PageTransCompound(page) && PageAnon(page))
2075− return;
2076−
2077− rmap_walk(page, &rwc);
1989+ rmap_walk(folio, &rwc);
20781990 }
20791991
20801992 #ifdef CONFIG_DEVICE_PRIVATE
@@ -2085,15 +1997,11 @@ struct make_exclusive_args { · 4 unmodified lines
20851997 bool valid;
20861998 };
20871999
2088−static bool page_make_device_exclusive_one(struct page *page,
2000+static bool page_make_device_exclusive_one(struct folio *folio,
20892001 struct vm_area_struct *vma, unsigned long address, void *priv)
20902002 {
20912003 struct mm_struct *mm = vma->vm_mm;
2092− struct page_vma_mapped_walk pvmw = {
2093− .page = page,
2094− .vma = vma,
2095− .address = address,
2096− };
2004+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
20972005 struct make_exclusive_args *args = priv;
20982006 pte_t pteval;
20992007 struct page *subpage;
@@ -2104,12 +2012,13 @@ static bool page_make_device_exclusive_one(struct page *page, · 4 unmodified lines
21042012
21052013 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma,
21062014 vma->vm_mm, address, min(vma->vm_end,
2107− address + page_size(page)), args->owner);
2015+ address + folio_size(folio)),
2016+ args->owner);
21082017 mmu_notifier_invalidate_range_start(&range);
21092018
21102019 while (page_vma_mapped_walk(&pvmw)) {
21112020 /* Unexpected PMD-mapped THP? */
2112− VM_BUG_ON_PAGE(!pvmw.pte, page);
2021+ VM_BUG_ON_FOLIO(!pvmw.pte, folio);
21132022
21142023 if (!pte_present(*pvmw.pte)) {
21152024 ret = false;
@@ -2117,16 +2026,17 @@ static bool page_make_device_exclusive_one(struct page *page, · 1 unmodified line
21172026 break;
21182027 }
21192028
2120− subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
2029+ subpage = folio_page(folio,
2030+ pte_pfn(*pvmw.pte) - folio_pfn(folio));
21212031 address = pvmw.address;
21222032
21232033 /* Nuke the page table entry. */
21242034 flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
21252035 pteval = ptep_clear_flush(vma, address, pvmw.pte);
21262036
2127− /* Move the dirty bit to the page. Now the pte is gone. */
2037+ /* Set the dirty flag on the folio now the pte is gone. */
21282038 if (pte_dirty(pteval))
2129− set_page_dirty(page);
2039+ folio_mark_dirty(folio);
21302040
21312041 /*
21322042 * Check that our target page is still mapped at the expected
@@ -2159,7 +2069,7 @@ static bool page_make_device_exclusive_one(struct page *page, · 26 unmodified lines
21592069 * There is a reference on the page for the swap entry which has
21602070 * been removed, so shouldn't take another.
21612071 */
2162− page_remove_rmap(subpage, false);
2072+ page_remove_rmap(subpage, vma, false);
21632073 }
21642074
21652075 mmu_notifier_invalidate_range_end(&range);
@@ -2168,21 +2078,22 @@ static bool page_make_device_exclusive_one(struct page *page, · 2 unmodified lines
21682078 }
21692079
21702080 /**
2171− * page_make_device_exclusive - mark the page exclusively owned by a device
2172− * @page: the page to replace page table entries for
2173− * @mm: the mm_struct where the page is expected to be mapped
2174− * @address: address where the page is expected to be mapped
2081+ * folio_make_device_exclusive - Mark the folio exclusively owned by a device.
2082+ * @folio: The folio to replace page table entries for.
2083+ * @mm: The mm_struct where the folio is expected to be mapped.
2084+ * @address: Address where the folio is expected to be mapped.
21752085 * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks
21762086 *
2177− * Tries to remove all the page table entries which are mapping this page and
2178− * replace them with special device exclusive swap entries to grant a device
2179− * exclusive access to the page. Caller must hold the page lock.
2087+ * Tries to remove all the page table entries which are mapping this
2088+ * folio and replace them with special device exclusive swap entries to
2089+ * grant a device exclusive access to the folio.
21802090 *
2181− * Returns false if the page is still mapped, or if it could not be unmapped
2091+ * Context: Caller must hold the folio lock.
2092+ * Return: false if the page is still mapped, or if it could not be unmapped
21822093 * from the expected address. Otherwise returns true (success).
21832094 */
2184−static bool page_make_device_exclusive(struct page *page, struct mm_struct *mm,
2185− unsigned long address, void *owner)
2095+static bool folio_make_device_exclusive(struct folio *folio,
2096+ struct mm_struct *mm, unsigned long address, void *owner)
21862097 {
21872098 struct make_exclusive_args args = {
21882099 .mm = mm,
@@ -2193,21 +2104,20 @@ static bool page_make_device_exclusive(struct page *page, struct mm_struct *mm, · 4 unmodified lines
21932104 struct rmap_walk_control rwc = {
21942105 .rmap_one = page_make_device_exclusive_one,
21952106 .done = page_not_mapped,
2196− .anon_lock = page_lock_anon_vma_read,
2107+ .anon_lock = folio_lock_anon_vma_read,
21972108 .arg = &args,
21982109 };
21992110
22002111 /*
2201− * Restrict to anonymous pages for now to avoid potential writeback
2202− * issues. Also tail pages shouldn't be passed to rmap_walk so skip
2203− * those.
2112+ * Restrict to anonymous folios for now to avoid potential writeback
2113+ * issues.
22042114 */
2205− if (!PageAnon(page) || PageTail(page))
2115+ if (!folio_test_anon(folio))
22062116 return false;
22072117
2208− rmap_walk(page, &rwc);
2118+ rmap_walk(folio, &rwc);
22092119
2210− return args.valid && !page_mapcount(page);
2120+ return args.valid && !folio_mapcount(folio);
22112121 }
22122122
22132123 /**
@@ -2245,15 +2155,16 @@ int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, · 31 unmodified lines
22452155 return npages;
22462156
22472157 for (i = 0; i < npages; i++, start += PAGE_SIZE) {
2248− if (!trylock_page(pages[i])) {
2249− put_page(pages[i]);
2158+ struct folio *folio = page_folio(pages[i]);
2159+ if (PageTail(pages[i]) || !folio_trylock(folio)) {
2160+ folio_put(folio);
22502161 pages[i] = NULL;
22512162 continue;
22522163 }
22532164
2254− if (!page_make_device_exclusive(pages[i], mm, start, owner)) {
2255− unlock_page(pages[i]);
2256− put_page(pages[i]);
2165+ if (!folio_make_device_exclusive(folio, mm, start, owner)) {
2166+ folio_unlock(folio);
2167+ folio_put(folio);
22572168 pages[i] = NULL;
22582169 }
22592170 }
@@ -2272,21 +2183,21 @@ void __put_anon_vma(struct anon_vma *anon_vma) · 12 unmodified lines
22722183 anon_vma_free(root);
22732184 }
22742185
2275−static struct anon_vma *rmap_walk_anon_lock(struct page *page,
2276− struct rmap_walk_control *rwc)
2186+static struct anon_vma *rmap_walk_anon_lock(struct folio *folio,
2187+ const struct rmap_walk_control *rwc)
22772188 {
22782189 struct anon_vma *anon_vma;
22792190
22802191 if (rwc->anon_lock)
2281− return rwc->anon_lock(page);
2192+ return rwc->anon_lock(folio);
22822193
22832194 /*
2284− * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read()
2195+ * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
22852196 * because that depends on page_mapped(); but not all its usages
22862197 * are holding mmap_lock. Users without mmap_lock are required to
22872198 * take a reference count to prevent the anon_vma disappearing
22882199 */
2289− anon_vma = page_anon_vma(page);
2200+ anon_vma = folio_anon_vma(folio);
22902201 if (!anon_vma)
22912202 return NULL;
22922203
@@ -2302,35 +2213,30 @@ static struct anon_vma *rmap_walk_anon_lock(struct page *page, · 9 unmodified lines
23022213 *
23032214 * Find all the mappings of a page using the mapping pointer and the vma chains
23042215 * contained in the anon_vma struct it points to.
2305− *
2306− * When called from page_mlock(), the mmap_lock of the mm containing the vma
2307− * where the page was found will be held for write. So, we won't recheck
2308− * vm_flags for that VMA. That should be OK, because that vma shouldn't be
2309− * LOCKED.
23102216 */
2311−static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc,
2312− bool locked)
2217+static void rmap_walk_anon(struct folio *folio,
2218+ const struct rmap_walk_control *rwc, bool locked)
23132219 {
23142220 struct anon_vma *anon_vma;
23152221 pgoff_t pgoff_start, pgoff_end;
23162222 struct anon_vma_chain *avc;
23172223
23182224 if (locked) {
2319− anon_vma = page_anon_vma(page);
2225+ anon_vma = folio_anon_vma(folio);
23202226 /* anon_vma disappear under us? */
2321− VM_BUG_ON_PAGE(!anon_vma, page);
2227+ VM_BUG_ON_FOLIO(!anon_vma, folio);
23222228 } else {
2323− anon_vma = rmap_walk_anon_lock(page, rwc);
2229+ anon_vma = rmap_walk_anon_lock(folio, rwc);
23242230 }
23252231 if (!anon_vma)
23262232 return;
23272233
2328− pgoff_start = page_to_pgoff(page);
2329− pgoff_end = pgoff_start + thp_nr_pages(page) - 1;
2234+ pgoff_start = folio_pgoff(folio);
2235+ pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
23302236 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
23312237 pgoff_start, pgoff_end) {
23322238 struct vm_area_struct *vma = avc->vma;
2333− unsigned long address = vma_address(page, vma);
2239+ unsigned long address = vma_address(&folio->page, vma);
23342240
23352241 VM_BUG_ON_VMA(address == -EFAULT, vma);
23362242 cond_resched();
@@ -2338,9 +2244,9 @@ static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, · 1 unmodified line
23382244 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
23392245 continue;
23402246
2341− if (!rwc->rmap_one(page, vma, address, rwc->arg))
2247+ if (!rwc->rmap_one(folio, vma, address, rwc->arg))
23422248 break;
2343− if (rwc->done && rwc->done(page))
2249+ if (rwc->done && rwc->done(folio))
23442250 break;
23452251 }
23462252
@@ -2355,16 +2261,11 @@ static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, · 8 unmodified lines
23552261 *
23562262 * Find all the mappings of a page using the mapping pointer and the vma chains
23572263 * contained in the address_space struct it points to.
2358− *
2359− * When called from page_mlock(), the mmap_lock of the mm containing the vma
2360− * where the page was found will be held for write. So, we won't recheck
2361− * vm_flags for that VMA. That should be OK, because that vma shouldn't be
2362− * LOCKED.
23632264 */
2364−static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc,
2365− bool locked)
2265+static void rmap_walk_file(struct folio *folio,
2266+ const struct rmap_walk_control *rwc, bool locked)
23662267 {
2367− struct address_space *mapping = page_mapping(page);
2268+ struct address_space *mapping = folio_mapping(folio);
23682269 pgoff_t pgoff_start, pgoff_end;
23692270 struct vm_area_struct *vma;
23702271
@@ -2374,18 +2275,18 @@ static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, · 3 unmodified lines
23742275 * structure at mapping cannot be freed and reused yet,
23752276 * so we can safely take mapping->i_mmap_rwsem.
23762277 */
2377− VM_BUG_ON_PAGE(!PageLocked(page), page);
2278+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
23782279
23792280 if (!mapping)
23802281 return;
23812282
2382− pgoff_start = page_to_pgoff(page);
2383− pgoff_end = pgoff_start + thp_nr_pages(page) - 1;
2283+ pgoff_start = folio_pgoff(folio);
2284+ pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
23842285 if (!locked)
23852286 i_mmap_lock_read(mapping);
23862287 vma_interval_tree_foreach(vma, &mapping->i_mmap,
23872288 pgoff_start, pgoff_end) {
2388− unsigned long address = vma_address(page, vma);
2289+ unsigned long address = vma_address(&folio->page, vma);
23892290
23902291 VM_BUG_ON_VMA(address == -EFAULT, vma);
23912292 cond_resched();
@@ -2393,9 +2294,9 @@ static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, · 1 unmodified line
23932294 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
23942295 continue;
23952296
2396− if (!rwc->rmap_one(page, vma, address, rwc->arg))
2297+ if (!rwc->rmap_one(folio, vma, address, rwc->arg))
23972298 goto done;
2398− if (rwc->done && rwc->done(page))
2299+ if (rwc->done && rwc->done(folio))
23992300 goto done;
24002301 }
24012302
@@ -2404,25 +2305,25 @@ done: · 2 unmodified lines
24042305 i_mmap_unlock_read(mapping);
24052306 }
24062307
2407−void rmap_walk(struct page *page, struct rmap_walk_control *rwc)
2308+void rmap_walk(struct folio *folio, const struct rmap_walk_control *rwc)
24082309 {
2409− if (unlikely(PageKsm(page)))
2410− rmap_walk_ksm(page, rwc);
2411− else if (PageAnon(page))
2412− rmap_walk_anon(page, rwc, false);
2310+ if (unlikely(folio_test_ksm(folio)))
2311+ rmap_walk_ksm(folio, rwc);
2312+ else if (folio_test_anon(folio))
2313+ rmap_walk_anon(folio, rwc, false);
24132314 else
2414− rmap_walk_file(page, rwc, false);
2315+ rmap_walk_file(folio, rwc, false);
24152316 }
24162317
24172318 /* Like rmap_walk, but caller holds relevant rmap lock */
2418−void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc)
2319+void rmap_walk_locked(struct folio *folio, const struct rmap_walk_control *rwc)
24192320 {
24202321 /* no ksm support for now */
2421− VM_BUG_ON_PAGE(PageKsm(page), page);
2422− if (PageAnon(page))
2423− rmap_walk_anon(page, rwc, true);
2322+ VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
2323+ if (folio_test_anon(folio))
2324+ rmap_walk_anon(folio, rwc, true);
24242325 else
2425− rmap_walk_file(page, rwc, true);
2326+ rmap_walk_file(folio, rwc, true);
24262327 }
24272328
24282329 #ifdef CONFIG_HUGETLB_PAGE
@@ -2450,8 +2351,7 @@ void hugepage_add_new_anon_rmap(struct page *page, · 21 unmodified lines
24502351 {
24512352 BUG_ON(address < vma->vm_start || address >= vma->vm_end);
24522353 atomic_set(compound_mapcount_ptr(page), 0);
2453− if (hpage_pincount_available(page))
2454− atomic_set(compound_pincount_ptr(page), 0);
2354+ atomic_set(compound_pincount_ptr(page), 0);
24552355
24562356 __page_set_anon_rmap(page, vma, address, 1);
24572357 }
mm/swap.c+68−105
@@ -74,8 +74,8 @@ static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = { · 73 unmodified lines
7474 };
7575
7676 /*
77− * This path almost never happens for VM activity - pages are normally
78− * freed via pagevecs. But it gets used by networking.
77+ * This path almost never happens for VM activity - pages are normally freed
78+ * via pagevecs. But it gets used by networking - and for compound pages.
7979 */
8080 static void __page_cache_release(struct page *page)
8181 {
@@ -89,6 +89,14 @@ static void __page_cache_release(struct page *page) · 7 unmodified lines
8989 __clear_page_lru_flags(page);
9090 unlock_page_lruvec_irqrestore(lruvec, flags);
9191 }
92+ /* See comment on PageMlocked in release_pages() */
93+ if (unlikely(PageMlocked(page))) {
94+ int nr_pages = thp_nr_pages(page);
95+
96+ __ClearPageMlocked(page);
97+ mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
98+ count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages);
99+ }
92100 __ClearPageWaiters(page);
93101 }
94102
@@ -114,17 +122,9 @@ static void __put_compound_page(struct page *page) · 19 unmodified lines
114122
115123 void __put_page(struct page *page)
116124 {
117− if (is_zone_device_page(page)) {
118− put_dev_pagemap(page->pgmap);
119−
120− /*
121− * The page belongs to the device that created pgmap. Do
122− * not return it to page allocator.
123− */
124− return;
125− }
126−
127− if (unlikely(PageCompound(page)))
125+ if (unlikely(is_zone_device_page(page)))
126+ free_zone_device_page(page);
127+ else if (unlikely(PageCompound(page)))
128128 __put_compound_page(page);
129129 else
130130 __put_single_page(page);
@@ -482,22 +482,12 @@ EXPORT_SYMBOL(folio_add_lru); · 351 unmodified lines
482482 void lru_cache_add_inactive_or_unevictable(struct page *page,
483483 struct vm_area_struct *vma)
484484 {
485− bool unevictable;
486−
487485 VM_BUG_ON_PAGE(PageLRU(page), page);
488486
489− unevictable = (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED;
490− if (unlikely(unevictable) && !TestSetPageMlocked(page)) {
491− int nr_pages = thp_nr_pages(page);
492− /*
493− * We use the irq-unsafe __mod_zone_page_state because this
494− * counter is not modified from interrupt context, and the pte
495− * lock is held(spinlock), which implies preemption disabled.
496− */
497− __mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages);
498− count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
499− }
500− lru_cache_add(page);
487+ if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED))
488+ mlock_new_page(page);
489+ else
490+ lru_cache_add(page);
501491 }
502492
503493 /*
@@ -636,35 +626,37 @@ void lru_add_drain_cpu(int cpu) · 132 unmodified lines
636626 pagevec_lru_move_fn(pvec, lru_lazyfree_fn);
637627
638628 activate_page_drain(cpu);
629+ mlock_page_drain(cpu);
639630 }
640631
641632 /**
642− * deactivate_file_page - forcefully deactivate a file page
643− * @page: page to deactivate
633+ * deactivate_file_folio() - Forcefully deactivate a file folio.
634+ * @folio: Folio to deactivate.
644635 *
645− * This function hints the VM that @page is a good reclaim candidate,
646− * for example if its invalidation fails due to the page being dirty
636+ * This function hints to the VM that @folio is a good reclaim candidate,
637+ * for example if its invalidation fails due to the folio being dirty
647638 * or under writeback.
639+ *
640+ * Context: Caller holds a reference on the page.
648641 */
649−void deactivate_file_page(struct page *page)
642+void deactivate_file_folio(struct folio *folio)
650643 {
644+ struct pagevec *pvec;
645+
651646 /*
652− * In a workload with many unevictable page such as mprotect,
653− * unevictable page deactivation for accelerating reclaim is pointless.
647+ * In a workload with many unevictable pages such as mprotect,
648+ * unevictable folio deactivation for accelerating reclaim is pointless.
654649 */
655− if (PageUnevictable(page))
650+ if (folio_test_unevictable(folio))
656651 return;
657652
658− if (likely(get_page_unless_zero(page))) {
659− struct pagevec *pvec;
660−
661− local_lock(&lru_pvecs.lock);
662− pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file);
653+ folio_get(folio);
654+ local_lock(&lru_pvecs.lock);
655+ pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file);
663656
664− if (pagevec_add_and_need_flush(pvec, page))
665− pagevec_lru_move_fn(pvec, lru_deactivate_file_fn);
666− local_unlock(&lru_pvecs.lock);
667− }
657+ if (pagevec_add_and_need_flush(pvec, &folio->page))
658+ pagevec_lru_move_fn(pvec, lru_deactivate_file_fn);
659+ local_unlock(&lru_pvecs.lock);
668660 }
669661
670662 /*
@@ -837,6 +829,7 @@ inline void __lru_add_drain_all(bool force_all_cpus) · 166 unmodified lines
837829 pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) ||
838830 pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) ||
839831 need_activate_page_drain(cpu) ||
832+ need_mlock_page_drain(cpu) ||
840833 has_bh_in_lru(cpu, NULL)) {
841834 INIT_WORK(work, lru_add_drain_per_cpu);
842835 queue_work_on(cpu, mm_percpu_wq, work);
@@ -935,18 +928,10 @@ void release_pages(struct page **pages, int nr) · 92 unmodified lines
935928 unlock_page_lruvec_irqrestore(lruvec, flags);
936929 lruvec = NULL;
937930 }
938− /*
939− * ZONE_DEVICE pages that return 'false' from
940− * page_is_devmap_managed() do not require special
941− * processing, and instead, expect a call to
942− * put_page_testzero().
943− */
944− if (page_is_devmap_managed(page)) {
945− put_devmap_managed_page(page);
931+ if (put_devmap_managed_page(page))
946932 continue;
947− }
948933 if (put_page_testzero(page))
949− put_dev_pagemap(page->pgmap);
934+ free_zone_device_page(page);
950935 continue;
951936 }
952937
@@ -974,6 +959,18 @@ void release_pages(struct page **pages, int nr) · 21 unmodified lines
974959 __clear_page_lru_flags(page);
975960 }
976961
962+ /*
963+ * In rare cases, when truncation or holepunching raced with
964+ * munlock after VM_LOCKED was cleared, Mlocked may still be
965+ * found set here. This does not indicate a problem, unless
966+ * "unevictable_pgs_cleared" appears worryingly large.
967+ */
968+ if (unlikely(PageMlocked(page))) {
969+ __ClearPageMlocked(page);
970+ dec_zone_page_state(page, NR_MLOCK);
971+ count_vm_event(UNEVICTABLE_PGCLEARED);
972+ }
973+
977974 __ClearPageWaiters(page);
978975
979976 list_add(&page->lru, &pages_to_free);
@@ -1014,43 +1011,32 @@ static void __pagevec_lru_add_fn(struct folio *folio, struct lruvec *lruvec) · 34 unmodified lines
10141011
10151012 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
10161013
1014+ folio_set_lru(folio);
10171015 /*
1018− * A folio becomes evictable in two ways:
1019− * 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()].
1020− * 2) Before acquiring LRU lock to put the folio on the correct LRU
1021− * and then
1022− * a) do PageLRU check with lock [check_move_unevictable_pages]
1023− * b) do PageLRU check before lock [clear_page_mlock]
1024− *
1025− * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need
1026− * following strict ordering:
1027− *
1028− * #0: __pagevec_lru_add_fn #1: clear_page_mlock
1016+ * Is an smp_mb__after_atomic() still required here, before
1017+ * folio_evictable() tests PageMlocked, to rule out the possibility
1018+ * of stranding an evictable folio on an unevictable LRU? I think
1019+ * not, because __munlock_page() only clears PageMlocked while the LRU
1020+ * lock is held.
10291021 *
1030− * folio_set_lru() folio_test_clear_mlocked()
1031− * smp_mb() // explicit ordering // above provides strict
1032− * // ordering
1033− * folio_test_mlocked() folio_test_lru()
1034− *
1035− *
1036− * if '#1' does not observe setting of PG_lru by '#0' and
1037− * fails isolation, the explicit barrier will make sure that
1038− * folio_evictable check will put the folio on the correct
1039− * LRU. Without smp_mb(), folio_set_lru() can be reordered
1040− * after folio_test_mlocked() check and can make '#1' fail the
1041− * isolation of the folio whose mlocked bit is cleared (#0 is
1042− * also looking at the same folio) and the evictable folio will
1043− * be stranded on an unevictable LRU.
1022+ * (That is not true of __page_cache_release(), and not necessarily
1023+ * true of release_pages(): but those only clear PageMlocked after
1024+ * put_page_testzero() has excluded any other users of the page.)
10441025 */
1045− folio_set_lru(folio);
1046− smp_mb__after_atomic();
1047−
10481026 if (folio_evictable(folio)) {
10491027 if (was_unevictable)
10501028 __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
10511029 } else {
10521030 folio_clear_active(folio);
10531031 folio_set_unevictable(folio);
1032+ /*
1033+ * folio->mlock_count = !!folio_test_mlocked(folio)?
1034+ * But that leaves __mlock_page() in doubt whether another
1035+ * actor has already counted the mlock or not. Err on the
1036+ * safe side, underestimate, let page reclaim fix it, rather
1037+ * than leaving a page on the unevictable LRU indefinitely.
1038+ */
1039+ folio->mlock_count = 0;
10541040 if (!was_unevictable)
10551041 __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);
10561042 }
@@ -1158,26 +1144,3 @@ void __init swap_setup(void) · 101 unmodified lines
11581144 * _really_ don't want to cluster much more
11591145 */
11601146 }
1161−
1162−#ifdef CONFIG_DEV_PAGEMAP_OPS
1163−void put_devmap_managed_page(struct page *page)
1164−{
1165− int count;
1166−
1167− if (WARN_ON_ONCE(!page_is_devmap_managed(page)))
1168− return;
1169−
1170− count = page_ref_dec_return(page);
1171−
1172− /*
1173− * devmap page refcounts are 1-based, rather than 0-based: if
1174− * refcount is 1, then the page is free and the refcount is
1175− * stable because nobody holds a reference on the page.
1176− */
1177− if (count == 1)
1178− free_devmap_managed_page(page);
1179− else if (!count)
1180− __put_page(page);
1181−}
1182−EXPORT_SYMBOL(put_devmap_managed_page);
1183−#endif
mm/truncate.c+51−60
@@ -193,27 +193,6 @@ static void truncate_cleanup_folio(struct folio *folio) · 192 unmodified lines
193193 folio_clear_mappedtodisk(folio);
194194 }
195195
196−/*
197− * This is for invalidate_mapping_pages(). That function can be called at
198− * any time, and is not supposed to throw away dirty pages. But pages can
199− * be marked dirty at any time too, so use remove_mapping which safely
200− * discards clean, unused pages.
201− *
202− * Returns non-zero if the page was successfully invalidated.
203− */
204−static int
205−invalidate_complete_page(struct address_space *mapping, struct page *page)
206−{
207−
208− if (page->mapping != mapping)
209− return 0;
210−
211− if (page_has_private(page) && !try_to_release_page(page, 0))
212− return 0;
213−
214− return remove_mapping(mapping, page);
215−}
216−
217196 int truncate_inode_folio(struct address_space *mapping, struct folio *folio)
218197 {
219198 if (folio->mapping != mapping)
@@ -294,22 +273,40 @@ int generic_error_remove_page(struct address_space *mapping, struct page *page) · 74 unmodified lines
294273 }
295274 EXPORT_SYMBOL(generic_error_remove_page);
296275
297−/*
276+static long mapping_evict_folio(struct address_space *mapping,
277+ struct folio *folio)
278+{
279+ if (folio_test_dirty(folio) || folio_test_writeback(folio))
280+ return 0;
281+ /* The refcount will be elevated if any page in the folio is mapped */
282+ if (folio_ref_count(folio) >
283+ folio_nr_pages(folio) + folio_has_private(folio) + 1)
284+ return 0;
285+ if (folio_has_private(folio) && !filemap_release_folio(folio, 0))
286+ return 0;
287+
288+ return remove_mapping(mapping, folio);
289+}
290+
291+/**
292+ * invalidate_inode_page() - Remove an unused page from the pagecache.
293+ * @page: The page to remove.
294+ *
298295 * Safely invalidate one page from its pagecache mapping.
299− * It only drops clean, unused pages. The page must be locked.
296+ * It only drops clean, unused pages.
300297 *
301− * Returns 1 if the page is successfully invalidated, otherwise 0.
298+ * Context: Page must be locked.
299+ * Return: The number of pages successfully removed.
302300 */
303−int invalidate_inode_page(struct page *page)
301+long invalidate_inode_page(struct page *page)
304302 {
305− struct address_space *mapping = page_mapping(page);
303+ struct folio *folio = page_folio(page);
304+ struct address_space *mapping = folio_mapping(folio);
305+
306+ /* The page may have been truncated before it was locked */
306307 if (!mapping)
307308 return 0;
308− if (PageDirty(page) || PageWriteback(page))
309− return 0;
310− if (page_mapped(page))
311− return 0;
312− return invalidate_complete_page(mapping, page);
309+ return mapping_evict_folio(mapping, folio);
313310 }
314311
315312 /**
@@ -497,7 +494,18 @@ void truncate_inode_pages_final(struct address_space *mapping) · 181 unmodified lines
497494 }
498495 EXPORT_SYMBOL(truncate_inode_pages_final);
499496
500−static unsigned long __invalidate_mapping_pages(struct address_space *mapping,
497+/**
498+ * invalidate_mapping_pagevec - Invalidate all the unlocked pages of one inode
499+ * @mapping: the address_space which holds the pages to invalidate
500+ * @start: the offset 'from' which to invalidate
501+ * @end: the offset 'to' which to invalidate (inclusive)
502+ * @nr_pagevec: invalidate failed page number for caller
503+ *
504+ * This helper is similar to invalidate_mapping_pages(), except that it accounts
505+ * for pages that are likely on a pagevec and counts them in @nr_pagevec, which
506+ * will be used by the caller.
507+ */
508+unsigned long invalidate_mapping_pagevec(struct address_space *mapping,
501509 pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
502510 {
503511 pgoff_t indices[PAGEVEC_SIZE];
@@ -510,27 +518,27 @@ static unsigned long __invalidate_mapping_pages(struct address_space *mapping, · 6 unmodified lines
510518 folio_batch_init(&fbatch);
511519 while (find_lock_entries(mapping, index, end, &fbatch, indices)) {
512520 for (i = 0; i < folio_batch_count(&fbatch); i++) {
513− struct page *page = &fbatch.folios[i]->page;
521+ struct folio *folio = fbatch.folios[i];
514522
515− /* We rely upon deletion not changing page->index */
523+ /* We rely upon deletion not changing folio->index */
516524 index = indices[i];
517525
518− if (xa_is_value(page)) {
526+ if (xa_is_value(folio)) {
519527 count += invalidate_exceptional_entry(mapping,
520528 index,
521− page);
529+ folio);
522530 continue;
523531 }
524− index += thp_nr_pages(page) - 1;
532+ index += folio_nr_pages(folio) - 1;
525533
526− ret = invalidate_inode_page(page);
527− unlock_page(page);
534+ ret = mapping_evict_folio(mapping, folio);
535+ folio_unlock(folio);
528536 /*
529− * Invalidation is a hint that the page is no longer
537+ * Invalidation is a hint that the folio is no longer
530538 * of interest and try to speed up its reclaim.
531539 */
532540 if (!ret) {
533− deactivate_file_page(page);
541+ deactivate_file_folio(folio);
534542 /* It is likely on the pagevec of a remote CPU */
535543 if (nr_pagevec)
536544 (*nr_pagevec)++;
@@ -562,29 +570,12 @@ static unsigned long __invalidate_mapping_pages(struct address_space *mapping, · 25 unmodified lines
562570 unsigned long invalidate_mapping_pages(struct address_space *mapping,
563571 pgoff_t start, pgoff_t end)
564572 {
565− return __invalidate_mapping_pages(mapping, start, end, NULL);
573+ return invalidate_mapping_pagevec(mapping, start, end, NULL);
566574 }
567575 EXPORT_SYMBOL(invalidate_mapping_pages);
568576
569−/**
570− * invalidate_mapping_pagevec - Invalidate all the unlocked pages of one inode
571− * @mapping: the address_space which holds the pages to invalidate
572− * @start: the offset 'from' which to invalidate
573− * @end: the offset 'to' which to invalidate (inclusive)
574− * @nr_pagevec: invalidate failed page number for caller
575− *
576− * This helper is similar to invalidate_mapping_pages(), except that it accounts
577− * for pages that are likely on a pagevec and counts them in @nr_pagevec, which
578− * will be used by the caller.
579− */
580−void invalidate_mapping_pagevec(struct address_space *mapping,
581− pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
582−{
583− __invalidate_mapping_pages(mapping, start, end, nr_pagevec);
584−}
585−
586577 /*
587− * This is like invalidate_complete_page(), except it ignores the page's
578+ * This is like invalidate_inode_page(), except it ignores the page's
588579 * refcount. We do this because invalidate_inode_pages2() needs stronger
589580 * invalidation guarantees, and cannot afford to leave pages behind because
590581 * shrink_page_list() has a temp ref on them, or because they're transiently
mm/userfaultfd.c+8−6
@@ -95,10 +95,15 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · 94 unmodified lines
9595 if (!pte_none(*dst_pte))
9696 goto out_unlock;
9797
98− if (page_in_cache)
99− page_add_file_rmap(page, false);
100− else
98+ if (page_in_cache) {
99+ /* Usually, cache pages are already added to LRU */
100+ if (newly_allocated)
101+ lru_cache_add(page);
102+ page_add_file_rmap(page, dst_vma, false);
103+ } else {
101104 page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
105+ lru_cache_add_inactive_or_unevictable(page, dst_vma);
106+ }
102107
103108 /*
104109 * Must happen after rmap, as mm_counter() checks mapping (via
@@ -106,9 +111,6 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · 1 unmodified line
106111 */
107112 inc_mm_counter(dst_mm, mm_counter(page));
108113
109− if (newly_allocated)
110− lru_cache_add_inactive_or_unevictable(page, dst_vma);
111−
112114 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
113115
114116 /* No need to invalidate - it was non-present before */
mm/util.c+34−2
@@ -681,9 +681,8 @@ bool folio_mapped(struct folio *folio) · 680 unmodified lines
681681 }
682682 EXPORT_SYMBOL(folio_mapped);
683683
684−struct anon_vma *page_anon_vma(struct page *page)
684+struct anon_vma *folio_anon_vma(struct folio *folio)
685685 {
686− struct folio *folio = page_folio(page);
687686 unsigned long mapping = (unsigned long)folio->mapping;
688687
689688 if ((mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
@@ -742,6 +741,39 @@ int __page_mapcount(struct page *page) · 52 unmodified lines
742741 }
743742 EXPORT_SYMBOL_GPL(__page_mapcount);
744743
744+/**
745+ * folio_mapcount() - Calculate the number of mappings of this folio.
746+ * @folio: The folio.
747+ *
748+ * A large folio tracks both how many times the entire folio is mapped,
749+ * and how many times each individual page in the folio is mapped.
750+ * This function calculates the total number of times the folio is
751+ * mapped.
752+ *
753+ * Return: The number of times this folio is mapped.
754+ */
755+int folio_mapcount(struct folio *folio)
756+{
757+ int i, compound, nr, ret;
758+
759+ if (likely(!folio_test_large(folio)))
760+ return atomic_read(&folio->_mapcount) + 1;
761+
762+ compound = folio_entire_mapcount(folio);
763+ nr = folio_nr_pages(folio);
764+ if (folio_test_hugetlb(folio))
765+ return compound;
766+ ret = compound;
767+ for (i = 0; i < nr; i++)
768+ ret += atomic_read(&folio_page(folio, i)->_mapcount) + 1;
769+ /* File pages has compound_mapcount included in _mapcount */
770+ if (!folio_test_anon(folio))
771+ return ret - compound * nr;
772+ if (folio_test_double_map(folio))
773+ ret -= nr;
774+ return ret;
775+}
776+
745777 /**
746778 * folio_copy - Copy the contents of one folio to another.
747779 * @dst: Folio to copy to.
mm/vmscan.c+156−149
@@ -979,36 +979,36 @@ void drop_slab(void) · 978 unmodified lines
979979 drop_slab_node(nid);
980980 }
981981
982−static inline int is_page_cache_freeable(struct page *page)
982+static inline int is_page_cache_freeable(struct folio *folio)
983983 {
984984 /*
985985 * A freeable page cache page is referenced only by the caller
986986 * that isolated the page, the page cache and optional buffer
987987 * heads at page->private.
988988 */
989− int page_cache_pins = thp_nr_pages(page);
990− return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
989+ return folio_ref_count(folio) - folio_test_private(folio) ==
990+ 1 + folio_nr_pages(folio);
991991 }
992992
993993 /*
994− * We detected a synchronous write error writing a page out. Probably
994+ * We detected a synchronous write error writing a folio out. Probably
995995 * -ENOSPC. We need to propagate that into the address_space for a subsequent
996996 * fsync(), msync() or close().
997997 *
998998 * The tricky part is that after writepage we cannot touch the mapping: nothing
999− * prevents it from being freed up. But we have a ref on the page and once
1000− * that page is locked, the mapping is pinned.
999+ * prevents it from being freed up. But we have a ref on the folio and once
1000+ * that folio is locked, the mapping is pinned.
10011001 *
1002− * We're allowed to run sleeping lock_page() here because we know the caller has
1002+ * We're allowed to run sleeping folio_lock() here because we know the caller has
10031003 * __GFP_FS.
10041004 */
10051005 static void handle_write_error(struct address_space *mapping,
1006− struct page *page, int error)
1006+ struct folio *folio, int error)
10071007 {
1008− lock_page(page);
1009− if (page_mapping(page) == mapping)
1008+ folio_lock(folio);
1009+ if (folio_mapping(folio) == mapping)
10101010 mapping_set_error(mapping, error);
1011− unlock_page(page);
1011+ folio_unlock(folio);
10121012 }
10131013
10141014 static bool skip_throttle_noprogress(pg_data_t *pgdat)
@@ -1155,35 +1155,35 @@ typedef enum { · 140 unmodified lines
11551155 * pageout is called by shrink_page_list() for each dirty page.
11561156 * Calls ->writepage().
11571157 */
1158−static pageout_t pageout(struct page *page, struct address_space *mapping)
1158+static pageout_t pageout(struct folio *folio, struct address_space *mapping)
11591159 {
11601160 /*
1161− * If the page is dirty, only perform writeback if that write
1161+ * If the folio is dirty, only perform writeback if that write
11621162 * will be non-blocking. To prevent this allocation from being
11631163 * stalled by pagecache activity. But note that there may be
11641164 * stalls if we need to run get_block(). We could test
11651165 * PagePrivate for that.
11661166 *
11671167 * If this process is currently in __generic_file_write_iter() against
1168− * this page's queue, we can perform writeback even if that
1168+ * this folio's queue, we can perform writeback even if that
11691169 * will block.
11701170 *
1171− * If the page is swapcache, write it back even if that would
1171+ * If the folio is swapcache, write it back even if that would
11721172 * block, for some throttling. This happens by accident, because
11731173 * swap_backing_dev_info is bust: it doesn't reflect the
11741174 * congestion state of the swapdevs. Easy to fix, if needed.
11751175 */
1176− if (!is_page_cache_freeable(page))
1176+ if (!is_page_cache_freeable(folio))
11771177 return PAGE_KEEP;
11781178 if (!mapping) {
11791179 /*
1180− * Some data journaling orphaned pages can have
1181− * page->mapping == NULL while being dirty with clean buffers.
1180+ * Some data journaling orphaned folios can have
1181+ * folio->mapping == NULL while being dirty with clean buffers.
11821182 */
1183− if (page_has_private(page)) {
1184− if (try_to_free_buffers(page)) {
1185− ClearPageDirty(page);
1186− pr_info("%s: orphaned page\n", __func__);
1183+ if (folio_test_private(folio)) {
1184+ if (try_to_free_buffers(&folio->page)) {
1185+ folio_clear_dirty(folio);
1186+ pr_info("%s: orphaned folio\n", __func__);
11871187 return PAGE_CLEAN;
11881188 }
11891189 }
@@ -1192,7 +1192,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping) · 2 unmodified lines
11921192 if (mapping->a_ops->writepage == NULL)
11931193 return PAGE_ACTIVATE;
11941194
1195− if (clear_page_dirty_for_io(page)) {
1195+ if (folio_clear_dirty_for_io(folio)) {
11961196 int res;
11971197 struct writeback_control wbc = {
11981198 .sync_mode = WB_SYNC_NONE,
@@ -1202,21 +1202,21 @@ static pageout_t pageout(struct page *page, struct address_space *mapping) · 3 unmodified lines
12021202 .for_reclaim = 1,
12031203 };
12041204
1205− SetPageReclaim(page);
1206− res = mapping->a_ops->writepage(page, &wbc);
1205+ folio_set_reclaim(folio);
1206+ res = mapping->a_ops->writepage(&folio->page, &wbc);
12071207 if (res < 0)
1208− handle_write_error(mapping, page, res);
1208+ handle_write_error(mapping, folio, res);
12091209 if (res == AOP_WRITEPAGE_ACTIVATE) {
1210− ClearPageReclaim(page);
1210+ folio_clear_reclaim(folio);
12111211 return PAGE_ACTIVATE;
12121212 }
12131213
1214− if (!PageWriteback(page)) {
1214+ if (!folio_test_writeback(folio)) {
12151215 /* synchronous write or broken a_ops? */
1216− ClearPageReclaim(page);
1216+ folio_clear_reclaim(folio);
12171217 }
1218− trace_mm_vmscan_writepage(page);
1219− inc_node_page_state(page, NR_VMSCAN_WRITE);
1218+ trace_mm_vmscan_write_folio(folio);
1219+ node_stat_add_folio(folio, NR_VMSCAN_WRITE);
12201220 return PAGE_SUCCESS;
12211221 }
12221222
@@ -1227,16 +1227,16 @@ static pageout_t pageout(struct page *page, struct address_space *mapping) · 4 unmodified lines
12271227 * Same as remove_mapping, but if the page is removed from the mapping, it
12281228 * gets returned with a refcount of 0.
12291229 */
1230−static int __remove_mapping(struct address_space *mapping, struct page *page,
1230+static int __remove_mapping(struct address_space *mapping, struct folio *folio,
12311231 bool reclaimed, struct mem_cgroup *target_memcg)
12321232 {
12331233 int refcount;
12341234 void *shadow = NULL;
12351235
1236− BUG_ON(!PageLocked(page));
1237− BUG_ON(mapping != page_mapping(page));
1236+ BUG_ON(!folio_test_locked(folio));
1237+ BUG_ON(mapping != folio_mapping(folio));
12381238
1239− if (!PageSwapCache(page))
1239+ if (!folio_test_swapcache(folio))
12401240 spin_lock(&mapping->host->i_lock);
12411241 xa_lock_irq(&mapping->i_pages);
12421242 /*
@@ -1264,23 +1264,23 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, · 21 unmodified lines
12641264 * Note that if SetPageDirty is always performed via set_page_dirty,
12651265 * and thus under the i_pages lock, then this ordering is not required.
12661266 */
1267− refcount = 1 + compound_nr(page);
1268− if (!page_ref_freeze(page, refcount))
1267+ refcount = 1 + folio_nr_pages(folio);
1268+ if (!folio_ref_freeze(folio, refcount))
12691269 goto cannot_free;
12701270 /* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
1271− if (unlikely(PageDirty(page))) {
1272− page_ref_unfreeze(page, refcount);
1271+ if (unlikely(folio_test_dirty(folio))) {
1272+ folio_ref_unfreeze(folio, refcount);
12731273 goto cannot_free;
12741274 }
12751275
1276− if (PageSwapCache(page)) {
1277− swp_entry_t swap = { .val = page_private(page) };
1278− mem_cgroup_swapout(page, swap);
1276+ if (folio_test_swapcache(folio)) {
1277+ swp_entry_t swap = folio_swap_entry(folio);
1278+ mem_cgroup_swapout(folio, swap);
12791279 if (reclaimed && !mapping_exiting(mapping))
1280− shadow = workingset_eviction(page, target_memcg);
1281− __delete_from_swap_cache(page, swap, shadow);
1280+ shadow = workingset_eviction(folio, target_memcg);
1281+ __delete_from_swap_cache(&folio->page, swap, shadow);
12821282 xa_unlock_irq(&mapping->i_pages);
1283− put_swap_page(page, swap);
1283+ put_swap_page(&folio->page, swap);
12841284 } else {
12851285 void (*freepage)(struct page *);
12861286
@@ -1301,61 +1301,67 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, · 14 unmodified lines
13011301 * exceptional entries and shadow exceptional entries in the
13021302 * same address_space.
13031303 */
1304− if (reclaimed && page_is_file_lru(page) &&
1304+ if (reclaimed && folio_is_file_lru(folio) &&
13051305 !mapping_exiting(mapping) && !dax_mapping(mapping))
1306− shadow = workingset_eviction(page, target_memcg);
1307− __delete_from_page_cache(page, shadow);
1306+ shadow = workingset_eviction(folio, target_memcg);
1307+ __filemap_remove_folio(folio, shadow);
13081308 xa_unlock_irq(&mapping->i_pages);
13091309 if (mapping_shrinkable(mapping))
13101310 inode_add_lru(mapping->host);
13111311 spin_unlock(&mapping->host->i_lock);
13121312
13131313 if (freepage != NULL)
1314− freepage(page);
1314+ freepage(&folio->page);
13151315 }
13161316
13171317 return 1;
13181318
13191319 cannot_free:
13201320 xa_unlock_irq(&mapping->i_pages);
1321− if (!PageSwapCache(page))
1321+ if (!folio_test_swapcache(folio))
13221322 spin_unlock(&mapping->host->i_lock);
13231323 return 0;
13241324 }
13251325
1326−/*
1327− * Attempt to detach a locked page from its ->mapping. If it is dirty or if
1328− * someone else has a ref on the page, abort and return 0. If it was
1329− * successfully detached, return 1. Assumes the caller has a single ref on
1330− * this page.
1326+/**
1327+ * remove_mapping() - Attempt to remove a folio from its mapping.
1328+ * @mapping: The address space.
1329+ * @folio: The folio to remove.
1330+ *
1331+ * If the folio is dirty, under writeback or if someone else has a ref
1332+ * on it, removal will fail.
1333+ * Return: The number of pages removed from the mapping. 0 if the folio
1334+ * could not be removed.
1335+ * Context: The caller should have a single refcount on the folio and
1336+ * hold its lock.
13311337 */
1332−int remove_mapping(struct address_space *mapping, struct page *page)
1338+long remove_mapping(struct address_space *mapping, struct folio *folio)
13331339 {
1334− if (__remove_mapping(mapping, page, false, NULL)) {
1340+ if (__remove_mapping(mapping, folio, false, NULL)) {
13351341 /*
1336− * Unfreezing the refcount with 1 rather than 2 effectively
1342+ * Unfreezing the refcount with 1 effectively
13371343 * drops the pagecache ref for us without requiring another
13381344 * atomic operation.
13391345 */
1340− page_ref_unfreeze(page, 1);
1341− return 1;
1346+ folio_ref_unfreeze(folio, 1);
1347+ return folio_nr_pages(folio);
13421348 }
13431349 return 0;
13441350 }
13451351
13461352 /**
1347− * putback_lru_page - put previously isolated page onto appropriate LRU list
1348− * @page: page to be put back to appropriate lru list
1353+ * folio_putback_lru - Put previously isolated folio onto appropriate LRU list.
1354+ * @folio: Folio to be returned to an LRU list.
13491355 *
1350− * Add previously isolated @page to appropriate LRU list.
1351− * Page may still be unevictable for other reasons.
1356+ * Add previously isolated @folio to appropriate LRU list.
1357+ * The folio may still be unevictable for other reasons.
13521358 *
1353− * lru_lock must not be held, interrupts must be enabled.
1359+ * Context: lru_lock must not be held, interrupts must be enabled.
13541360 */
1355−void putback_lru_page(struct page *page)
1361+void folio_putback_lru(struct folio *folio)
13561362 {
1357− lru_cache_add(page);
1358− put_page(page); /* drop ref from isolate */
1363+ folio_add_lru(folio);
1364+ folio_put(folio); /* drop ref from isolate */
13591365 }
13601366
13611367 enum page_references {
@@ -1365,61 +1371,61 @@ enum page_references { · 3 unmodified lines
13651371 PAGEREF_ACTIVATE,
13661372 };
13671373
1368−static enum page_references page_check_references(struct page *page,
1374+static enum page_references folio_check_references(struct folio *folio,
13691375 struct scan_control *sc)
13701376 {
1371− int referenced_ptes, referenced_page;
1377+ int referenced_ptes, referenced_folio;
13721378 unsigned long vm_flags;
13731379
1374− referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
1375− &vm_flags);
1376− referenced_page = TestClearPageReferenced(page);
1380+ referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,
1381+ &vm_flags);
1382+ referenced_folio = folio_test_clear_referenced(folio);
13771383
13781384 /*
1379− * Mlock lost the isolation race with us. Let try_to_unmap()
1380− * move the page to the unevictable list.
1385+ * The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
1386+ * Let the folio, now marked Mlocked, be moved to the unevictable list.
13811387 */
13821388 if (vm_flags & VM_LOCKED)
1383− return PAGEREF_RECLAIM;
1389+ return PAGEREF_ACTIVATE;
13841390
13851391 if (referenced_ptes) {
13861392 /*
1387− * All mapped pages start out with page table
1393+ * All mapped folios start out with page table
13881394 * references from the instantiating fault, so we need
1389− * to look twice if a mapped file/anon page is used more
1395+ * to look twice if a mapped file/anon folio is used more
13901396 * than once.
13911397 *
13921398 * Mark it and spare it for another trip around the
13931399 * inactive list. Another page table reference will
13941400 * lead to its activation.
13951401 *
1396− * Note: the mark is set for activated pages as well
1397− * so that recently deactivated but used pages are
1402+ * Note: the mark is set for activated folios as well
1403+ * so that recently deactivated but used folios are
13981404 * quickly recovered.
13991405 */
1400− SetPageReferenced(page);
1406+ folio_set_referenced(folio);
14011407
1402− if (referenced_page || referenced_ptes > 1)
1408+ if (referenced_folio || referenced_ptes > 1)
14031409 return PAGEREF_ACTIVATE;
14041410
14051411 /*
1406− * Activate file-backed executable pages after first usage.
1412+ * Activate file-backed executable folios after first usage.
14071413 */
1408− if ((vm_flags & VM_EXEC) && !PageSwapBacked(page))
1414+ if ((vm_flags & VM_EXEC) && !folio_test_swapbacked(folio))
14091415 return PAGEREF_ACTIVATE;
14101416
14111417 return PAGEREF_KEEP;
14121418 }
14131419
1414− /* Reclaim if clean, defer dirty pages to writeback */
1415− if (referenced_page && !PageSwapBacked(page))
1420+ /* Reclaim if clean, defer dirty folios to writeback */
1421+ if (referenced_folio && !folio_test_swapbacked(folio))
14161422 return PAGEREF_RECLAIM_CLEAN;
14171423
14181424 return PAGEREF_RECLAIM;
14191425 }
14201426
14211427 /* Check if a page is dirty or under writeback */
1422−static void page_check_dirty_writeback(struct page *page,
1428+static void folio_check_dirty_writeback(struct folio *folio,
14231429 bool *dirty, bool *writeback)
14241430 {
14251431 struct address_space *mapping;
@@ -1428,24 +1434,24 @@ static void page_check_dirty_writeback(struct page *page, · 2 unmodified lines
14281434 * Anonymous pages are not handled by flushers and must be written
14291435 * from reclaim context. Do not stall reclaim based on them
14301436 */
1431− if (!page_is_file_lru(page) ||
1432− (PageAnon(page) && !PageSwapBacked(page))) {
1437+ if (!folio_is_file_lru(folio) ||
1438+ (folio_test_anon(folio) && !folio_test_swapbacked(folio))) {
14331439 *dirty = false;
14341440 *writeback = false;
14351441 return;
14361442 }
14371443
1438− /* By default assume that the page flags are accurate */
1439− *dirty = PageDirty(page);
1440− *writeback = PageWriteback(page);
1444+ /* By default assume that the folio flags are accurate */
1445+ *dirty = folio_test_dirty(folio);
1446+ *writeback = folio_test_writeback(folio);
14411447
14421448 /* Verify dirty/writeback state if the filesystem supports it */
1443− if (!page_has_private(page))
1449+ if (!folio_test_private(folio))
14441450 return;
14451451
1446− mapping = page_mapping(page);
1452+ mapping = folio_mapping(folio);
14471453 if (mapping && mapping->a_ops->is_dirty_writeback)
1448− mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
1454+ mapping->a_ops->is_dirty_writeback(&folio->page, dirty, writeback);
14491455 }
14501456
14511457 static struct page *alloc_demote_page(struct page *page, unsigned long node)
@@ -1519,14 +1525,16 @@ retry: · 67 unmodified lines
15191525 while (!list_empty(page_list)) {
15201526 struct address_space *mapping;
15211527 struct page *page;
1528+ struct folio *folio;
15221529 enum page_references references = PAGEREF_RECLAIM;
15231530 bool dirty, writeback, may_enter_fs;
15241531 unsigned int nr_pages;
15251532
15261533 cond_resched();
15271534
1528− page = lru_to_page(page_list);
1529− list_del(&page->lru);
1535+ folio = lru_to_folio(page_list);
1536+ list_del(&folio->lru);
1537+ page = &folio->page;
15301538
15311539 if (!trylock_page(page))
15321540 goto keep;
@@ -1552,12 +1560,12 @@ retry: · 19 unmodified lines
15521560 * reclaim_congested. kswapd will stall and start writing
15531561 * pages if the tail of the LRU is all dirty unqueued pages.
15541562 */
1555− page_check_dirty_writeback(page, &dirty, &writeback);
1563+ folio_check_dirty_writeback(folio, &dirty, &writeback);
15561564 if (dirty || writeback)
1557− stat->nr_dirty++;
1565+ stat->nr_dirty += nr_pages;
15581566
15591567 if (dirty && !writeback)
1560− stat->nr_unqueued_dirty++;
1568+ stat->nr_unqueued_dirty += nr_pages;
15611569
15621570 /*
15631571 * Treat this page as congested if the underlying BDI is or if
@@ -1567,7 +1575,7 @@ retry: · 3 unmodified lines
15671575 */
15681576 mapping = page_mapping(page);
15691577 if (writeback && PageReclaim(page))
1570− stat->nr_congested++;
1578+ stat->nr_congested += nr_pages;
15711579
15721580 /*
15731581 * If a page at the tail of the LRU is under writeback, there
@@ -1616,7 +1624,7 @@ retry: · 42 unmodified lines
16161624 if (current_is_kswapd() &&
16171625 PageReclaim(page) &&
16181626 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1619− stat->nr_immediate++;
1627+ stat->nr_immediate += nr_pages;
16201628 goto activate_locked;
16211629
16221630 /* Case 2 above */
@@ -1634,7 +1642,7 @@ retry: · 11 unmodified lines
16341642 * and it's also appropriate in global reclaim.
16351643 */
16361644 SetPageReclaim(page);
1637− stat->nr_writeback++;
1645+ stat->nr_writeback += nr_pages;
16381646 goto activate_locked;
16391647
16401648 /* Case 3 above */
@@ -1648,7 +1656,7 @@ retry: · 7 unmodified lines
16481656 }
16491657
16501658 if (!ignore_references)
1651− references = page_check_references(page, sc);
1659+ references = folio_check_references(folio, sc);
16521660
16531661 switch (references) {
16541662 case PAGEREF_ACTIVATE:
@@ -1681,28 +1689,28 @@ retry: · 26 unmodified lines
16811689 if (!PageSwapCache(page)) {
16821690 if (!(sc->gfp_mask & __GFP_IO))
16831691 goto keep_locked;
1684− if (page_maybe_dma_pinned(page))
1692+ if (folio_maybe_dma_pinned(folio))
16851693 goto keep_locked;
16861694 if (PageTransHuge(page)) {
16871695 /* cannot split THP, skip it */
1688− if (!can_split_huge_page(page, NULL))
1696+ if (!can_split_folio(folio, NULL))
16891697 goto activate_locked;
16901698 /*
16911699 * Split pages without a PMD map right
16921700 * away. Chances are some or all of the
16931701 * tail pages can be freed without IO.
16941702 */
1695− if (!compound_mapcount(page) &&
1696− split_huge_page_to_list(page,
1697− page_list))
1703+ if (!folio_entire_mapcount(folio) &&
1704+ split_folio_to_list(folio,
1705+ page_list))
16981706 goto activate_locked;
16991707 }
17001708 if (!add_to_swap(page)) {
17011709 if (!PageTransHuge(page))
17021710 goto activate_locked_split;
17031711 /* Fallback to swap normal pages */
1704− if (split_huge_page_to_list(page,
1705− page_list))
1712+ if (split_folio_to_list(folio,
1713+ page_list))
17061714 goto activate_locked;
17071715 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
17081716 count_vm_event(THP_SWPOUT_FALLBACK);
@@ -1716,9 +1724,9 @@ retry: · 7 unmodified lines
17161724 /* Adding to swap updated mapping */
17171725 mapping = page_mapping(page);
17181726 }
1719− } else if (unlikely(PageTransHuge(page))) {
1720− /* Split file THP */
1721− if (split_huge_page_to_list(page, page_list))
1727+ } else if (PageSwapBacked(page) && PageTransHuge(page)) {
1728+ /* Split shmem THP */
1729+ if (split_folio_to_list(folio, page_list))
17221730 goto keep_locked;
17231731 }
17241732
@@ -1742,10 +1750,11 @@ retry: · 17 unmodified lines
17421750 enum ttu_flags flags = TTU_BATCH_FLUSH;
17431751 bool was_swapbacked = PageSwapBacked(page);
17441752
1745− if (unlikely(PageTransHuge(page)))
1753+ if (PageTransHuge(page) &&
1754+ thp_order(page) >= HPAGE_PMD_ORDER)
17461755 flags |= TTU_SPLIT_HUGE_PMD;
17471756
1748− try_to_unmap(page, flags);
1757+ try_to_unmap(folio, flags);
17491758 if (page_mapped(page)) {
17501759 stat->nr_unmap_fail += nr_pages;
17511760 if (!was_swapbacked && PageSwapBacked(page))
@@ -1793,13 +1802,13 @@ retry: · 41 unmodified lines
17931802 * starts and then write it out here.
17941803 */
17951804 try_to_unmap_flush_dirty();
1796− switch (pageout(page, mapping)) {
1805+ switch (pageout(folio, mapping)) {
17971806 case PAGE_KEEP:
17981807 goto keep_locked;
17991808 case PAGE_ACTIVATE:
18001809 goto activate_locked;
18011810 case PAGE_SUCCESS:
1802− stat->nr_pageout += thp_nr_pages(page);
1811+ stat->nr_pageout += nr_pages;
18031812
18041813 if (PageWriteback(page))
18051814 goto keep;
@@ -1877,7 +1886,7 @@ retry: · 71 unmodified lines
18771886 */
18781887 count_vm_event(PGLAZYFREED);
18791888 count_memcg_page_event(page, PGLAZYFREED);
1880− } else if (!mapping || !__remove_mapping(mapping, page, true,
1889+ } else if (!mapping || !__remove_mapping(mapping, folio, true,
18811890 sc->target_mem_cgroup))
18821891 goto keep_locked;
18831892
@@ -2132,45 +2141,40 @@ move: · 248 unmodified lines
21322141 }
21332142
21342143 /**
2135− * isolate_lru_page - tries to isolate a page from its LRU list
2136− * @page: page to isolate from its LRU list
2144+ * folio_isolate_lru() - Try to isolate a folio from its LRU list.
2145+ * @folio: Folio to isolate from its LRU list.
21372146 *
2138− * Isolates a @page from an LRU list, clears PageLRU and adjusts the
2139− * vmstat statistic corresponding to whatever LRU list the page was on.
2147+ * Isolate a @folio from an LRU list and adjust the vmstat statistic
2148+ * corresponding to whatever LRU list the folio was on.
21402149 *
2141− * Returns 0 if the page was removed from an LRU list.
2142− * Returns -EBUSY if the page was not on an LRU list.
2143− *
2144− * The returned page will have PageLRU() cleared. If it was found on
2145− * the active list, it will have PageActive set. If it was found on
2146− * the unevictable list, it will have the PageUnevictable bit set. That flag
2150+ * The folio will have its LRU flag cleared. If it was found on the
2151+ * active list, it will have the Active flag set. If it was found on the
2152+ * unevictable list, it will have the Unevictable flag set. These flags
21472153 * may need to be cleared by the caller before letting the page go.
21482154 *
2149− * The vmstat statistic corresponding to the list on which the page was
2150− * found will be decremented.
2151− *
2152− * Restrictions:
2155+ * Context:
21532156 *
21542157 * (1) Must be called with an elevated refcount on the page. This is a
2155− * fundamental difference from isolate_lru_pages (which is called
2158+ * fundamental difference from isolate_lru_pages() (which is called
21562159 * without a stable reference).
2157− * (2) the lru_lock must not be held.
2158− * (3) interrupts must be enabled.
2160+ * (2) The lru_lock must not be held.
2161+ * (3) Interrupts must be enabled.
2162+ *
2163+ * Return: 0 if the folio was removed from an LRU list.
2164+ * -EBUSY if the folio was not on an LRU list.
21592165 */
2160−int isolate_lru_page(struct page *page)
2166+int folio_isolate_lru(struct folio *folio)
21612167 {
2162− struct folio *folio = page_folio(page);
21632168 int ret = -EBUSY;
21642169
2165− VM_BUG_ON_PAGE(!page_count(page), page);
2166− WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
2170+ VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio);
21672171
2168− if (TestClearPageLRU(page)) {
2172+ if (folio_test_clear_lru(folio)) {
21692173 struct lruvec *lruvec;
21702174
2171− get_page(page);
2175+ folio_get(folio);
21722176 lruvec = folio_lruvec_lock_irq(folio);
2173− del_page_from_lru_list(page, lruvec);
2177+ lruvec_del_folio(lruvec, folio);
21742178 unlock_page_lruvec_irq(lruvec);
21752179 ret = 0;
21762180 }
@@ -2406,7 +2410,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 229 unmodified lines
24062410 *
24072411 * If the pages are mostly unmapped, the processing is fast and it is
24082412 * appropriate to hold lru_lock across the whole operation. But if
2409− * the pages are mapped, the processing is slow (page_referenced()), so
2413+ * the pages are mapped, the processing is slow (folio_referenced()), so
24102414 * we should drop lru_lock around each page. It's impossible to balance
24112415 * this, so instead we remove the pages from the LRU while processing them.
24122416 * It is safe to rely on PG_active against the non-LRU pages in here because
@@ -2426,7 +2430,6 @@ static void shrink_active_list(unsigned long nr_to_scan, · 13 unmodified lines
24262430 LIST_HEAD(l_hold); /* The pages which were snipped off */
24272431 LIST_HEAD(l_active);
24282432 LIST_HEAD(l_inactive);
2429− struct page *page;
24302433 unsigned nr_deactivate, nr_activate;
24312434 unsigned nr_rotated = 0;
24322435 int file = is_file_lru(lru);
@@ -2448,9 +2451,13 @@ static void shrink_active_list(unsigned long nr_to_scan, · 15 unmodified lines
24482451 spin_unlock_irq(&lruvec->lru_lock);
24492452
24502453 while (!list_empty(&l_hold)) {
2454+ struct folio *folio;
2455+ struct page *page;
2456+
24512457 cond_resched();
2452− page = lru_to_page(&l_hold);
2453− list_del(&page->lru);
2458+ folio = lru_to_folio(&l_hold);
2459+ list_del(&folio->lru);
2460+ page = &folio->page;
24542461
24552462 if (unlikely(!page_evictable(page))) {
24562463 putback_lru_page(page);
@@ -2465,8 +2472,8 @@ static void shrink_active_list(unsigned long nr_to_scan, · 8 unmodified lines
24652472 }
24662473 }
24672474
2468− if (page_referenced(page, 0, sc->target_mem_cgroup,
2469− &vm_flags)) {
2475+ if (folio_referenced(folio, 0, sc->target_mem_cgroup,
2476+ &vm_flags)) {
24702477 /*
24712478 * Identify referenced, file-backed active pages and
24722479 * give them one more trip around the active list. So
mm/workingset.c+13−12
@@ -245,31 +245,32 @@ void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages) · 244 unmodified lines
245245 }
246246
247247 /**
248− * workingset_eviction - note the eviction of a page from memory
248+ * workingset_eviction - note the eviction of a folio from memory
249249 * @target_memcg: the cgroup that is causing the reclaim
250− * @page: the page being evicted
250+ * @folio: the folio being evicted
251251 *
252− * Return: a shadow entry to be stored in @page->mapping->i_pages in place
253− * of the evicted @page so that a later refault can be detected.
252+ * Return: a shadow entry to be stored in @folio->mapping->i_pages in place
253+ * of the evicted @folio so that a later refault can be detected.
254254 */
255−void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg)
255+void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg)
256256 {
257− struct pglist_data *pgdat = page_pgdat(page);
257+ struct pglist_data *pgdat = folio_pgdat(folio);
258258 unsigned long eviction;
259259 struct lruvec *lruvec;
260260 int memcgid;
261261
262− /* Page is fully exclusive and pins page's memory cgroup pointer */
263− VM_BUG_ON_PAGE(PageLRU(page), page);
264− VM_BUG_ON_PAGE(page_count(page), page);
265− VM_BUG_ON_PAGE(!PageLocked(page), page);
262+ /* Folio is fully exclusive and pins folio's memory cgroup pointer */
263+ VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
264+ VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
265+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
266266
267267 lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
268268 /* XXX: target_memcg can be NULL, go through lruvec */
269269 memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
270270 eviction = atomic_long_read(&lruvec->nonresident_age);
271− workingset_age_nonresident(lruvec, thp_nr_pages(page));
272− return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));
271+ workingset_age_nonresident(lruvec, folio_nr_pages(folio));
272+ return pack_shadow(memcgid, pgdat, eviction,
273+ folio_test_workingset(folio));
273274 }
274275
275276 /**
tools/testing/selftests/vm/transhuge-stress.c+24−11
@@ -26,15 +26,17 @@ · 25 unmodified lines
2626 #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1))
2727
2828 int pagemap_fd;
29+int backing_fd = -1;
30+int mmap_flags = MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE;
31+#define PROT_RW (PROT_READ | PROT_WRITE)
2932
3033 int64_t allocate_transhuge(void *ptr)
3134 {
3235 uint64_t ent[2];
3336
3437 /* drop pmd */
35− if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
36− MAP_FIXED | MAP_ANONYMOUS |
37− MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
38+ if (mmap(ptr, HPAGE_SIZE, PROT_RW, MAP_FIXED | mmap_flags,
39+ backing_fd, 0) != ptr)
3840 errx(2, "mmap transhuge");
3941
4042 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
@@ -60,6 +62,8 @@ int main(int argc, char **argv) · 19 unmodified lines
6062 size_t ram, len;
6163 void *ptr, *p;
6264 struct timespec a, b;
65+ int i = 0;
66+ char *name = NULL;
6367 double s;
6468 uint8_t *map;
6569 size_t map_len;
@@ -69,13 +73,23 @@ int main(int argc, char **argv) · 3 unmodified lines
6973 ram = SIZE_MAX / 4;
7074 else
7175 ram *= sysconf(_SC_PAGESIZE);
76+ len = ram;
77+
78+ while (++i < argc) {
79+ if (!strcmp(argv[i], "-h"))
80+ errx(1, "usage: %s [size in MiB]", argv[0]);
81+ else if (!strcmp(argv[i], "-f"))
82+ name = argv[++i];
83+ else
84+ len = atoll(argv[i]) << 20;
85+ }
7286
73− if (argc == 1)
74− len = ram;
75− else if (!strcmp(argv[1], "-h"))
76− errx(1, "usage: %s [size in MiB]", argv[0]);
77− else
78− len = atoll(argv[1]) << 20;
87+ if (name) {
88+ backing_fd = open(name, O_RDWR);
89+ if (backing_fd == -1)
90+ errx(2, "open %s", name);
91+ mmap_flags = MAP_SHARED;
92+ }
7993
8094 warnx("allocate %zd transhuge pages, using %zd MiB virtual memory"
8195 " and %zd MiB of ram", len >> HPAGE_SHIFT, len >> 20,
@@ -86,8 +100,7 @@ int main(int argc, char **argv) · 4 unmodified lines
86100 err(2, "open pagemap");
87101
88102 len -= len % HPAGE_SIZE;
89− ptr = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
90− MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
103+ ptr = mmap(NULL, len + HPAGE_SIZE, PROT_RW, mmap_flags, backing_fd, 0);
91104 if (ptr == MAP_FAILED)
92105 err(2, "initial mmap");
93106 ptr += HPAGE_SIZE - (uintptr_t)ptr % HPAGE_SIZE;