lael-labs/linux

Documentation/core-api/pin_user_pages.rst+9−9

@@ -55,18 +55,18 @@ flags the caller provides. The caller is required to pass in a non-null struct · 54 unmodified lines
55	pages* array, and the function then pins pages by incrementing each by a special	55	pages* array, and the function then pins pages by incrementing each by a special
56	value: GUP_PIN_COUNTING_BIAS.	56	value: GUP_PIN_COUNTING_BIAS.
57		57
58	For huge pages (and in fact, any compound page of more than 2 pages), the	58	For compound pages, the GUP_PIN_COUNTING_BIAS scheme is not used. Instead,
59	GUP_PIN_COUNTING_BIAS scheme is not used. Instead, an exact form of pin counting	59	an exact form of pin counting is achieved, by using the 2nd struct page
60	is achieved, by using the 3rd struct page in the compound page. A new struct	60	in the compound page. A new struct page field, compound_pincount, has
61	page field, hpage_pinned_refcount, has been added in order to support this.	61	been added in order to support this.
62		62
63	This approach for compound pages avoids the counting upper limit problems that	63	This approach for compound pages avoids the counting upper limit problems that
64	are discussed below. Those limitations would have been aggravated severely by	64	are discussed below. Those limitations would have been aggravated severely by
65	huge pages, because each tail page adds a refcount to the head page. And in	65	huge pages, because each tail page adds a refcount to the head page. And in
66	fact, testing revealed that, without a separate hpage_pinned_refcount field,	66	fact, testing revealed that, without a separate compound_pincount field,
67	page overflows were seen in some huge page stress tests.	67	page overflows were seen in some huge page stress tests.
68		68
69	This also means that huge pages and compound pages (of order > 1) do not suffer	69	This also means that huge pages and compound pages do not suffer
70	from the false positives problem that is mentioned below.::	70	from the false positives problem that is mentioned below.::
71		71
72	Function	72	Function
@@ -264,9 +264,9 @@ place.) · 191 unmodified lines
264	Other diagnostics	264	Other diagnostics
265	=================	265	=================
266		266
267	dump_page() has been enhanced slightly, to handle these new counting fields, and	267	dump_page() has been enhanced slightly, to handle these new counting
268	to better report on compound pages in general. Specifically, for compound pages	268	fields, and to better report on compound pages in general. Specifically,
269	with order > 1, the exact (hpage_pinned_refcount) pincount is reported.	269	for compound pages, the exact (compound_pincount) pincount is reported.
270		270
271	References	271	References
272	==========	272	==========

arch/alpha/include/asm/pgtable.h+1−0

@@ -233,6 +233,7 @@ pmd_page_vaddr(pmd_t pmd) · 232 unmodified lines
233	return ((pmd_val(pmd) & _PFN_MASK) >> (32-PAGE_SHIFT)) + PAGE_OFFSET;	233	return ((pmd_val(pmd) & _PFN_MASK) >> (32-PAGE_SHIFT)) + PAGE_OFFSET;
234	}	234	}
235		235
		236	#define pmd_pfn(pmd) (pmd_val(pmd) >> 32)
236	#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> 32))	237	#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> 32))
237	#define pud_page(pud) (pfn_to_page(pud_val(pud) >> 32))	238	#define pud_page(pud) (pfn_to_page(pud_val(pud) >> 32))
238		239

arch/arc/include/asm/hugepage.h+0−1

@@ -31,7 +31,6 @@ static inline pmd_t pte_pmd(pte_t pte) · 30 unmodified lines
31		31
32	#define pmd_write(pmd) pte_write(pmd_pte(pmd))	32	#define pmd_write(pmd) pte_write(pmd_pte(pmd))
33	#define pmd_young(pmd) pte_young(pmd_pte(pmd))	33	#define pmd_young(pmd) pte_young(pmd_pte(pmd))
34	#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
35	#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))	34	#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
36		35
37	#define mk_pmd(page, prot) pte_pmd(mk_pte(page, prot))	36	#define mk_pmd(page, prot) pte_pmd(mk_pte(page, prot))

arch/arc/include/asm/pgtable-levels.h+1−0

@@ -161,6 +161,7 @@ · 160 unmodified lines
161	#define pmd_present(x) (pmd_val(x))	161	#define pmd_present(x) (pmd_val(x))
162	#define pmd_clear(xp) do { pmd_val(*(xp)) = 0; } while (0)	162	#define pmd_clear(xp) do { pmd_val(*(xp)) = 0; } while (0)
163	#define pmd_page_vaddr(pmd) (pmd_val(pmd) & PAGE_MASK)	163	#define pmd_page_vaddr(pmd) (pmd_val(pmd) & PAGE_MASK)
		164	#define pmd_pfn(pmd) ((pmd_val(pmd) & PAGE_MASK) >> PAGE_SHIFT)
164	#define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd))	165	#define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd))
165	#define set_pmd(pmdp, pmd) (*(pmdp) = pmd)	166	#define set_pmd(pmdp, pmd) (*(pmdp) = pmd)
166	#define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd))	167	#define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd))

arch/arm/include/asm/pgtable-2level.h+2−0

@@ -208,6 +208,8 @@ static inline pmd_t pmd_offset(pud_t pud, unsigned long addr) · 207 unmodified lines
208	}	208	}
209	#define pmd_offset pmd_offset	209	#define pmd_offset pmd_offset
210		210
		211	#define pmd_pfn(pmd) (__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
		212
211	#define pmd_large(pmd) (pmd_val(pmd) & 2)	213	#define pmd_large(pmd) (pmd_val(pmd) & 2)
212	#define pmd_leaf(pmd) (pmd_val(pmd) & 2)	214	#define pmd_leaf(pmd) (pmd_val(pmd) & 2)
213	#define pmd_bad(pmd) (pmd_val(pmd) & 2)	215	#define pmd_bad(pmd) (pmd_val(pmd) & 2)

arch/arm64/mm/mmu.c+1−0

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	#include <linux/mman.h>	17	#include <linux/mman.h>
18	#include <linux/nodemask.h>	18	#include <linux/nodemask.h>
19	#include <linux/memblock.h>	19	#include <linux/memblock.h>
		20	#include <linux/memremap.h>
20	#include <linux/memory.h>	21	#include <linux/memory.h>
21	#include <linux/fs.h>	22	#include <linux/fs.h>
22	#include <linux/io.h>	23	#include <linux/io.h>

arch/csky/include/asm/pgtable.h+1−0

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	#define pgd_ERROR(e) \	30	#define pgd_ERROR(e) \
31	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))	31	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
32		32
		33	#define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT)
33	#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))	34	#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
34	#define pte_clear(mm, addr, ptep) set_pte((ptep), \	35	#define pte_clear(mm, addr, ptep) set_pte((ptep), \
35	(((unsigned int) addr >= PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0)))	36	(((unsigned int) addr >= PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0)))

arch/hexagon/include/asm/pgtable.h+5−0

 	return 0;
 }
+/*
+ * pmd_pfn - converts a PMD entry to a page frame number
+ */
+#define pmd_pfn(pmd)  (pmd_val(pmd) >> PAGE_SHIFT)
 /*
  * pmd_page - converts a PMD entry to a page pointer
  */

arch/ia64/include/asm/pgtable.h+1−0

@@ -267,6 +267,7 @@ ia64_phys_addr_valid (unsigned long addr) · 266 unmodified lines
267	#define pmd_present(pmd) (pmd_val(pmd) != 0UL)	267	#define pmd_present(pmd) (pmd_val(pmd) != 0UL)
268	#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)	268	#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
269	#define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val(pmd) & _PFN_MASK))	269	#define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val(pmd) & _PFN_MASK))
		270	#define pmd_pfn(pmd) ((pmd_val(pmd) & _PFN_MASK) >> PAGE_SHIFT)
270	#define pmd_page(pmd) virt_to_page((pmd_val(pmd) + PAGE_OFFSET))	271	#define pmd_page(pmd) virt_to_page((pmd_val(pmd) + PAGE_OFFSET))
271		272
272	#define pud_none(pud) (!pud_val(pud))	273	#define pud_none(pud) (!pud_val(pud))

arch/m68k/include/asm/mcf_pgtable.h+1−0

@@ -322,6 +322,7 @@ extern pgd_t kernel_pg_dir[PTRS_PER_PGD]; · 321 unmodified lines
322	#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })	322	#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
323	#define __swp_entry_to_pte(x) (__pte((x).val))	323	#define __swp_entry_to_pte(x) (__pte((x).val))
324		324
		325	#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
325	#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT))	326	#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT))
326		327
327	#define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) \| pgprot_val(prot))	328	#define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) \| pgprot_val(prot))

arch/m68k/include/asm/motorola_pgtable.h+1−0

@@ -147,6 +147,7 @@ static inline void pud_set(pud_t pudp, pmd_t pmdp) · 146 unmodified lines
147	#define pmd_present(pmd) (pmd_val(pmd) & _PAGE_TABLE)	147	#define pmd_present(pmd) (pmd_val(pmd) & _PAGE_TABLE)
148	#define pmd_clear(pmdp) ({ pmd_val(*pmdp) = 0; })	148	#define pmd_clear(pmdp) ({ pmd_val(*pmdp) = 0; })
149		149
		150	#define pmd_pfn(pmd) ((pmd_val(pmd) & _TABLE_MASK) >> PAGE_SHIFT)
150	/*	151	/*
151	* m68k does not have huge pages (020/030 actually could), but generic code	152	* m68k does not have huge pages (020/030 actually could), but generic code
152	* expects pmd_page() to exists, only to then DCE it all. Provide a dummy to	153	* expects pmd_page() to exists, only to then DCE it all. Provide a dummy to

arch/m68k/include/asm/sun3_pgtable.h+1−0

@@ -130,6 +130,7 @@ static inline void pte_clear (struct mm_struct mm, unsigned long addr, pte_t p · 129 unmodified lines
130	({ pte_t __pte; pte_val(__pte) = pfn \| pgprot_val(pgprot); __pte; })	130	({ pte_t __pte; pte_val(__pte) = pfn \| pgprot_val(pgprot); __pte; })
131		131
132	#define pte_page(pte) virt_to_page(__pte_page(pte))	132	#define pte_page(pte) virt_to_page(__pte_page(pte))
		133	#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
133	#define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd))	134	#define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd))
134		135
135		136

arch/microblaze/include/asm/pgtable.h+3−0

@@ -399,6 +399,9 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 398 unmodified lines
399	return ((unsigned long) (pmd_val(pmd) & PAGE_MASK));	399	return ((unsigned long) (pmd_val(pmd) & PAGE_MASK));
400	}	400	}
401		401
		402	/* returns pfn of the pmd entry*/
		403	#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT)
		404
402	/* returns struct page of the pmd entry/	405	/* returns struct page of the pmd entry/
403	#define pmd_page(pmd) (pfn_to_page(__pa(pmd_val(pmd)) >> PAGE_SHIFT))	406	#define pmd_page(pmd) (pfn_to_page(__pa(pmd_val(pmd)) >> PAGE_SHIFT))
404		407

arch/mips/include/asm/pgtable.h+5−5

  */
 #define pmd_phys(pmd)		virt_to_phys((void *)pmd_val(pmd))
+static inline unsigned long pmd_pfn(pmd_t pmd)
+{
+	return pmd_val(pmd) >> _PFN_SHIFT;
+}
 #ifndef CONFIG_MIPS_HUGE_TLB_SUPPORT
 #define pmd_page(pmd)		(pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
 #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
 	return !!(pmd_val(pmd) & _PAGE_WRITE);
 }
-static inline unsigned long pmd_pfn(pmd_t pmd)
-{
-	return pmd_val(pmd) >> _PFN_SHIFT;
-}
 static inline struct page *pmd_page(pmd_t pmd)
 {
 	if (pmd_val(pmd) & _PAGE_HUGE)

arch/nds32/include/asm/pgtable.h+1−0

@@ -308,6 +308,7 @@ static inline pmd_t __mk_pmd(pte_t * ptep, unsigned long prot) · 307 unmodified lines
308	return pmd;	308	return pmd;
309	}	309	}
310		310
		311	#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
311	#define pmd_page(pmd) virt_to_page(__va(pmd_val(pmd)))	312	#define pmd_page(pmd) virt_to_page(__va(pmd_val(pmd)))
312		313
313	/*	314	/*

arch/nios2/include/asm/pgtable.h+1−0

@@ -235,6 +235,7 @@ static inline void pte_clear(struct mm_struct *mm, · 234 unmodified lines
235	* and a page entry and page directory to the page they refer to.	235	* and a page entry and page directory to the page they refer to.
236	*/	236	*/
237	#define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd))	237	#define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd))
		238	#define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT)
238	#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))	239	#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
239		240
240	static inline unsigned long pmd_page_vaddr(pmd_t pmd)	241	static inline unsigned long pmd_page_vaddr(pmd_t pmd)

arch/openrisc/include/asm/pgtable.h+1−0

@@ -361,6 +361,7 @@ static inline void pmd_set(pmd_t pmdp, pte_t ptep) · 360 unmodified lines
361	pmd_val(*pmdp) = _KERNPG_TABLE \| (unsigned long) ptep;	361	pmd_val(*pmdp) = _KERNPG_TABLE \| (unsigned long) ptep;
362	}	362	}
363		363
		364	#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
364	#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT))	365	#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT))
365		366
366	static inline unsigned long pmd_page_vaddr(pmd_t pmd)	367	static inline unsigned long pmd_page_vaddr(pmd_t pmd)

arch/parisc/include/asm/pgtable.h+1−0

@@ -408,6 +408,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 407 unmodified lines
408	return ((unsigned long) __va(pmd_address(pmd)));	408	return ((unsigned long) __va(pmd_address(pmd)));
409	}	409	}
410		410
		411	#define pmd_pfn(pmd) (pmd_address(pmd) >> PAGE_SHIFT)
411	#define __pmd_page(pmd) ((unsigned long) __va(pmd_address(pmd)))	412	#define __pmd_page(pmd) ((unsigned long) __va(pmd_address(pmd)))
412	#define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd))	413	#define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd))
413		414

arch/powerpc/include/asm/book3s/32/pgtable.h+2−2

@@ -372,8 +372,8 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, · 371 unmodified lines
372	#define __HAVE_ARCH_PTE_SAME	372	#define __HAVE_ARCH_PTE_SAME
373	#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)	373	#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)
374		374
375	#define pmd_page(pmd) \	375	#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
376	pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)	376	#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
377		377
378	/*	378	/*
379	* Encode and decode a swap entry.	379	* Encode and decode a swap entry.

arch/powerpc/include/asm/mmu_context.h+0−1

@@ -21,7 +21,6 @@ extern void destroy_context(struct mm_struct *mm); · 20 unmodified lines
21	#ifdef CONFIG_SPAPR_TCE_IOMMU	21	#ifdef CONFIG_SPAPR_TCE_IOMMU
22	struct mm_iommu_table_group_mem_t;	22	struct mm_iommu_table_group_mem_t;
23		23
24	extern int isolate_lru_page(struct page page); / from internal.h */
25	extern bool mm_iommu_preregistered(struct mm_struct *mm);	24	extern bool mm_iommu_preregistered(struct mm_struct *mm);
26	extern long mm_iommu_new(struct mm_struct *mm,	25	extern long mm_iommu_new(struct mm_struct *mm,
27	unsigned long ua, unsigned long entries,	26	unsigned long ua, unsigned long entries,

arch/powerpc/include/asm/nohash/32/pgtable.h+3−4

  * of the pte page.  -- paulus
  */
 #ifndef CONFIG_BOOKE
-#define pmd_page(pmd)		\
+#define pmd_pfn(pmd)		(pmd_val(pmd) >> PAGE_SHIFT)
-	pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
 #else
 #define pmd_page_vaddr(pmd)	\
 	((unsigned long)(pmd_val(pmd) & ~(PTE_TABLE_SIZE - 1)))
-#define pmd_page(pmd)		\
+#define pmd_pfn(pmd)		(__pa(pmd_val(pmd)) >> PAGE_SHIFT)
-	pfn_to_page((__pa(pmd_val(pmd)) >> PAGE_SHIFT))
 #endif
+#define pmd_page(pmd)		pfn_to_page(pmd_pfn(pmd))
 /*
  * Encode and decode a swap entry.
  * Note that the bits we use in a PTE for representing a swap entry

arch/powerpc/include/asm/nohash/64/pgtable.h+1−0

@@ -142,6 +142,7 @@ static inline pte_t pmd_pte(pmd_t pmd) · 141 unmodified lines
142	#define pmd_present(pmd) (!pmd_none(pmd))	142	#define pmd_present(pmd) (!pmd_none(pmd))
143	#define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS)	143	#define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS)
144	extern struct page *pmd_page(pmd_t pmd);	144	extern struct page *pmd_page(pmd_t pmd);
		145	#define pmd_pfn(pmd) (page_to_pfn(pmd_page(pmd)))
145		146
146	static inline void pud_set(pud_t *pudp, unsigned long val)	147	static inline void pud_set(pud_t *pudp, unsigned long val)
147	{	148	{

arch/powerpc/kvm/book3s_hv_uvmem.c+1−1

 #include <linux/kvm_host.h>
 #include <linux/ksm.h>
 #include <linux/of.h>
+#include <linux/memremap.h>
 #include <asm/ultravisor.h>
 #include <asm/mman.h>
 #include <asm/kvm_ppc.h>
 	dpage = pfn_to_page(uvmem_pfn);
 	dpage->zone_device_data = pvt;
-	get_page(dpage);
 	lock_page(dpage);
 	return dpage;
 out_clear:

arch/powerpc/mm/book3s64/pgtable.c+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	#include <linux/sched.h>	6	#include <linux/sched.h>
7	#include <linux/mm_types.h>	7	#include <linux/mm_types.h>
8	#include <linux/memblock.h>	8	#include <linux/memblock.h>
		9	#include <linux/memremap.h>
9	#include <linux/debugfs.h>	10	#include <linux/debugfs.h>
10	#include <misc/cxl-base.h>	11	#include <misc/cxl-base.h>
11		12

arch/sh/include/asm/pgtable_32.h+1−0

@@ -406,6 +406,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 405 unmodified lines
406	return (unsigned long)pmd_val(pmd);	406	return (unsigned long)pmd_val(pmd);
407	}	407	}
408		408
		409	#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT)
409	#define pmd_page(pmd) (virt_to_page(pmd_val(pmd)))	410	#define pmd_page(pmd) (virt_to_page(pmd_val(pmd)))
410		411
411	#ifdef CONFIG_X2TLB	412	#ifdef CONFIG_X2TLB

arch/sparc/include/asm/pgtable_32.h+6−1

 	return ((x & 0xF0000000) != 0);
 }
+static inline unsigned long pmd_pfn(pmd_t pmd)
+{
+	return (pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4);
+}
 static inline struct page *pmd_page(pmd_t pmd)
 {
 	if (srmmu_device_memory(pmd_val(pmd)))
 		BUG();
-	return pfn_to_page((pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4));
+	return pfn_to_page(pmd_pfn(pmd));
 }
 static inline unsigned long __pmd_page(pmd_t pmd)

arch/um/include/asm/pgtable.h+1−0

@@ -109,6 +109,7 @@ extern unsigned long end_iomem; · 108 unmodified lines
109	#define p4d_newpage(x) (p4d_val(x) & _PAGE_NEWPAGE)	109	#define p4d_newpage(x) (p4d_val(x) & _PAGE_NEWPAGE)
110	#define p4d_mkuptodate(x) (p4d_val(x) &= ~_PAGE_NEWPAGE)	110	#define p4d_mkuptodate(x) (p4d_val(x) &= ~_PAGE_NEWPAGE)
111		111
		112	#define pmd_pfn(pmd) (pmd_val(pmd) >> PAGE_SHIFT)
112	#define pmd_page(pmd) phys_to_page(pmd_val(pmd) & PAGE_MASK)	113	#define pmd_page(pmd) phys_to_page(pmd_val(pmd) & PAGE_MASK)
113		114
114	#define pte_page(x) pfn_to_page(pte_pfn(x))	115	#define pte_page(x) pfn_to_page(pte_pfn(x))

arch/xtensa/include/asm/pgtable.h+1−0

@@ -241,6 +241,7 @@ static inline void paging_init(void) { } · 240 unmodified lines
241	* The pmd contains the kernel virtual address of the pte page.	241	* The pmd contains the kernel virtual address of the pte page.
242	*/	242	*/
243	#define pmd_page_vaddr(pmd) ((unsigned long)(pmd_val(pmd) & PAGE_MASK))	243	#define pmd_page_vaddr(pmd) ((unsigned long)(pmd_val(pmd) & PAGE_MASK))
		244	#define pmd_pfn(pmd) (__pa(pmd_val(pmd)) >> PAGE_SHIFT)
244	#define pmd_page(pmd) virt_to_page(pmd_val(pmd))	245	#define pmd_page(pmd) virt_to_page(pmd_val(pmd))
245		246
246	/*	247	/*

drivers/block/xen-blkback/xenbus.c+1−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10		10
11	#include <linux/module.h>	11	#include <linux/module.h>
12	#include <linux/kthread.h>	12	#include <linux/kthread.h>
		13	#include <linux/pagemap.h>
13	#include <xen/events.h>	14	#include <xen/events.h>
14	#include <xen/grant_table.h>	15	#include <xen/grant_table.h>
15	#include "common.h"	16	#include "common.h"

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c+1−1

 #include <linux/hmm.h>
 #include <linux/dma-direction.h>
 #include <linux/dma-mapping.h>
+#include <linux/migrate.h>
 #include "amdgpu_sync.h"
 #include "amdgpu_object.h"
 #include "amdgpu_vm.h"
 	page = pfn_to_page(pfn);
 	svm_range_bo_ref(prange->svm_bo);
 	page->zone_device_data = prange->svm_bo;
-	get_page(page);
 	lock_page(page);
 }

drivers/gpu/drm/amd/amdkfd/kfd_priv.h+1−0

@@ -25,6 +25,7 @@ · 24 unmodified lines
25		25
26	#include <linux/hashtable.h>	26	#include <linux/hashtable.h>
27	#include <linux/mmu_notifier.h>	27	#include <linux/mmu_notifier.h>
		28	#include <linux/memremap.h>
28	#include <linux/mutex.h>	29	#include <linux/mutex.h>
29	#include <linux/types.h>	30	#include <linux/types.h>
30	#include <linux/atomic.h>	31	#include <linux/atomic.h>

drivers/gpu/drm/drm_cache.c+1−1

 /*
  * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
  */
 #include <linux/dma-buf-map.h>
 #include <linux/export.h>
 #include <linux/highmem.h>
 #include <linux/cc_platform.h>
+#include <linux/ioport.h>
 #include <xen/xen.h>
 #include <drm/drm_cache.h>

drivers/gpu/drm/nouveau/nouveau_dmem.c+2−1

 #include <linux/sched/mm.h>
 #include <linux/hmm.h>
+#include <linux/memremap.h>
+#include <linux/migrate.h>
 /*
  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
 			return NULL;
 	}
-	get_page(page);
 	lock_page(page);
 	return page;
 }

drivers/gpu/drm/nouveau/nouveau_svm.c+1−0

@@ -35,6 +35,7 @@ · 34 unmodified lines
35	#include <linux/sched/mm.h>	35	#include <linux/sched/mm.h>
36	#include <linux/sort.h>	36	#include <linux/sort.h>
37	#include <linux/hmm.h>	37	#include <linux/hmm.h>
		38	#include <linux/memremap.h>
38	#include <linux/rmap.h>	39	#include <linux/rmap.h>
39		40
40	struct nouveau_svm {	41	struct nouveau_svm {

drivers/infiniband/core/rw.c+1−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	/*	2	/*
3	* Copyright (c) 2016 HGST, a Western Digital Company.	3	* Copyright (c) 2016 HGST, a Western Digital Company.
4	*/	4	*/
		5	#include <linux/memremap.h>
5	#include <linux/moduleparam.h>	6	#include <linux/moduleparam.h>
6	#include <linux/slab.h>	7	#include <linux/slab.h>
7	#include <linux/pci-p2pdma.h>	8	#include <linux/pci-p2pdma.h>

drivers/nvdimm/pmem.h+1−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	#define __NVDIMM_PMEM_H__	3	#define __NVDIMM_PMEM_H__
4	#include <linux/page-flags.h>	4	#include <linux/page-flags.h>
5	#include <linux/badblocks.h>	5	#include <linux/badblocks.h>
		6	#include <linux/memremap.h>
6	#include <linux/types.h>	7	#include <linux/types.h>
7	#include <linux/pfn_t.h>	8	#include <linux/pfn_t.h>
8	#include <linux/fs.h>	9	#include <linux/fs.h>

drivers/nvme/host/pci.c+1−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	#include <linux/init.h>	15	#include <linux/init.h>
16	#include <linux/interrupt.h>	16	#include <linux/interrupt.h>
17	#include <linux/io.h>	17	#include <linux/io.h>
		18	#include <linux/memremap.h>
18	#include <linux/mm.h>	19	#include <linux/mm.h>
19	#include <linux/module.h>	20	#include <linux/module.h>
20	#include <linux/mutex.h>	21	#include <linux/mutex.h>

drivers/nvme/target/io-cmd-bdev.c+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt	6	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7	#include <linux/blkdev.h>	7	#include <linux/blkdev.h>
8	#include <linux/blk-integrity.h>	8	#include <linux/blk-integrity.h>
		9	#include <linux/memremap.h>
9	#include <linux/module.h>	10	#include <linux/module.h>
10	#include "nvmet.h"	11	#include "nvmet.h"
11		12

drivers/usb/gadget/function/f_mass_storage.c+1−0

@@ -179,6 +179,7 @@ · 178 unmodified lines
179	#include <linux/kthread.h>	179	#include <linux/kthread.h>
180	#include <linux/sched/signal.h>	180	#include <linux/sched/signal.h>
181	#include <linux/limits.h>	181	#include <linux/limits.h>
		182	#include <linux/pagemap.h>
182	#include <linux/rwsem.h>	183	#include <linux/rwsem.h>
183	#include <linux/slab.h>	184	#include <linux/slab.h>
184	#include <linux/spinlock.h>	185	#include <linux/spinlock.h>

fs/Kconfig+1−1

@@ -48,7 +48,7 @@ config FS_DAX · 47 unmodified lines
48	bool "File system based Direct Access (DAX) support"	48	bool "File system based Direct Access (DAX) support"
49	depends on MMU	49	depends on MMU
50	depends on !(ARM \|\| MIPS \|\| SPARC)	50	depends on !(ARM \|\| MIPS \|\| SPARC)
51	select DEV_PAGEMAP_OPS if (ZONE_DEVICE && !FS_DAX_LIMITED)	51	depends on ZONE_DEVICE \|\| FS_DAX_LIMITED
52	select FS_IOMAP	52	select FS_IOMAP
53	select DAX	53	select DAX
54	help	54	help

fs/coda/file.c+1−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	#include <linux/time.h>	14	#include <linux/time.h>
15	#include <linux/file.h>	15	#include <linux/file.h>
16	#include <linux/fs.h>	16	#include <linux/fs.h>
		17	#include <linux/pagemap.h>
17	#include <linux/stat.h>	18	#include <linux/stat.h>
18	#include <linux/cred.h>	19	#include <linux/cred.h>
19	#include <linux/errno.h>	20	#include <linux/errno.h>

fs/fuse/virtio_fs.c+1−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	#include <linux/dax.h>	8	#include <linux/dax.h>
9	#include <linux/pci.h>	9	#include <linux/pci.h>
10	#include <linux/pfn_t.h>	10	#include <linux/pfn_t.h>
		11	#include <linux/memremap.h>
11	#include <linux/module.h>	12	#include <linux/module.h>
12	#include <linux/virtio.h>	13	#include <linux/virtio.h>
13	#include <linux/virtio_fs.h>	14	#include <linux/virtio_fs.h>

fs/iomap/fiemap.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#include <linux/fs.h>	7	#include <linux/fs.h>
8	#include <linux/iomap.h>	8	#include <linux/iomap.h>
9	#include <linux/fiemap.h>	9	#include <linux/fiemap.h>
		10	#include <linux/pagemap.h>
10		11
11	static int iomap_to_fiemap(struct fiemap_extent_info *fi,	12	static int iomap_to_fiemap(struct fiemap_extent_info *fi,
12	const struct iomap *iomap, u32 flags)	13	const struct iomap *iomap, u32 flags)

fs/nfsd/filecache.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#include <linux/hash.h>	7	#include <linux/hash.h>
8	#include <linux/slab.h>	8	#include <linux/slab.h>
9	#include <linux/file.h>	9	#include <linux/file.h>
		10	#include <linux/pagemap.h>
10	#include <linux/sched.h>	11	#include <linux/sched.h>
11	#include <linux/list_lru.h>	12	#include <linux/list_lru.h>
12	#include <linux/fsnotify_backend.h>	13	#include <linux/fsnotify_backend.h>

fs/nfsd/vfs.c+1−0

@@ -26,6 +26,7 @@ · 25 unmodified lines
26	#include <linux/xattr.h>	26	#include <linux/xattr.h>
27	#include <linux/jhash.h>	27	#include <linux/jhash.h>
28	#include <linux/ima.h>	28	#include <linux/ima.h>
		29	#include <linux/pagemap.h>
29	#include <linux/slab.h>	30	#include <linux/slab.h>
30	#include <linux/uaccess.h>	31	#include <linux/uaccess.h>
31	#include <linux/exportfs.h>	32	#include <linux/exportfs.h>

fs/proc/page.c+1−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	#include <linux/proc_fs.h>	10	#include <linux/proc_fs.h>
11	#include <linux/seq_file.h>	11	#include <linux/seq_file.h>
12	#include <linux/hugetlb.h>	12	#include <linux/hugetlb.h>
		13	#include <linux/memremap.h>
13	#include <linux/memcontrol.h>	14	#include <linux/memcontrol.h>
14	#include <linux/mmu_notifier.h>	15	#include <linux/mmu_notifier.h>
15	#include <linux/page_idle.h>	16	#include <linux/page_idle.h>

fs/splice.c+12−12

@@ -46,45 +46,45 @@ · 45 unmodified lines
46	static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,	46	static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
47	struct pipe_buffer *buf)	47	struct pipe_buffer *buf)
48	{	48	{
49	struct page *page = buf->page;	49	struct folio *folio = page_folio(buf->page);
50	struct address_space *mapping;	50	struct address_space *mapping;
51		51
52	lock_page(page);	52	folio_lock(folio);
53		53
54	mapping = page_mapping(page);	54	mapping = folio_mapping(folio);
55	if (mapping) {	55	if (mapping) {
56	WARN_ON(!PageUptodate(page));	56	WARN_ON(!folio_test_uptodate(folio));
57		57
58	/*	58	/*
59	* At least for ext2 with nobh option, we need to wait on	59	* At least for ext2 with nobh option, we need to wait on
60	* writeback completing on this page, since we'll remove it	60	* writeback completing on this folio, since we'll remove it
61	* from the pagecache. Otherwise truncate wont wait on the	61	* from the pagecache. Otherwise truncate wont wait on the
62	* page, allowing the disk blocks to be reused by someone else	62	* folio, allowing the disk blocks to be reused by someone else
63	* before we actually wrote our data to them. fs corruption	63	* before we actually wrote our data to them. fs corruption
64	* ensues.	64	* ensues.
65	*/	65	*/
66	wait_on_page_writeback(page);	66	folio_wait_writeback(folio);
67		67
68	if (page_has_private(page) &&	68	if (folio_has_private(folio) &&
69	!try_to_release_page(page, GFP_KERNEL))	69	!filemap_release_folio(folio, GFP_KERNEL))
70	goto out_unlock;	70	goto out_unlock;
71		71
72	/*	72	/*
73	* If we succeeded in removing the mapping, set LRU flag	73	* If we succeeded in removing the mapping, set LRU flag
74	* and return good.	74	* and return good.
75	*/	75	*/
76	if (remove_mapping(mapping, page)) {	76	if (remove_mapping(mapping, folio)) {
77	buf->flags \|= PIPE_BUF_FLAG_LRU;	77	buf->flags \|= PIPE_BUF_FLAG_LRU;
78	return true;	78	return true;
79	}	79	}
80	}	80	}
81		81
82	/*	82	/*
83	* Raced with truncate or failed to remove page from current	83	* Raced with truncate or failed to remove folio from current
84	* address space, unlock and return failure.	84	* address space, unlock and return failure.
85	*/	85	*/
86	out_unlock:	86	out_unlock:
87	unlock_page(page);	87	folio_unlock(folio);
88	return false;	88	return false;
89	}	89	}
90		90

fs/vboxsf/utils.c+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	#include <linux/namei.h>	9	#include <linux/namei.h>
10	#include <linux/nls.h>	10	#include <linux/nls.h>
11	#include <linux/sizes.h>	11	#include <linux/sizes.h>
		12	#include <linux/pagemap.h>
12	#include <linux/vfs.h>	13	#include <linux/vfs.h>
13	#include "vfsmod.h"	14	#include "vfsmod.h"
14		15

include/linux/fs.h+0−120

@@ -2753,54 +2753,6 @@ extern void init_special_inode(struct inode *, umode_t, dev_t); · 2752 unmodified lines
2753	extern void make_bad_inode(struct inode *);	2753	extern void make_bad_inode(struct inode *);
2754	extern bool is_bad_inode(struct inode *);	2754	extern bool is_bad_inode(struct inode *);
2755		2755
2756	unsigned long invalidate_mapping_pages(struct address_space *mapping,
2757	pgoff_t start, pgoff_t end);
2758
2759	void invalidate_mapping_pagevec(struct address_space *mapping,
2760	pgoff_t start, pgoff_t end,
2761	unsigned long *nr_pagevec);
2762
2763	static inline void invalidate_remote_inode(struct inode *inode)
2764	{
2765	if (S_ISREG(inode->i_mode) \|\| S_ISDIR(inode->i_mode) \|\|
2766	S_ISLNK(inode->i_mode))
2767	invalidate_mapping_pages(inode->i_mapping, 0, -1);
2768	}
2769	extern int invalidate_inode_pages2(struct address_space *mapping);
2770	extern int invalidate_inode_pages2_range(struct address_space *mapping,
2771	pgoff_t start, pgoff_t end);
2772	extern int write_inode_now(struct inode *, int);
2773	extern int filemap_fdatawrite(struct address_space *);
2774	extern int filemap_flush(struct address_space *);
2775	extern int filemap_fdatawait_keep_errors(struct address_space *mapping);
2776	extern int filemap_fdatawait_range(struct address_space *, loff_t lstart,
2777	loff_t lend);
2778	extern int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
2779	loff_t start_byte, loff_t end_byte);
2780
2781	static inline int filemap_fdatawait(struct address_space *mapping)
2782	{
2783	return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
2784	}
2785
2786	extern bool filemap_range_has_page(struct address_space *, loff_t lstart,
2787	loff_t lend);
2788	extern int filemap_write_and_wait_range(struct address_space *mapping,
2789	loff_t lstart, loff_t lend);
2790	extern int __filemap_fdatawrite_range(struct address_space *mapping,
2791	loff_t start, loff_t end, int sync_mode);
2792	extern int filemap_fdatawrite_range(struct address_space *mapping,
2793	loff_t start, loff_t end);
2794	extern int filemap_check_errors(struct address_space *mapping);
2795	extern void __filemap_set_wb_err(struct address_space *mapping, int err);
2796	int filemap_fdatawrite_wbc(struct address_space *mapping,
2797	struct writeback_control *wbc);
2798
2799	static inline int filemap_write_and_wait(struct address_space *mapping)
2800	{
2801	return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
2802	}
2803
2804	extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,	2756	extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,
2805	loff_t lend);	2757	loff_t lend);
2806	extern int __must_check file_check_and_advance_wb_err(struct file *file);	2758	extern int __must_check file_check_and_advance_wb_err(struct file *file);
@@ -2812,67 +2764,6 @@ static inline int file_write_and_wait(struct file *file) · 5 unmodified lines
2812	return file_write_and_wait_range(file, 0, LLONG_MAX);	2764	return file_write_and_wait_range(file, 0, LLONG_MAX);
2813	}	2765	}
2814		2766
2815	/**
2816	* filemap_set_wb_err - set a writeback error on an address_space
2817	* @mapping: mapping in which to set writeback error
2818	* @err: error to be set in mapping
2819	*
2820	* When writeback fails in some way, we must record that error so that
2821	* userspace can be informed when fsync and the like are called. We endeavor
2822	* to report errors on any file that was open at the time of the error. Some
2823	* internal callers also need to know when writeback errors have occurred.
2824	*
2825	* When a writeback error occurs, most filesystems will want to call
2826	* filemap_set_wb_err to record the error in the mapping so that it will be
2827	* automatically reported whenever fsync is called on the file.
2828	*/
2829	static inline void filemap_set_wb_err(struct address_space *mapping, int err)
2830	{
2831	/* Fastpath for common case of no error */
2832	if (unlikely(err))
2833	__filemap_set_wb_err(mapping, err);
2834	}
2835
2836	/**
2837	* filemap_check_wb_err - has an error occurred since the mark was sampled?
2838	* @mapping: mapping to check for writeback errors
2839	* @since: previously-sampled errseq_t
2840	*
2841	* Grab the errseq_t value from the mapping, and see if it has changed "since"
2842	* the given value was sampled.
2843	*
2844	* If it has then report the latest error set, otherwise return 0.
2845	*/
2846	static inline int filemap_check_wb_err(struct address_space *mapping,
2847	errseq_t since)
2848	{
2849	return errseq_check(&mapping->wb_err, since);
2850	}
2851
2852	/**
2853	* filemap_sample_wb_err - sample the current errseq_t to test for later errors
2854	* @mapping: mapping to be sampled
2855	*
2856	* Writeback errors are always reported relative to a particular sample point
2857	* in the past. This function provides those sample points.
2858	*/
2859	static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
2860	{
2861	return errseq_sample(&mapping->wb_err);
2862	}
2863
2864	/**
2865	* file_sample_sb_err - sample the current errseq_t to test for later errors
2866	* @file: file pointer to be sampled
2867	*
2868	* Grab the most current superblock-level errseq_t value for the given
2869	* struct file.
2870	*/
2871	static inline errseq_t file_sample_sb_err(struct file *file)
2872	{
2873	return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
2874	}
2875
2876	extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,	2767	extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,
2877	int datasync);	2768	int datasync);
2878	extern int vfs_fsync(struct file *file, int datasync);	2769	extern int vfs_fsync(struct file *file, int datasync);
@@ -3627,15 +3518,4 @@ extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len, · 748 unmodified lines
3627	extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,	3518	extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
3628	int advice);	3519	int advice);
3629		3520
3630	/*
3631	* Flush file data before changing attributes. Caller must hold any locks
3632	* required to prevent further writes to this file until we're done setting
3633	* flags.
3634	*/
3635	static inline int inode_drain_writes(struct inode *inode)
3636	{
3637	inode_dio_wait(inode);
3638	return filemap_write_and_wait(inode->i_mapping);
3639	}
3640
3641	#endif /* _LINUX_FS_H */	3521	#endif /* _LINUX_FS_H */

include/linux/hmm.h+2−7

 #ifndef LINUX_HMM_H
 #define LINUX_HMM_H
-#include <linux/kconfig.h>
+#include <linux/mm.h>
-#include <linux/pgtable.h>
-#include <linux/device.h>
+struct mmu_interval_notifier;
-#include <linux/migrate.h>
-#include <linux/memremap.h>
-#include <linux/completion.h>
-#include <linux/mmu_notifier.h>
 /*
  * On output:

include/linux/huge_mm.h+9−50

 void free_transhuge_page(struct page *page);
 bool is_transparent_hugepage(struct page *page);
-bool can_split_huge_page(struct page *page, int *pextra_pins);
+bool can_split_folio(struct folio *folio, int *pextra_pins);
 int split_huge_page_to_list(struct page *page, struct list_head *list);
 static inline int split_huge_page(struct page *page)
 {
 void deferred_split_huge_page(struct page *page);
 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
-		unsigned long address, bool freeze, struct page *page);
+		unsigned long address, bool freeze, struct folio *folio);
 #define split_huge_pmd(__vma, __pmd, __address)				\
 	do {								\
 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
-		bool freeze, struct page *page);
+		bool freeze, struct folio *folio);
 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
 		unsigned long address);
 		return NULL;
 }
-/**
- * thp_order - Order of a transparent huge page.
- * @page: Head page of a transparent huge page.
- */
-static inline unsigned int thp_order(struct page *page)
-{
-	VM_BUG_ON_PGFLAGS(PageTail(page), page);
-	if (PageHead(page))
-		return HPAGE_PMD_ORDER;
-	return 0;
-}
-/**
- * thp_nr_pages - The number of regular pages in this huge page.
- * @page: The head page of a huge page.
- */
-static inline int thp_nr_pages(struct page *page)
-{
-	VM_BUG_ON_PGFLAGS(PageTail(page), page);
-	if (PageHead(page))
-		return HPAGE_PMD_NR;
-	return 1;
-}
 /**
  * folio_test_pmd_mappable - Can we map this folio with a PMD?
  * @folio: The folio to test
 #define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
 #define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
-static inline unsigned int thp_order(struct page *page)
-{
-	VM_BUG_ON_PGFLAGS(PageTail(page), page);
-	return 0;
-}
-static inline int thp_nr_pages(struct page *page)
-{
-	VM_BUG_ON_PGFLAGS(PageTail(page), page);
-	return 1;
-}
 static inline bool folio_test_pmd_mappable(struct folio *folio)
 {
 	return false;
 #define thp_get_unmapped_area	NULL
 static inline bool
-can_split_huge_page(struct page *page, int *pextra_pins)
+can_split_folio(struct folio *folio, int *pextra_pins)
 {
 	BUILD_BUG();
 	return false;
 	do { } while (0)
 static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
-		unsigned long address, bool freeze, struct page *page) {}
+		unsigned long address, bool freeze, struct folio *folio) {}
 static inline void split_huge_pmd_address(struct vm_area_struct *vma,
-		unsigned long address, bool freeze, struct page *page) {}
+		unsigned long address, bool freeze, struct folio *folio) {}
 #define split_huge_pud(__vma, __pmd, __address)	\
 	do { } while (0)
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
-/**
+static inline int split_folio_to_list(struct folio *folio,
- * thp_size - Size of a transparent huge page.
+		struct list_head *list)
- * @page: Head page of a transparent huge page.
- *
- * Return: Number of bytes in this page.
- */
-static inline unsigned long thp_size(struct page *page)
 {
-	return PAGE_SIZE << thp_order(page);
+	return split_huge_page_to_list(&folio->page, list);
 }
 #endif /* _LINUX_HUGE_MM_H */

include/linux/hugetlb.h+5−0

 	return NULL;
 }
+static inline struct hstate *size_to_hstate(unsigned long size)
+{
+	return NULL;
+}
 static inline unsigned long huge_page_size(struct hstate *h)
 {
 	return PAGE_SIZE;

include/linux/ksm.h+3−3

@@ -51,7 +51,7 @@ static inline void ksm_exit(struct mm_struct *mm) · 50 unmodified lines
51	struct page ksm_might_need_to_copy(struct page page,	51	struct page ksm_might_need_to_copy(struct page page,
52	struct vm_area_struct *vma, unsigned long address);	52	struct vm_area_struct *vma, unsigned long address);
53		53
54	void rmap_walk_ksm(struct page page, struct rmap_walk_control rwc);	54	void rmap_walk_ksm(struct folio folio, const struct rmap_walk_control rwc);
55	void folio_migrate_ksm(struct folio newfolio, struct folio folio);	55	void folio_migrate_ksm(struct folio newfolio, struct folio folio);
56		56
57	#else /* !CONFIG_KSM */	57	#else /* !CONFIG_KSM */
@@ -78,8 +78,8 @@ static inline struct page ksm_might_need_to_copy(struct page page, · 20 unmodified lines
78	return page;	78	return page;
79	}	79	}
80		80
81	static inline void rmap_walk_ksm(struct page *page,	81	static inline void rmap_walk_ksm(struct folio *folio,
82	struct rmap_walk_control *rwc)	82	const struct rmap_walk_control *rwc)
83	{	83	{
84	}	84	}
85		85

include/linux/memremap.h+24−3

 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef _LINUX_MEMREMAP_H_
 #define _LINUX_MEMREMAP_H_
+#include <linux/mm.h>
 #include <linux/range.h>
 #include <linux/ioport.h>
 #include <linux/percpu-refcount.h>
 struct dev_pagemap_ops {
 	/*
-	 * Called once the page refcount reaches 1.  (ZONE_DEVICE pages never
+	 * Called once the page refcount reaches 0.  The reference count will be
-	 * reach 0 refcount unless there is a refcount bug. This allows the
+	 * reset to one by the core code after the method is called to prepare
-	 * device driver to implement its own memory management.)
+	 * for handing out the page again.
 	 */
 	void (*page_free)(struct page *page);
 	return 1 << pgmap->vmemmap_shift;
 }
+static inline bool is_device_private_page(const struct page *page)
+{
+	return IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
+		is_zone_device_page(page) &&
+		page->pgmap->type == MEMORY_DEVICE_PRIVATE;
+}
+static inline bool folio_is_device_private(const struct folio *folio)
+{
+	return is_device_private_page(&folio->page);
+}
+static inline bool is_pci_p2pdma_page(const struct page *page)
+{
+	return IS_ENABLED(CONFIG_PCI_P2PDMA) &&
+		is_zone_device_page(page) &&
+		page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
+}
 #ifdef CONFIG_ZONE_DEVICE
 void *memremap_pages(struct dev_pagemap *pgmap, int nid);
 void memunmap_pages(struct dev_pagemap *pgmap);

include/linux/mm.h+152−144

@@ -3,9 +3,6 @@ · 2 unmodified lines
3	#define _LINUX_MM_H	3	#define _LINUX_MM_H
4		4
5	#include <linux/errno.h>	5	#include <linux/errno.h>
6
7	#ifdef __KERNEL__
8
9	#include <linux/mmdebug.h>	6	#include <linux/mmdebug.h>
10	#include <linux/gfp.h>	7	#include <linux/gfp.h>
11	#include <linux/bug.h>	8	#include <linux/bug.h>
@@ -26,7 +23,6 @@ · 14 unmodified lines
26	#include <linux/err.h>	23	#include <linux/err.h>
27	#include <linux/page-flags.h>	24	#include <linux/page-flags.h>
28	#include <linux/page_ref.h>	25	#include <linux/page_ref.h>
29	#include <linux/memremap.h>
30	#include <linux/overflow.h>	26	#include <linux/overflow.h>
31	#include <linux/sizes.h>	27	#include <linux/sizes.h>
32	#include <linux/sched.h>	28	#include <linux/sched.h>
@@ -216,8 +212,10 @@ int overcommit_policy_handler(struct ctl_table , int, void , size_t *, · 183 unmodified lines
216		212
217	#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)	213	#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
218	#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))	214	#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
		215	#define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio))
219	#else	216	#else
220	#define nth_page(page,n) ((page) + (n))	217	#define nth_page(page,n) ((page) + (n))
		218	#define folio_page_idx(folio, p) ((p) - &(folio)->page)
221	#endif	219	#endif
222		220
223	/* to align the pointer to the (next) page boundary */	221	/* to align the pointer to the (next) page boundary */
@@ -227,6 +225,10 @@ int overcommit_policy_handler(struct ctl_table , int, void , size_t *, · 3 unmodified lines
227	#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)	225	#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
228		226
229	#define lru_to_page(head) (list_entry((head)->prev, struct page, lru))	227	#define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
		228	static inline struct folio lru_to_folio(struct list_head head)
		229	{
		230	return list_entry((head)->prev, struct folio, lru);
		231	}
230		232
231	void setup_initial_init_mm(void start_code, void end_code,	233	void setup_initial_init_mm(void start_code, void end_code,
232	void end_data, void brk);	234	void end_data, void brk);
@@ -775,21 +777,26 @@ static inline int is_vmalloc_or_module_addr(const void *x) · 542 unmodified lines
775	}	777	}
776	#endif	778	#endif
777		779
778	static inline int head_compound_mapcount(struct page *head)	780	/*
		781	* How many times the entire folio is mapped as a single unit (eg by a
		782	* PMD or PUD entry). This is probably not what you want, except for
		783	* debugging purposes; look at folio_mapcount() or page_mapcount()
		784	* instead.
		785	*/
		786	static inline int folio_entire_mapcount(struct folio *folio)
779	{	787	{
780	return atomic_read(compound_mapcount_ptr(head)) + 1;	788	VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
		789	return atomic_read(folio_mapcount_ptr(folio)) + 1;
781	}	790	}
782		791
783	/*	792	/*
784	* Mapcount of compound page as a whole, does not include mapped sub-pages.	793	* Mapcount of compound page as a whole, does not include mapped sub-pages.
785	*	794	*
786	* Must be called only for compound pages or any their tail sub-pages.	795	* Must be called only for compound pages.
787	*/	796	*/
788	static inline int compound_mapcount(struct page *page)	797	static inline int compound_mapcount(struct page *page)
789	{	798	{
790	VM_BUG_ON_PAGE(!PageCompound(page), page);	799	return folio_entire_mapcount(page_folio(page));
791	page = compound_head(page);
792	return head_compound_mapcount(page);
793	}	800	}
794		801
795	/*	802	/*
@@ -819,8 +826,14 @@ static inline int page_mapcount(struct page *page) · 23 unmodified lines
819	return atomic_read(&page->_mapcount) + 1;	826	return atomic_read(&page->_mapcount) + 1;
820	}	827	}
821		828
		829	int folio_mapcount(struct folio *folio);
		830
822	#ifdef CONFIG_TRANSPARENT_HUGEPAGE	831	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
823	int total_mapcount(struct page *page);	832	static inline int total_mapcount(struct page *page)
		833	{
		834	return folio_mapcount(page_folio(page));
		835	}
		836
824	int page_trans_huge_mapcount(struct page *page);	837	int page_trans_huge_mapcount(struct page *page);
825	#else	838	#else
826	static inline int total_mapcount(struct page *page)	839	static inline int total_mapcount(struct page *page)
@@ -890,33 +903,17 @@ static inline void destroy_compound_page(struct page *page) · 63 unmodified lines
890	compound_page_dtors[page[1].compound_dtor](page);	903	compound_page_dtors[page[1].compound_dtor](page);
891	}	904	}
892		905
893	static inline bool hpage_pincount_available(struct page *page)
894	{
895	/*
896	* Can the page->hpage_pinned_refcount field be used? That field is in
897	* the 3rd page of the compound page, so the smallest (2-page) compound
898	* pages cannot support it.
899	*/
900	page = compound_head(page);
901	return PageCompound(page) && compound_order(page) > 1;
902	}
903
904	static inline int head_compound_pincount(struct page *head)	906	static inline int head_compound_pincount(struct page *head)
905	{	907	{
906	return atomic_read(compound_pincount_ptr(head));	908	return atomic_read(compound_pincount_ptr(head));
907	}	909	}
908		910
909	static inline int compound_pincount(struct page *page)
910	{
911	VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
912	page = compound_head(page);
913	return head_compound_pincount(page);
914	}
915
916	static inline void set_compound_order(struct page *page, unsigned int order)	911	static inline void set_compound_order(struct page *page, unsigned int order)
917	{	912	{
918	page[1].compound_order = order;	913	page[1].compound_order = order;
		914	#ifdef CONFIG_64BIT
919	page[1].compound_nr = 1U << order;	915	page[1].compound_nr = 1U << order;
		916	#endif
920	}	917	}
921		918
922	/* Returns the number of pages in this potentially compound page. */	919	/* Returns the number of pages in this potentially compound page. */
@@ -924,7 +921,11 @@ static inline unsigned long compound_nr(struct page *page) · 1 unmodified line
924	{	921	{
925	if (!PageHead(page))	922	if (!PageHead(page))
926	return 1;	923	return 1;
		924	#ifdef CONFIG_64BIT
927	return page[1].compound_nr;	925	return page[1].compound_nr;
		926	#else
		927	return 1UL << compound_order(page);
		928	#endif
928	}	929	}
929		930
930	/* Returns the number of bytes in this potentially compound page. */	931	/* Returns the number of bytes in this potentially compound page. */
@@ -939,6 +940,37 @@ static inline unsigned int page_shift(struct page *page) · 8 unmodified lines
939	return PAGE_SHIFT + compound_order(page);	940	return PAGE_SHIFT + compound_order(page);
940	}	941	}
941		942
		943	/**
		944	* thp_order - Order of a transparent huge page.
		945	* @page: Head page of a transparent huge page.
		946	*/
		947	static inline unsigned int thp_order(struct page *page)
		948	{
		949	VM_BUG_ON_PGFLAGS(PageTail(page), page);
		950	return compound_order(page);
		951	}
		952
		953	/**
		954	* thp_nr_pages - The number of regular pages in this huge page.
		955	* @page: The head page of a huge page.
		956	*/
		957	static inline int thp_nr_pages(struct page *page)
		958	{
		959	VM_BUG_ON_PGFLAGS(PageTail(page), page);
		960	return compound_nr(page);
		961	}
		962
		963	/**
		964	* thp_size - Size of a transparent huge page.
		965	* @page: Head page of a transparent huge page.
		966	*
		967	* Return: Number of bytes in this page.
		968	*/
		969	static inline unsigned long thp_size(struct page *page)
		970	{
		971	return PAGE_SIZE << thp_order(page);
		972	}
		973
942	void free_compound_page(struct page *page);	974	void free_compound_page(struct page *page);
943		975
944	#ifdef CONFIG_MMU	976	#ifdef CONFIG_MMU
@@ -1090,59 +1122,35 @@ static inline bool is_zone_device_page(const struct page *page) · 145 unmodified lines
1090	}	1122	}
1091	#endif	1123	#endif
1092		1124
		1125	static inline bool folio_is_zone_device(const struct folio *folio)
		1126	{
		1127	return is_zone_device_page(&folio->page);
		1128	}
		1129
1093	static inline bool is_zone_movable_page(const struct page *page)	1130	static inline bool is_zone_movable_page(const struct page *page)
1094	{	1131	{
1095	return page_zonenum(page) == ZONE_MOVABLE;	1132	return page_zonenum(page) == ZONE_MOVABLE;
1096	}	1133	}
1097		1134
1098	#ifdef CONFIG_DEV_PAGEMAP_OPS	1135	#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX)
1099	void free_devmap_managed_page(struct page *page);
1100	DECLARE_STATIC_KEY_FALSE(devmap_managed_key);	1136	DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
1101		1137
1102	static inline bool page_is_devmap_managed(struct page *page)	1138	bool __put_devmap_managed_page(struct page *page);
		1139	static inline bool put_devmap_managed_page(struct page *page)
1103	{	1140	{
1104	if (!static_branch_unlikely(&devmap_managed_key))	1141	if (!static_branch_unlikely(&devmap_managed_key))
1105	return false;	1142	return false;
1106	if (!is_zone_device_page(page))	1143	if (!is_zone_device_page(page))
1107	return false;	1144	return false;
1108	switch (page->pgmap->type) {	1145	return __put_devmap_managed_page(page);
1109	case MEMORY_DEVICE_PRIVATE:
1110	case MEMORY_DEVICE_FS_DAX:
1111	return true;
1112	default:
1113	break;
1114	}
1115	return false;
1116	}	1146	}
1117		1147
1118	void put_devmap_managed_page(struct page *page);	1148	#else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
1119		1149	static inline bool put_devmap_managed_page(struct page *page)
1120	#else /* CONFIG_DEV_PAGEMAP_OPS */
1121	static inline bool page_is_devmap_managed(struct page *page)
1122	{	1150	{
1123	return false;	1151	return false;
1124	}	1152	}
1125		1153	#endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
1126	static inline void put_devmap_managed_page(struct page *page)
1127	{
1128	}
1129	#endif /* CONFIG_DEV_PAGEMAP_OPS */
1130
1131	static inline bool is_device_private_page(const struct page *page)
1132	{
1133	return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
1134	IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
1135	is_zone_device_page(page) &&
1136	page->pgmap->type == MEMORY_DEVICE_PRIVATE;
1137	}
1138
1139	static inline bool is_pci_p2pdma_page(const struct page *page)
1140	{
1141	return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
1142	IS_ENABLED(CONFIG_PCI_P2PDMA) &&
1143	is_zone_device_page(page) &&
1144	page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
1145	}
1146		1154
1147	/* 127: arbitrary random number, small enough to assemble well */	1155	/* 127: arbitrary random number, small enough to assemble well */
1148	#define folio_ref_zero_or_close_to_overflow(folio) \	1156	#define folio_ref_zero_or_close_to_overflow(folio) \
@@ -1168,9 +1176,6 @@ static inline void get_page(struct page *page) · 19 unmodified lines
1168	}	1176	}
1169		1177
1170	bool __must_check try_grab_page(struct page *page, unsigned int flags);	1178	bool __must_check try_grab_page(struct page *page, unsigned int flags);
1171	struct page try_grab_compound_head(struct page page, int refs,
1172	unsigned int flags);
1173
1174		1179
1175	static inline __must_check bool try_get_page(struct page *page)	1180	static inline __must_check bool try_get_page(struct page *page)
1176	{	1181	{
@@ -1225,16 +1230,11 @@ static inline void put_page(struct page *page) · 48 unmodified lines
1225	struct folio *folio = page_folio(page);	1230	struct folio *folio = page_folio(page);
1226		1231
1227	/*	1232	/*
1228	* For devmap managed pages we need to catch refcount transition from	1233	* For some devmap managed pages we need to catch refcount transition
1229	* 2 to 1, when refcount reach one it means the page is free and we	1234	* from 2 to 1:
1230	* need to inform the device driver through callback. See
1231	* include/linux/memremap.h and HMM for details.
1232	*/	1235	*/
1233	if (page_is_devmap_managed(&folio->page)) {	1236	if (put_devmap_managed_page(&folio->page))
1234	put_devmap_managed_page(&folio->page);
1235	return;	1237	return;
1236	}
1237
1238	folio_put(folio);	1238	folio_put(folio);
1239	}	1239	}
1240		1240
@@ -1264,10 +1264,9 @@ static inline void put_page(struct page *page) · 23 unmodified lines
1264	* applications that don't have huge page reference counts, this won't be an	1264	* applications that don't have huge page reference counts, this won't be an
1265	* issue.	1265	* issue.
1266	*	1266	*
1267	* Locking: the lockless algorithm described in page_cache_get_speculative()	1267	* Locking: the lockless algorithm described in folio_try_get_rcu()
1268	* and page_cache_gup_pin_speculative() provides safe operation for	1268	* provides safe operation for get_user_pages(), page_mkclean() and
1269	* get_user_pages and page_mkclean and other calls that race to set up page	1269	* other calls that race to set up page table entries.
1270	* table entries.
1271	*/	1270	*/
1272	#define GUP_PIN_COUNTING_BIAS (1U << 10)	1271	#define GUP_PIN_COUNTING_BIAS (1U << 10)
1273		1272
@@ -1278,70 +1277,11 @@ void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, · 4 unmodified lines
1278	bool make_dirty);	1277	bool make_dirty);
1279	void unpin_user_pages(struct page **pages, unsigned long npages);	1278	void unpin_user_pages(struct page **pages, unsigned long npages);
1280		1279
1281	/**
1282	* page_maybe_dma_pinned - Report if a page is pinned for DMA.
1283	* @page: The page.
1284	*
1285	* This function checks if a page has been pinned via a call to
1286	* a function in the pin_user_pages() family.
1287	*
1288	* For non-huge pages, the return value is partially fuzzy: false is not fuzzy,
1289	* because it means "definitely not pinned for DMA", but true means "probably
1290	* pinned for DMA, but possibly a false positive due to having at least
1291	* GUP_PIN_COUNTING_BIAS worth of normal page references".
1292	*
1293	* False positives are OK, because: a) it's unlikely for a page to get that many
1294	* refcounts, and b) all the callers of this routine are expected to be able to
1295	* deal gracefully with a false positive.
1296	*
1297	* For huge pages, the result will be exactly correct. That's because we have
1298	* more tracking data available: the 3rd struct page in the compound page is
1299	* used to track the pincount (instead using of the GUP_PIN_COUNTING_BIAS
1300	* scheme).
1301	*
1302	* For more information, please see Documentation/core-api/pin_user_pages.rst.
1303	*
1304	* Return: True, if it is likely that the page has been "dma-pinned".
1305	* False, if the page is definitely not dma-pinned.
1306	*/
1307	static inline bool page_maybe_dma_pinned(struct page *page)
1308	{
1309	if (hpage_pincount_available(page))
1310	return compound_pincount(page) > 0;
1311
1312	/*
1313	* page_ref_count() is signed. If that refcount overflows, then
1314	* page_ref_count() returns a negative value, and callers will avoid
1315	* further incrementing the refcount.
1316	*
1317	* Here, for that overflow case, use the signed bit to count a little
1318	* bit higher via unsigned math, and thus still get an accurate result.
1319	*/
1320	return ((unsigned int)page_ref_count(compound_head(page))) >=
1321	GUP_PIN_COUNTING_BIAS;
1322	}
1323
1324	static inline bool is_cow_mapping(vm_flags_t flags)	1280	static inline bool is_cow_mapping(vm_flags_t flags)
1325	{	1281	{
1326	return (flags & (VM_SHARED \| VM_MAYWRITE)) == VM_MAYWRITE;	1282	return (flags & (VM_SHARED \| VM_MAYWRITE)) == VM_MAYWRITE;
1327	}	1283	}
1328		1284
1329	/*
1330	* This should most likely only be called during fork() to see whether we
1331	* should break the cow immediately for a page on the src mm.
1332	*/
1333	static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma,
1334	struct page *page)
1335	{
1336	if (!is_cow_mapping(vma->vm_flags))
1337	return false;
1338
1339	if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
1340	return false;
1341
1342	return page_maybe_dma_pinned(page);
1343	}
1344
1345	#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)	1285	#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
1346	#define SECTION_IN_PAGE_FLAGS	1286	#define SECTION_IN_PAGE_FLAGS
1347	#endif	1287	#endif
@@ -1586,6 +1526,74 @@ static inline unsigned long folio_pfn(struct folio *folio) · 238 unmodified lines
1586	return page_to_pfn(&folio->page);	1526	return page_to_pfn(&folio->page);
1587	}	1527	}
1588		1528
		1529	static inline atomic_t folio_pincount_ptr(struct folio folio)
		1530	{
		1531	return &folio_page(folio, 1)->compound_pincount;
		1532	}
		1533
		1534	/**
		1535	* folio_maybe_dma_pinned - Report if a folio may be pinned for DMA.
		1536	* @folio: The folio.
		1537	*
		1538	* This function checks if a folio has been pinned via a call to
		1539	* a function in the pin_user_pages() family.
		1540	*
		1541	* For small folios, the return value is partially fuzzy: false is not fuzzy,
		1542	* because it means "definitely not pinned for DMA", but true means "probably
		1543	* pinned for DMA, but possibly a false positive due to having at least
		1544	* GUP_PIN_COUNTING_BIAS worth of normal folio references".
		1545	*
		1546	* False positives are OK, because: a) it's unlikely for a folio to
		1547	* get that many refcounts, and b) all the callers of this routine are
		1548	* expected to be able to deal gracefully with a false positive.
		1549	*
		1550	* For large folios, the result will be exactly correct. That's because
		1551	* we have more tracking data available: the compound_pincount is used
		1552	* instead of the GUP_PIN_COUNTING_BIAS scheme.
		1553	*
		1554	* For more information, please see Documentation/core-api/pin_user_pages.rst.
		1555	*
		1556	* Return: True, if it is likely that the page has been "dma-pinned".
		1557	* False, if the page is definitely not dma-pinned.
		1558	*/
		1559	static inline bool folio_maybe_dma_pinned(struct folio *folio)
		1560	{
		1561	if (folio_test_large(folio))
		1562	return atomic_read(folio_pincount_ptr(folio)) > 0;
		1563
		1564	/*
		1565	* folio_ref_count() is signed. If that refcount overflows, then
		1566	* folio_ref_count() returns a negative value, and callers will avoid
		1567	* further incrementing the refcount.
		1568	*
		1569	* Here, for that overflow case, use the sign bit to count a little
		1570	* bit higher via unsigned math, and thus still get an accurate result.
		1571	*/
		1572	return ((unsigned int)folio_ref_count(folio)) >=
		1573	GUP_PIN_COUNTING_BIAS;
		1574	}
		1575
		1576	static inline bool page_maybe_dma_pinned(struct page *page)
		1577	{
		1578	return folio_maybe_dma_pinned(page_folio(page));
		1579	}
		1580
		1581	/*
		1582	* This should most likely only be called during fork() to see whether we
		1583	* should break the cow immediately for a page on the src mm.
		1584	*/
		1585	static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma,
		1586	struct page *page)
		1587	{
		1588	if (!is_cow_mapping(vma->vm_flags))
		1589	return false;
		1590
		1591	if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
		1592	return false;
		1593
		1594	return page_maybe_dma_pinned(page);
		1595	}
		1596
1589	/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */	1597	/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */
1590	#ifdef CONFIG_MIGRATION	1598	#ifdef CONFIG_MIGRATION
1591	static inline bool is_pinnable_page(struct page *page)	1599	static inline bool is_pinnable_page(struct page *page)
@@ -1600,6 +1608,11 @@ static inline bool is_pinnable_page(struct page *page) · 8 unmodified lines
1600	}	1608	}
1601	#endif	1609	#endif
1602		1610
		1611	static inline bool folio_is_pinnable(struct folio *folio)
		1612	{
		1613	return is_pinnable_page(&folio->page);
		1614	}
		1615
1603	static inline void set_page_zone(struct page *page, enum zone_type zone)	1616	static inline void set_page_zone(struct page *page, enum zone_type zone)
1604	{	1617	{
1605	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);	1618	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
@@ -1749,7 +1762,6 @@ static inline void folio_address(const struct folio folio) · 143 unmodified lines
1749	}	1762	}
1750		1763
1751	extern void page_rmapping(struct page page);	1764	extern void page_rmapping(struct page page);
1752	extern struct anon_vma page_anon_vma(struct page page);
1753	extern pgoff_t __page_file_index(struct page *page);	1765	extern pgoff_t __page_file_index(struct page *page);
1754		1766
1755	/*	1767	/*
@@ -1855,7 +1867,6 @@ extern void truncate_setsize(struct inode *inode, loff_t newsize); · 99 unmodified lines
1855	void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);	1867	void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
1856	void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);	1868	void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1857	int generic_error_remove_page(struct address_space mapping, struct page page);	1869	int generic_error_remove_page(struct address_space mapping, struct page page);
1858	int invalidate_inode_page(struct page *page);
1859		1870
1860	#ifdef CONFIG_MMU	1871	#ifdef CONFIG_MMU
1861	extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,	1872	extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
@@ -2921,13 +2932,11 @@ struct page follow_page(struct vm_area_struct vma, unsigned long address, · 1059 unmodified lines
2921	#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */	2932	#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
2922	#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO	2933	#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
2923	* and return without waiting upon it */	2934	* and return without waiting upon it */
2924	#define FOLL_POPULATE 0x40 /* fault in pages (with FOLL_MLOCK) */
2925	#define FOLL_NOFAULT 0x80 /* do not fault in pages */	2935	#define FOLL_NOFAULT 0x80 /* do not fault in pages */
2926	#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */	2936	#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2927	#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */	2937	#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2928	#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */	2938	#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2929	#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */	2939	#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
2930	#define FOLL_MLOCK 0x1000 /* lock present pages */
2931	#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */	2940	#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
2932	#define FOLL_COW 0x4000 /* internal GUP flag */	2941	#define FOLL_COW 0x4000 /* internal GUP flag */
2933	#define FOLL_ANON 0x8000 /* don't do file mappings */	2942	#define FOLL_ANON 0x8000 /* don't do file mappings */
@@ -3381,5 +3390,4 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start, · 447 unmodified lines
3381	}	3390	}
3382	#endif	3391	#endif
3383		3392
3384	#endif /* __KERNEL__ */
3385	#endif /* _LINUX_MM_H */	3393	#endif /* _LINUX_MM_H */

include/linux/mm_inline.h+8−3

@@ -99,7 +99,8 @@ void lruvec_add_folio(struct lruvec lruvec, struct folio folio) · 98 unmodified lines
99		99
100	update_lru_size(lruvec, lru, folio_zonenum(folio),	100	update_lru_size(lruvec, lru, folio_zonenum(folio),
101	folio_nr_pages(folio));	101	folio_nr_pages(folio));
102	list_add(&folio->lru, &lruvec->lists[lru]);	102	if (lru != LRU_UNEVICTABLE)
		103	list_add(&folio->lru, &lruvec->lists[lru]);
103	}	104	}
104		105
105	static __always_inline void add_page_to_lru_list(struct page *page,	106	static __always_inline void add_page_to_lru_list(struct page *page,
@@ -115,6 +116,7 @@ void lruvec_add_folio_tail(struct lruvec lruvec, struct folio folio) · 9 unmodified lines
115		116
116	update_lru_size(lruvec, lru, folio_zonenum(folio),	117	update_lru_size(lruvec, lru, folio_zonenum(folio),
117	folio_nr_pages(folio));	118	folio_nr_pages(folio));
		119	/* This is not expected to be used on LRU_UNEVICTABLE */
118	list_add_tail(&folio->lru, &lruvec->lists[lru]);	120	list_add_tail(&folio->lru, &lruvec->lists[lru]);
119	}	121	}
120		122
@@ -127,8 +129,11 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page, · 6 unmodified lines
127	static __always_inline	129	static __always_inline
128	void lruvec_del_folio(struct lruvec lruvec, struct folio folio)	130	void lruvec_del_folio(struct lruvec lruvec, struct folio folio)
129	{	131	{
130	list_del(&folio->lru);	132	enum lru_list lru = folio_lru_list(folio);
131	update_lru_size(lruvec, folio_lru_list(folio), folio_zonenum(folio),	133
		134	if (lru != LRU_UNEVICTABLE)
		135	list_del(&folio->lru);
		136	update_lru_size(lruvec, lru, folio_zonenum(folio),
132	-folio_nr_pages(folio));	137	-folio_nr_pages(folio));
133	}	138	}
134		139

include/linux/mm_types.h+22−4

 			 * lruvec->lru_lock.  Sometimes used as a generic list
 			 * by the page owner.
 			 */
-			struct list_head lru;
+			union {
+				struct list_head lru;
+				/* Or, for the Unevictable "LRU list" slot */
+				struct {
+					/* Always even, to negate PageTail */
+					void *__filler;
+					/* Count page's or folio's mlocks */
+					unsigned int mlock_count;
+				};
+			};
 			/* See page-flags.h for PAGE_MAPPING_FLAGS */
 			struct address_space *mapping;
 			pgoff_t index;		/* Our offset within mapping. */
 			unsigned char compound_dtor;
 			unsigned char compound_order;
 			atomic_t compound_mapcount;
+			atomic_t compound_pincount;
+#ifdef CONFIG_64BIT
 			unsigned int compound_nr; /* 1 << compound_order */
+#endif
 		};
 		struct {	/* Second tail page of compound page */
 			unsigned long _compound_pad_1;	/* compound_head */
-			atomic_t hpage_pinned_refcount;
+			unsigned long _compound_pad_2;
 			/* For both global and memcg */
 			struct list_head deferred_list;
 		};
 		struct {
 	/* public: */
 			unsigned long flags;
-			struct list_head lru;
+			union {
+				struct list_head lru;
+				struct {
+					void *__filler;
+					unsigned int mlock_count;
+				};
+			};
 			struct address_space *mapping;
 			pgoff_t index;
 			void *private;
 static inline atomic_t *compound_pincount_ptr(struct page *page)
 {
-	return &page[2].hpage_pinned_refcount;
+	return &page[1].compound_pincount;
 }
 /*

include/linux/pagemap.h+133−13

@@ -18,6 +18,120 @@ · 17 unmodified lines
18		18
19	struct folio_batch;	19	struct folio_batch;
20		20
		21	unsigned long invalidate_mapping_pages(struct address_space *mapping,
		22	pgoff_t start, pgoff_t end);
		23
		24	static inline void invalidate_remote_inode(struct inode *inode)
		25	{
		26	if (S_ISREG(inode->i_mode) \|\| S_ISDIR(inode->i_mode) \|\|
		27	S_ISLNK(inode->i_mode))
		28	invalidate_mapping_pages(inode->i_mapping, 0, -1);
		29	}
		30	int invalidate_inode_pages2(struct address_space *mapping);
		31	int invalidate_inode_pages2_range(struct address_space *mapping,
		32	pgoff_t start, pgoff_t end);
		33	int write_inode_now(struct inode *, int sync);
		34	int filemap_fdatawrite(struct address_space *);
		35	int filemap_flush(struct address_space *);
		36	int filemap_fdatawait_keep_errors(struct address_space *mapping);
		37	int filemap_fdatawait_range(struct address_space *, loff_t lstart, loff_t lend);
		38	int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
		39	loff_t start_byte, loff_t end_byte);
		40
		41	static inline int filemap_fdatawait(struct address_space *mapping)
		42	{
		43	return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
		44	}
		45
		46	bool filemap_range_has_page(struct address_space *, loff_t lstart, loff_t lend);
		47	int filemap_write_and_wait_range(struct address_space *mapping,
		48	loff_t lstart, loff_t lend);
		49	int __filemap_fdatawrite_range(struct address_space *mapping,
		50	loff_t start, loff_t end, int sync_mode);
		51	int filemap_fdatawrite_range(struct address_space *mapping,
		52	loff_t start, loff_t end);
		53	int filemap_check_errors(struct address_space *mapping);
		54	void __filemap_set_wb_err(struct address_space *mapping, int err);
		55	int filemap_fdatawrite_wbc(struct address_space *mapping,
		56	struct writeback_control *wbc);
		57
		58	static inline int filemap_write_and_wait(struct address_space *mapping)
		59	{
		60	return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
		61	}
		62
		63	/**
		64	* filemap_set_wb_err - set a writeback error on an address_space
		65	* @mapping: mapping in which to set writeback error
		66	* @err: error to be set in mapping
		67	*
		68	* When writeback fails in some way, we must record that error so that
		69	* userspace can be informed when fsync and the like are called. We endeavor
		70	* to report errors on any file that was open at the time of the error. Some
		71	* internal callers also need to know when writeback errors have occurred.
		72	*
		73	* When a writeback error occurs, most filesystems will want to call
		74	* filemap_set_wb_err to record the error in the mapping so that it will be
		75	* automatically reported whenever fsync is called on the file.
		76	*/
		77	static inline void filemap_set_wb_err(struct address_space *mapping, int err)
		78	{
		79	/* Fastpath for common case of no error */
		80	if (unlikely(err))
		81	__filemap_set_wb_err(mapping, err);
		82	}
		83
		84	/**
		85	* filemap_check_wb_err - has an error occurred since the mark was sampled?
		86	* @mapping: mapping to check for writeback errors
		87	* @since: previously-sampled errseq_t
		88	*
		89	* Grab the errseq_t value from the mapping, and see if it has changed "since"
		90	* the given value was sampled.
		91	*
		92	* If it has then report the latest error set, otherwise return 0.
		93	*/
		94	static inline int filemap_check_wb_err(struct address_space *mapping,
		95	errseq_t since)
		96	{
		97	return errseq_check(&mapping->wb_err, since);
		98	}
		99
		100	/**
		101	* filemap_sample_wb_err - sample the current errseq_t to test for later errors
		102	* @mapping: mapping to be sampled
		103	*
		104	* Writeback errors are always reported relative to a particular sample point
		105	* in the past. This function provides those sample points.
		106	*/
		107	static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
		108	{
		109	return errseq_sample(&mapping->wb_err);
		110	}
		111
		112	/**
		113	* file_sample_sb_err - sample the current errseq_t to test for later errors
		114	* @file: file pointer to be sampled
		115	*
		116	* Grab the most current superblock-level errseq_t value for the given
		117	* struct file.
		118	*/
		119	static inline errseq_t file_sample_sb_err(struct file *file)
		120	{
		121	return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
		122	}
		123
		124	/*
		125	* Flush file data before changing attributes. Caller must hold any locks
		126	* required to prevent further writes to this file until we're done setting
		127	* flags.
		128	*/
		129	static inline int inode_drain_writes(struct inode *inode)
		130	{
		131	inode_dio_wait(inode);
		132	return filemap_write_and_wait(inode->i_mapping);
		133	}
		134
21	static inline bool mapping_empty(struct address_space *mapping)	135	static inline bool mapping_empty(struct address_space *mapping)
22	{	136	{
23	return xa_empty(&mapping->i_pages);	137	return xa_empty(&mapping->i_pages);
@@ -192,9 +306,14 @@ static inline void mapping_set_large_folios(struct address_space *mapping) · 168 unmodified lines
192	__set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);	306	__set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
193	}	307	}
194		308
		309	/*
		310	* Large folio support currently depends on THP. These dependencies are
		311	* being worked on but are not yet fixed.
		312	*/
195	static inline bool mapping_large_folio_support(struct address_space *mapping)	313	static inline bool mapping_large_folio_support(struct address_space *mapping)
196	{	314	{
197	return test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);	315	return IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
		316	test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
198	}	317	}
199		318
200	static inline int filemap_nr_thps(struct address_space *mapping)	319	static inline int filemap_nr_thps(struct address_space *mapping)
@@ -212,7 +331,7 @@ static inline void filemap_nr_thps_inc(struct address_space *mapping) · 11 unmodified lines
212	if (!mapping_large_folio_support(mapping))	331	if (!mapping_large_folio_support(mapping))
213	atomic_inc(&mapping->nr_thps);	332	atomic_inc(&mapping->nr_thps);
214	#else	333	#else
215	WARN_ON_ONCE(1);	334	WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
216	#endif	335	#endif
217	}	336	}
218		337
@@ -222,7 +341,7 @@ static inline void filemap_nr_thps_dec(struct address_space *mapping) · 3 unmodified lines
222	if (!mapping_large_folio_support(mapping))	341	if (!mapping_large_folio_support(mapping))
223	atomic_dec(&mapping->nr_thps);	342	atomic_dec(&mapping->nr_thps);
224	#else	343	#else
225	WARN_ON_ONCE(1);	344	WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
226	#endif	345	#endif
227	}	346	}
228		347
@@ -283,16 +402,6 @@ static inline struct inode folio_inode(struct folio folio) · 54 unmodified lines
283	return folio->mapping->host;	402	return folio->mapping->host;
284	}	403	}
285		404
286	static inline bool page_cache_add_speculative(struct page *page, int count)
287	{
288	return folio_ref_try_add_rcu((struct folio *)page, count);
289	}
290
291	static inline bool page_cache_get_speculative(struct page *page)
292	{
293	return page_cache_add_speculative(page, 1);
294	}
295
296	/**	405	/**
297	* folio_attach_private - Attach private data to a folio.	406	* folio_attach_private - Attach private data to a folio.
298	* @folio: Folio to attach data to.	407	* @folio: Folio to attach data to.
@@ -706,6 +815,17 @@ static inline loff_t folio_file_pos(struct folio *folio) · 407 unmodified lines
706	return page_file_offset(&folio->page);	815	return page_file_offset(&folio->page);
707	}	816	}
708		817
		818	/*
		819	* Get the offset in PAGE_SIZE (even for hugetlb folios).
		820	* (TODO: hugetlb folios should have ->index in PAGE_SIZE)
		821	*/
		822	static inline pgoff_t folio_pgoff(struct folio *folio)
		823	{
		824	if (unlikely(folio_test_hugetlb(folio)))
		825	return hugetlb_basepage_index(&folio->page);
		826	return folio->index;
		827	}
		828
709	extern pgoff_t linear_hugepage_index(struct vm_area_struct *vma,	829	extern pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
710	unsigned long address);	830	unsigned long address);
711		831

include/linux/rmap.h+47−29

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	#include <linux/rwsem.h>	11	#include <linux/rwsem.h>
12	#include <linux/memcontrol.h>	12	#include <linux/memcontrol.h>
13	#include <linux/highmem.h>	13	#include <linux/highmem.h>
		14	#include <linux/pagemap.h>
14		15
15	/*	16	/*
16	* The anon_vma heads a list of private "related" vmas, to scan if	17	* The anon_vma heads a list of private "related" vmas, to scan if
@@ -167,18 +168,19 @@ struct anon_vma page_get_anon_vma(struct page page); · 150 unmodified lines
167	*/	168	*/
168	void page_move_anon_rmap(struct page , struct vm_area_struct );	169	void page_move_anon_rmap(struct page , struct vm_area_struct );
169	void page_add_anon_rmap(struct page , struct vm_area_struct ,	170	void page_add_anon_rmap(struct page , struct vm_area_struct ,
170	unsigned long, bool);	171	unsigned long address, bool compound);
171	void do_page_add_anon_rmap(struct page , struct vm_area_struct ,	172	void do_page_add_anon_rmap(struct page , struct vm_area_struct ,
172	unsigned long, int);	173	unsigned long address, int flags);
173	void page_add_new_anon_rmap(struct page , struct vm_area_struct ,	174	void page_add_new_anon_rmap(struct page , struct vm_area_struct ,
174	unsigned long, bool);	175	unsigned long address, bool compound);
175	void page_add_file_rmap(struct page *, bool);	176	void page_add_file_rmap(struct page , struct vm_area_struct ,
176	void page_remove_rmap(struct page *, bool);	177	bool compound);
177		178	void page_remove_rmap(struct page , struct vm_area_struct ,
		179	bool compound);
178	void hugepage_add_anon_rmap(struct page , struct vm_area_struct ,	180	void hugepage_add_anon_rmap(struct page , struct vm_area_struct ,
179	unsigned long);	181	unsigned long address);
180	void hugepage_add_new_anon_rmap(struct page , struct vm_area_struct ,	182	void hugepage_add_new_anon_rmap(struct page , struct vm_area_struct ,
181	unsigned long);	183	unsigned long address);
182		184
183	static inline void page_dup_rmap(struct page *page, bool compound)	185	static inline void page_dup_rmap(struct page *page, bool compound)
184	{	186	{
@@ -188,11 +190,11 @@ static inline void page_dup_rmap(struct page *page, bool compound) · 3 unmodified lines
188	/*	190	/*
189	* Called from mm/vmscan.c to handle paging out	191	* Called from mm/vmscan.c to handle paging out
190	*/	192	*/
191	int page_referenced(struct page *, int is_locked,	193	int folio_referenced(struct folio *, int is_locked,
192	struct mem_cgroup memcg, unsigned long vm_flags);	194	struct mem_cgroup memcg, unsigned long vm_flags);
193		195
194	void try_to_migrate(struct page *page, enum ttu_flags flags);	196	void try_to_migrate(struct folio *folio, enum ttu_flags flags);
195	void try_to_unmap(struct page *, enum ttu_flags flags);	197	void try_to_unmap(struct folio *, enum ttu_flags flags);
196		198
197	int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,	199	int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
198	unsigned long end, struct page **pages,	200	unsigned long end, struct page **pages,
@@ -200,11 +202,13 @@ int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, · 1 unmodified line
200		202
201	/* Avoid racy checks */	203	/* Avoid racy checks */
202	#define PVMW_SYNC (1 << 0)	204	#define PVMW_SYNC (1 << 0)
203	/* Look for migarion entries rather than present PTEs */	205	/* Look for migration entries rather than present PTEs */
204	#define PVMW_MIGRATION (1 << 1)	206	#define PVMW_MIGRATION (1 << 1)
205		207
206	struct page_vma_mapped_walk {	208	struct page_vma_mapped_walk {
207	struct page *page;	209	unsigned long pfn;
		210	unsigned long nr_pages;
		211	pgoff_t pgoff;
208	struct vm_area_struct *vma;	212	struct vm_area_struct *vma;
209	unsigned long address;	213	unsigned long address;
210	pmd_t *pmd;	214	pmd_t *pmd;
@@ -213,10 +217,30 @@ struct page_vma_mapped_walk { · 2 unmodified lines
213	unsigned int flags;	217	unsigned int flags;
214	};	218	};
215		219
		220	#define DEFINE_PAGE_VMA_WALK(name, _page, _vma, _address, _flags) \
		221	struct page_vma_mapped_walk name = { \
		222	.pfn = page_to_pfn(_page), \
		223	.nr_pages = compound_nr(page), \
		224	.pgoff = page_to_pgoff(page), \
		225	.vma = _vma, \
		226	.address = _address, \
		227	.flags = _flags, \
		228	}
		229
		230	#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \
		231	struct page_vma_mapped_walk name = { \
		232	.pfn = folio_pfn(_folio), \
		233	.nr_pages = folio_nr_pages(_folio), \
		234	.pgoff = folio_pgoff(_folio), \
		235	.vma = _vma, \
		236	.address = _address, \
		237	.flags = _flags, \
		238	}
		239
216	static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)	240	static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
217	{	241	{
218	/* HugeTLB pte is set to the relevant page table entry without pte_mapped. */	242	/* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
219	if (pvmw->pte && !PageHuge(pvmw->page))	243	if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma))
220	pte_unmap(pvmw->pte);	244	pte_unmap(pvmw->pte);
221	if (pvmw->ptl)	245	if (pvmw->ptl)
222	spin_unlock(pvmw->ptl);	246	spin_unlock(pvmw->ptl);
@@ -237,18 +261,12 @@ unsigned long page_address_in_vma(struct page , struct vm_area_struct ); · 14 unmodified lines
237	*/	261	*/
238	int folio_mkclean(struct folio *);	262	int folio_mkclean(struct folio *);
239		263
240	/*	264	void remove_migration_ptes(struct folio src, struct folio dst, bool locked);
241	* called in munlock()/munmap() path to check for other vmas holding
242	* the page mlocked.
243	*/
244	void page_mlock(struct page *page);
245
246	void remove_migration_ptes(struct page old, struct page new, bool locked);
247		265
248	/*	266	/*
249	* Called by memory-failure.c to kill processes.	267	* Called by memory-failure.c to kill processes.
250	*/	268	*/
251	struct anon_vma page_lock_anon_vma_read(struct page page);	269	struct anon_vma folio_lock_anon_vma_read(struct folio folio);
252	void page_unlock_anon_vma_read(struct anon_vma *anon_vma);	270	void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
253	int page_mapped_in_vma(struct page page, struct vm_area_struct vma);	271	int page_mapped_in_vma(struct page page, struct vm_area_struct vma);
254		272
@@ -267,15 +285,15 @@ struct rmap_walk_control { · 12 unmodified lines
267	* Return false if page table scanning in rmap_walk should be stopped.	285	* Return false if page table scanning in rmap_walk should be stopped.
268	* Otherwise, return true.	286	* Otherwise, return true.
269	*/	287	*/
270	bool (rmap_one)(struct page page, struct vm_area_struct *vma,	288	bool (rmap_one)(struct folio folio, struct vm_area_struct *vma,
271	unsigned long addr, void *arg);	289	unsigned long addr, void *arg);
272	int (done)(struct page page);	290	int (done)(struct folio folio);
273	struct anon_vma (anon_lock)(struct page *page);	291	struct anon_vma (anon_lock)(struct folio *folio);
274	bool (invalid_vma)(struct vm_area_struct vma, void *arg);	292	bool (invalid_vma)(struct vm_area_struct vma, void *arg);
275	};	293	};
276		294
277	void rmap_walk(struct page page, struct rmap_walk_control rwc);	295	void rmap_walk(struct folio folio, const struct rmap_walk_control rwc);
278	void rmap_walk_locked(struct page page, struct rmap_walk_control rwc);	296	void rmap_walk_locked(struct folio folio, const struct rmap_walk_control rwc);
279		297
280	#else /* !CONFIG_MMU */	298	#else /* !CONFIG_MMU */
281		299
@@ -283,7 +301,7 @@ void rmap_walk_locked(struct page page, struct rmap_walk_control rwc); · 1 unmodified line
283	#define anon_vma_prepare(vma) (0)	301	#define anon_vma_prepare(vma) (0)
284	#define anon_vma_link(vma) do {} while (0)	302	#define anon_vma_link(vma) do {} while (0)
285		303
286	static inline int page_referenced(struct page *page, int is_locked,	304	static inline int folio_referenced(struct folio *folio, int is_locked,
287	struct mem_cgroup *memcg,	305	struct mem_cgroup *memcg,
288	unsigned long *vm_flags)	306	unsigned long *vm_flags)
289	{	307	{
@@ -291,7 +309,7 @@ static inline int page_referenced(struct page *page, int is_locked, · 1 unmodified line
291	return 0;	309	return 0;
292	}	310	}
293		311
294	static inline void try_to_unmap(struct page *page, enum ttu_flags flags)	312	static inline void try_to_unmap(struct folio *folio, enum ttu_flags flags)
295	{	313	{
296	}	314	}
297		315

include/linux/swap.h+4−5

 /* linux/mm/workingset.c */
 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
-void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg);
+void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg);
 void workingset_refault(struct folio *folio, void *shadow);
 void workingset_activation(struct folio *folio);
 extern void lru_add_drain_cpu(int cpu);
 extern void lru_add_drain_cpu_zone(struct zone *zone);
 extern void lru_add_drain_all(void);
-extern void deactivate_file_page(struct page *page);
 extern void deactivate_page(struct page *page);
 extern void mark_page_lazyfree(struct page *page);
 extern void swap_setup(void);
 						unsigned long *nr_scanned);
 extern unsigned long shrink_all_memory(unsigned long nr_pages);
 extern int vm_swappiness;
-extern int remove_mapping(struct address_space *mapping, struct page *page);
+long remove_mapping(struct address_space *mapping, struct folio *folio);
 extern unsigned long reclaim_pages(struct list_head *page_list);
 #ifdef CONFIG_NUMA
 #endif
 #ifdef CONFIG_MEMCG_SWAP
-extern void mem_cgroup_swapout(struct page *page, swp_entry_t entry);
+void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry);
 extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
 static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
 {
 extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
 extern bool mem_cgroup_swap_full(struct page *page);
 #else
-static inline void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
+static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
 {
 }

include/trace/events/vmscan.h+5−5

@@ -327,11 +327,11 @@ TRACE_EVENT(mm_vmscan_lru_isolate, · 326 unmodified lines
327	__print_symbolic(__entry->lru, LRU_NAMES))	327	__print_symbolic(__entry->lru, LRU_NAMES))
328	);	328	);
329		329
330	TRACE_EVENT(mm_vmscan_writepage,	330	TRACE_EVENT(mm_vmscan_write_folio,
331		331
332	TP_PROTO(struct page *page),	332	TP_PROTO(struct folio *folio),
333		333
334	TP_ARGS(page),	334	TP_ARGS(folio),
335		335
336	TP_STRUCT__entry(	336	TP_STRUCT__entry(
337	__field(unsigned long, pfn)	337	__field(unsigned long, pfn)
@@ -339,9 +339,9 @@ TRACE_EVENT(mm_vmscan_writepage, · 1 unmodified line
339	),	339	),
340		340
341	TP_fast_assign(	341	TP_fast_assign(
342	__entry->pfn = page_to_pfn(page);	342	__entry->pfn = folio_pfn(folio);
343	__entry->reclaim_flags = trace_reclaim_flags(	343	__entry->reclaim_flags = trace_reclaim_flags(
344	page_is_file_lru(page));	344	folio_is_file_lru(folio));
345	),	345	),
346		346
347	TP_printk("page=%p pfn=0x%lx flags=%s",	347	TP_printk("page=%p pfn=0x%lx flags=%s",

kernel/events/uprobes.c+3−10

 				struct page *old_page, struct page *new_page)
 {
 	struct mm_struct *mm = vma->vm_mm;
-	struct page_vma_mapped_walk pvmw = {
+	DEFINE_FOLIO_VMA_WALK(pvmw, page_folio(old_page), vma, addr, 0);
-		.page = compound_head(old_page),
-		.vma = vma,
-		.address = addr,
-	};
 	int err;
 	struct mmu_notifier_range range;
 			return err;
 	}
-	/* For try_to_free_swap() and munlock_vma_page() below */
+	/* For try_to_free_swap() below */
 	lock_page(old_page);
 	mmu_notifier_invalidate_range_start(&range);
 		set_pte_at_notify(mm, addr, pvmw.pte,
 				  mk_pte(new_page, vma->vm_page_prot));
-	page_remove_rmap(old_page, false);
+	page_remove_rmap(old_page, vma, false);
 	if (!page_mapped(old_page))
 		try_to_free_swap(old_page);
 	page_vma_mapped_walk_done(&pvmw);
-	if ((vma->vm_flags & VM_LOCKED) && !PageCompound(old_page))
-		munlock_vma_page(old_page);
 	put_page(old_page);
 	err = 0;

kernel/futex/core.c+1−1

@@ -302,7 +302,7 @@ again: · 301 unmodified lines
302	* found it, but truncated or holepunched or subjected to	302	* found it, but truncated or holepunched or subjected to
303	* invalidate_complete_page2 before we got the page lock (also	303	* invalidate_complete_page2 before we got the page lock (also
304	* cases which we are happy to fail). And we hold a reference,	304	* cases which we are happy to fail). And we hold a reference,
305	* so refcount care in invalidate_complete_page's remove_mapping	305	* so refcount care in invalidate_inode_page's remove_mapping
306	* prevents drop_caches from setting mapping to NULL beneath us.	306	* prevents drop_caches from setting mapping to NULL beneath us.
307	*	307	*
308	* The case we do have to guard against is when memory pressure made	308	* The case we do have to guard against is when memory pressure made

lib/test_hmm.c+3−1

 #include <linux/kernel.h>
 #include <linux/cdev.h>
 #include <linux/device.h>
+#include <linux/memremap.h>
 #include <linux/mutex.h>
 #include <linux/rwsem.h>
 #include <linux/sched.h>
 #include <linux/sched/mm.h>
 #include <linux/platform_device.h>
 #include <linux/rmap.h>
+#include <linux/mmu_notifier.h>
+#include <linux/migrate.h>
 #include "test_hmm_uapi.h"
 	}
 	dpage->zone_device_data = rpage;
-	get_page(dpage);
 	lock_page(dpage);
 	return dpage;

mm/Kconfig+3−4

 	  pages as migration can relocate pages to satisfy a huge page
 	  allocation instead of reclaiming.
+config DEVICE_MIGRATION
+	def_bool MIGRATION && ZONE_DEVICE
 config ARCH_ENABLE_HUGEPAGE_MIGRATION
 	bool
 	  If FS_DAX is enabled, then say Y.
-config DEV_PAGEMAP_OPS
-	bool
 #
 # Helpers to mirror range of the CPU page tables of a process into device page
 # tables.
 config DEVICE_PRIVATE
 	bool "Unaddressable device memory (GPU memory, ...)"
 	depends on ZONE_DEVICE
-	select DEV_PAGEMAP_OPS
 	help
 	  Allows creation of struct pages to represent unaddressable device

mm/Makefile+1−0

@@ -92,6 +92,7 @@ obj-$(CONFIG_KFENCE) += kfence/ · 91 unmodified lines
92	obj-$(CONFIG_FAILSLAB) += failslab.o	92	obj-$(CONFIG_FAILSLAB) += failslab.o
93	obj-$(CONFIG_MEMTEST) += memtest.o	93	obj-$(CONFIG_MEMTEST) += memtest.o
94	obj-$(CONFIG_MIGRATION) += migrate.o	94	obj-$(CONFIG_MIGRATION) += migrate.o
		95	obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o
95	obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o	96	obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o
96	obj-$(CONFIG_PAGE_COUNTER) += page_counter.o	97	obj-$(CONFIG_PAGE_COUNTER) += page_counter.o
97	obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o	98	obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o

mm/damon/paddr.c+29−33

 #include "../internal.h"
 #include "ops-common.h"
-static bool __damon_pa_mkold(struct page *page, struct vm_area_struct *vma,
+static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma,
 		unsigned long addr, void *arg)
 {
-	struct page_vma_mapped_walk pvmw = {
+	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
-		.page = page,
-		.vma = vma,
-		.address = addr,
-	};
 	while (page_vma_mapped_walk(&pvmw)) {
 		addr = pvmw.address;
 static void damon_pa_mkold(unsigned long paddr)
 {
+	struct folio *folio;
 	struct page *page = damon_get_page(PHYS_PFN(paddr));
 	struct rmap_walk_control rwc = {
 		.rmap_one = __damon_pa_mkold,
-		.anon_lock = page_lock_anon_vma_read,
+		.anon_lock = folio_lock_anon_vma_read,
 	};
 	bool need_lock;
 	if (!page)
 		return;
+	folio = page_folio(page);
-	if (!page_mapped(page) || !page_rmapping(page)) {
+	if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
-		set_page_idle(page);
+		folio_set_idle(folio);
 		goto out;
 	}
-	need_lock = !PageAnon(page) || PageKsm(page);
+	need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
-	if (need_lock && !trylock_page(page))
+	if (need_lock && !folio_trylock(folio))
 		goto out;
-	rmap_walk(page, &rwc);
+	rmap_walk(folio, &rwc);
 	if (need_lock)
-		unlock_page(page);
+		folio_unlock(folio);
 out:
-	put_page(page);
+	folio_put(folio);
 }
 static void __damon_pa_prepare_access_check(struct damon_ctx *ctx,
 	bool accessed;
 };
-static bool __damon_pa_young(struct page *page, struct vm_area_struct *vma,
+static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma,
 		unsigned long addr, void *arg)
 {
 	struct damon_pa_access_chk_result *result = arg;
-	struct page_vma_mapped_walk pvmw = {
+	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
-		.page = page,
-		.vma = vma,
-		.address = addr,
-	};
 	result->accessed = false;
 	result->page_sz = PAGE_SIZE;
 		addr = pvmw.address;
 		if (pvmw.pte) {
 			result->accessed = pte_young(*pvmw.pte) ||
-				!page_is_idle(page) ||
+				!folio_test_idle(folio) ||
 				mmu_notifier_test_young(vma->vm_mm, addr);
 		} else {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 			result->accessed = pmd_young(*pvmw.pmd) ||
-				!page_is_idle(page) ||
+				!folio_test_idle(folio) ||
 				mmu_notifier_test_young(vma->vm_mm, addr);
 			result->page_sz = ((1UL) << HPAGE_PMD_SHIFT);
 #else
 static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz)
 {
+	struct folio *folio;
 	struct page *page = damon_get_page(PHYS_PFN(paddr));
 	struct damon_pa_access_chk_result result = {
 		.page_sz = PAGE_SIZE,
 	struct rmap_walk_control rwc = {
 		.arg = &result,
 		.rmap_one = __damon_pa_young,
-		.anon_lock = page_lock_anon_vma_read,
+		.anon_lock = folio_lock_anon_vma_read,
 	};
 	bool need_lock;
 	if (!page)
 		return false;
+	folio = page_folio(page);
-	if (!page_mapped(page) || !page_rmapping(page)) {
+	if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
-		if (page_is_idle(page))
+		if (folio_test_idle(folio))
 			result.accessed = false;
 		else
 			result.accessed = true;
-		put_page(page);
+		folio_put(folio);
 		goto out;
 	}
-	need_lock = !PageAnon(page) || PageKsm(page);
+	need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
-	if (need_lock && !trylock_page(page)) {
+	if (need_lock && !folio_trylock(folio)) {
-		put_page(page);
+		folio_put(folio);
-		return NULL;
+		return false;
 	}
-	rmap_walk(page, &rwc);
+	rmap_walk(folio, &rwc);
 	if (need_lock)
-		unlock_page(page);
+		folio_unlock(folio);
-	put_page(page);
+	folio_put(folio);
 out:
 	*page_sz = result.page_sz;

mm/debug.c+7−11

 static void __dump_page(struct page *page)
 {
-	struct page *head = compound_head(page);
+	struct folio *folio = page_folio(page);
+	struct page *head = &folio->page;
 	struct address_space *mapping;
 	bool compound = PageCompound(page);
 	/*
 		else
 			mapping = (void *)(tmp & ~PAGE_MAPPING_FLAGS);
 		head = page;
+		folio = (struct folio *)page;
 		compound = false;
 	} else {
 		mapping = page_mapping(page);
 			page, page_ref_count(head), mapcount, mapping,
 			page_to_pgoff(page), page_to_pfn(page));
 	if (compound) {
-		if (hpage_pincount_available(page)) {
+		pr_warn("head:%p order:%u compound_mapcount:%d compound_pincount:%d\n",
-			pr_warn("head:%p order:%u compound_mapcount:%d compound_pincount:%d\n",
+				head, compound_order(head),
-					head, compound_order(head),
+				folio_entire_mapcount(folio),
-					head_compound_mapcount(head),
+				head_compound_pincount(head));
-					head_compound_pincount(head));
-		} else {
-			pr_warn("head:%p order:%u compound_mapcount:%d\n",
-					head, compound_order(head),
-					head_compound_mapcount(head));
-		}
 	}
 #ifdef CONFIG_MEMCG

mm/filemap.c+41−18

 {
 	XA_STATE(xas, &mapping->i_pages, index);
 	int huge = folio_test_hugetlb(folio);
-	int error;
 	bool charged = false;
+	long nr = 1;
 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
 	VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio);
 	mapping_set_update(&xas, mapping);
-	folio_get(folio);
-	folio->mapping = mapping;
-	folio->index = index;
 	if (!huge) {
-		error = mem_cgroup_charge(folio, NULL, gfp);
+		int error = mem_cgroup_charge(folio, NULL, gfp);
 		VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio);
 		if (error)
-			goto error;
+			return error;
 		charged = true;
+		xas_set_order(&xas, index, folio_order(folio));
+		nr = folio_nr_pages(folio);
 	}
 	gfp &= GFP_RECLAIM_MASK;
+	folio_ref_add(folio, nr);
+	folio->mapping = mapping;
+	folio->index = xas.xa_index;
 	do {
 		unsigned int order = xa_get_order(xas.xa, xas.xa_index);
 			/* entry may have been split before we acquired lock */
 			order = xa_get_order(xas.xa, xas.xa_index);
 			if (order > folio_order(folio)) {
+				/* How to handle large swap entries? */
+				BUG_ON(shmem_mapping(mapping));
 				xas_split(&xas, old, order);
 				xas_reset(&xas);
 			}
 		if (xas_error(&xas))
 			goto unlock;
-		mapping->nrpages++;
+		mapping->nrpages += nr;
 		/* hugetlb pages do not participate in page cache accounting */
-		if (!huge)
+		if (!huge) {
-			__lruvec_stat_add_folio(folio, NR_FILE_PAGES);
+			__lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
+			if (folio_test_pmd_mappable(folio))
+				__lruvec_stat_mod_folio(folio,
+						NR_FILE_THPS, nr);
+		}
 unlock:
 		xas_unlock_irq(&xas);
 	} while (xas_nomem(&xas, gfp));
-	if (xas_error(&xas)) {
+	if (xas_error(&xas))
-		error = xas_error(&xas);
-		if (charged)
-			mem_cgroup_uncharge(folio);
 		goto error;
-	}
 	trace_mm_filemap_add_to_page_cache(folio);
 	return 0;
 error:
+	if (charged)
+		mem_cgroup_uncharge(folio);
 	folio->mapping = NULL;
 	/* Leave page->index set: truncation relies upon it */
-	folio_put(folio);
+	folio_put_refs(folio, nr);
-	return error;
+	return xas_error(&xas);
 }
 ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO);
 	struct file *fpin = NULL;
 	unsigned int mmap_miss;
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	/* Use the readahead code, even if readahead is disabled */
+	if (vmf->vma->vm_flags & VM_HUGEPAGE) {
+		fpin = maybe_unlock_mmap_for_io(vmf, fpin);
+		ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1);
+		ra->size = HPAGE_PMD_NR;
+		/*
+		 * Fetch two PMD folios, so we get the chance to actually
+		 * readahead, unless we've been told not to.
+		 */
+		if (!(vmf->vma->vm_flags & VM_RAND_READ))
+			ra->size *= 2;
+		ra->async_size = HPAGE_PMD_NR;
+		page_cache_ra_order(&ractl, ra, HPAGE_PMD_ORDER);
+		return fpin;
+	}
+#endif
 	/* If we don't want any read-ahead, don't bother */
 	if (vmf->vma->vm_flags & VM_RAND_READ)
 		return fpin;
 	ra->size = ra->ra_pages;
 	ra->async_size = ra->ra_pages / 4;
 	ractl._index = ra->start;
-	do_page_cache_ra(&ractl, ra->size, ra->async_size);
+	page_cache_ra_order(&ractl, ra, 0);
 	return fpin;
 }

mm/folio-compat.c+13−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#include <linux/migrate.h>	7	#include <linux/migrate.h>
8	#include <linux/pagemap.h>	8	#include <linux/pagemap.h>
9	#include <linux/swap.h>	9	#include <linux/swap.h>
		10	#include "internal.h"
10		11
11	struct address_space page_mapping(struct page page)	12	struct address_space page_mapping(struct page page)
12	{	13	{
@@ -151,3 +152,15 @@ int try_to_release_page(struct page *page, gfp_t gfp) · 138 unmodified lines
151	return filemap_release_folio(page_folio(page), gfp);	152	return filemap_release_folio(page_folio(page), gfp);
152	}	153	}
153	EXPORT_SYMBOL(try_to_release_page);	154	EXPORT_SYMBOL(try_to_release_page);
		155
		156	int isolate_lru_page(struct page *page)
		157	{
		158	if (WARN_RATELIMIT(PageTail(page), "trying to isolate tail page"))
		159	return -EBUSY;
		160	return folio_isolate_lru((struct folio *)page);
		161	}
		162
		163	void putback_lru_page(struct page *page)
		164	{
		165	folio_putback_lru(page_folio(page));
		166	}

mm/gup.c+205−281

@@ -29,107 +29,71 @@ struct follow_page_context { · 28 unmodified lines
29	unsigned int page_mask;	29	unsigned int page_mask;
30	};	30	};
31		31
32	static void hpage_pincount_add(struct page *page, int refs)
33	{
34	VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
35	VM_BUG_ON_PAGE(page != compound_head(page), page);
36
37	atomic_add(refs, compound_pincount_ptr(page));
38	}
39
40	static void hpage_pincount_sub(struct page *page, int refs)
41	{
42	VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
43	VM_BUG_ON_PAGE(page != compound_head(page), page);
44
45	atomic_sub(refs, compound_pincount_ptr(page));
46	}
47
48	/* Equivalent to calling put_page() @refs times. */
49	static void put_page_refs(struct page *page, int refs)
50	{
51	#ifdef CONFIG_DEBUG_VM
52	if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))
53	return;
54	#endif
55
56	/*
57	* Calling put_page() for each ref is unnecessarily slow. Only the last
58	* ref needs a put_page().
59	*/
60	if (refs > 1)
61	page_ref_sub(page, refs - 1);
62	put_page(page);
63	}
64
65	/*	32	/*
66	* Return the compound head page with ref appropriately incremented,	33	* Return the folio with ref appropriately incremented,
67	* or NULL if that failed.	34	* or NULL if that failed.
68	*/	35	*/
69	static inline struct page try_get_compound_head(struct page page, int refs)	36	static inline struct folio try_get_folio(struct page page, int refs)
70	{	37	{
71	struct page *head = compound_head(page);	38	struct folio *folio;
72		39
73	if (WARN_ON_ONCE(page_ref_count(head) < 0))	40	retry:
		41	folio = page_folio(page);
		42	if (WARN_ON_ONCE(folio_ref_count(folio) < 0))
74	return NULL;	43	return NULL;
75	if (unlikely(!page_cache_add_speculative(head, refs)))	44	if (unlikely(!folio_ref_try_add_rcu(folio, refs)))
76	return NULL;	45	return NULL;
77		46
78	/*	47	/*
79	* At this point we have a stable reference to the head page; but it	48	* At this point we have a stable reference to the folio; but it
80	* could be that between the compound_head() lookup and the refcount	49	* could be that between calling page_folio() and the refcount
81	* increment, the compound page was split, in which case we'd end up	50	* increment, the folio was split, in which case we'd end up
82	* holding a reference on a page that has nothing to do with the page	51	* holding a reference on a folio that has nothing to do with the page
83	* we were given anymore.	52	* we were given anymore.
84	* So now that the head page is stable, recheck that the pages still	53	* So now that the folio is stable, recheck that the page still
85	* belong together.	54	* belongs to this folio.
86	*/	55	*/
87	if (unlikely(compound_head(page) != head)) {	56	if (unlikely(page_folio(page) != folio)) {
88	put_page_refs(head, refs);	57	folio_put_refs(folio, refs);
89	return NULL;	58	goto retry;
90	}	59	}
91		60
92	return head;	61	return folio;
93	}	62	}
94		63
95	/**	64	/**
96	* try_grab_compound_head() - attempt to elevate a page's refcount, by a	65	* try_grab_folio() - Attempt to get or pin a folio.
97	* flags-dependent amount.
98	*
99	* Even though the name includes "compound_head", this function is still
100	* appropriate for callers that have a non-compound @page to get.
101	*
102	* @page: pointer to page to be grabbed	66	* @page: pointer to page to be grabbed
103	* @refs: the value to (effectively) add to the page's refcount	67	* @refs: the value to (effectively) add to the folio's refcount
104	* @flags: gup flags: these are the FOLL_* flag values.	68	* @flags: gup flags: these are the FOLL_* flag values.
105	*	69	*
106	* "grab" names in this file mean, "look at flags to decide whether to use	70	* "grab" names in this file mean, "look at flags to decide whether to use
107	* FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.	71	* FOLL_PIN or FOLL_GET behavior, when incrementing the folio's refcount.
108	*	72	*
109	* Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the	73	* Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
110	* same time. (That's true throughout the get_user_pages*() and	74	* same time. (That's true throughout the get_user_pages*() and
111	* pin_user_pages*() APIs.) Cases:	75	* pin_user_pages*() APIs.) Cases:
112	*	76	*
113	* FOLL_GET: page's refcount will be incremented by @refs.	77	* FOLL_GET: folio's refcount will be incremented by @refs.
114	*	78	*
115	* FOLL_PIN on compound pages that are > two pages long: page's refcount will	79	* FOLL_PIN on large folios: folio's refcount will be incremented by
116	* be incremented by @refs, and page[2].hpage_pinned_refcount will be	80	* @refs, and its compound_pincount will be incremented by @refs.
117	* incremented by @refs * GUP_PIN_COUNTING_BIAS.
118	*	81	*
119	* FOLL_PIN on normal pages, or compound pages that are two pages long:	82	* FOLL_PIN on single-page folios: folio's refcount will be incremented by
120	* page's refcount will be incremented by @refs * GUP_PIN_COUNTING_BIAS.	83	* @refs * GUP_PIN_COUNTING_BIAS.
121	*	84	*
122	* Return: head page (with refcount appropriately incremented) for success, or	85	* Return: The folio containing @page (with refcount appropriately
123	* NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's	86	* incremented) for success, or NULL upon failure. If neither FOLL_GET
124	* considered failure, and furthermore, a likely bug in the caller, so a warning	87	* nor FOLL_PIN was set, that's considered failure, and furthermore,
125	* is also emitted.	88	* a likely bug in the caller, so a warning is also emitted.
126	*/	89	*/
127	__maybe_unused struct page try_grab_compound_head(struct page page,	90	struct folio try_grab_folio(struct page page, int refs, unsigned int flags)
128	int refs, unsigned int flags)
129	{	91	{
130	if (flags & FOLL_GET)	92	if (flags & FOLL_GET)
131	return try_get_compound_head(page, refs);	93	return try_get_folio(page, refs);
132	else if (flags & FOLL_PIN) {	94	else if (flags & FOLL_PIN) {
		95	struct folio *folio;
		96
133	/*	97	/*
134	* Can't do FOLL_LONGTERM + FOLL_PIN gup fast path if not in a	98	* Can't do FOLL_LONGTERM + FOLL_PIN gup fast path if not in a
135	* right zone, so fail and let the caller fall back to the slow	99	* right zone, so fail and let the caller fall back to the slow
@@ -143,63 +107,57 @@ __maybe_unused struct page try_grab_compound_head(struct page page, · 7 unmodified lines
143	* CAUTION: Don't use compound_head() on the page before this	107	* CAUTION: Don't use compound_head() on the page before this
144	* point, the result won't be stable.	108	* point, the result won't be stable.
145	*/	109	*/
146	page = try_get_compound_head(page, refs);	110	folio = try_get_folio(page, refs);
147	if (!page)	111	if (!folio)
148	return NULL;	112	return NULL;
149		113
150	/*	114	/*
151	* When pinning a compound page of order > 1 (which is what	115	* When pinning a large folio, use an exact count to track it.
152	* hpage_pincount_available() checks for), use an exact count to
153	* track it, via hpage_pincount_add/_sub().
154	*	116	*
155	* However, be sure to also increment the normal page refcount	117	* However, be sure to also increment the normal folio
156	* field at least once, so that the page really is pinned.	118	* refcount field at least once, so that the folio really
157	* That's why the refcount from the earlier	119	* is pinned. That's why the refcount from the earlier
158	* try_get_compound_head() is left intact.	120	* try_get_folio() is left intact.
159	*/	121	*/
160	if (hpage_pincount_available(page))	122	if (folio_test_large(folio))
161	hpage_pincount_add(page, refs);	123	atomic_add(refs, folio_pincount_ptr(folio));
162	else	124	else
163	page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));	125	folio_ref_add(folio,
164		126	refs * (GUP_PIN_COUNTING_BIAS - 1));
165	mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,	127	node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);
166	refs);
167		128
168	return page;	129	return folio;
169	}	130	}
170		131
171	WARN_ON_ONCE(1);	132	WARN_ON_ONCE(1);
172	return NULL;	133	return NULL;
173	}	134	}
174		135
175	static void put_compound_head(struct page *page, int refs, unsigned int flags)	136	static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)
176	{	137	{
177	if (flags & FOLL_PIN) {	138	if (flags & FOLL_PIN) {
178	mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,	139	node_stat_mod_folio(folio, NR_FOLL_PIN_RELEASED, refs);
179	refs);	140	if (folio_test_large(folio))
180		141	atomic_sub(refs, folio_pincount_ptr(folio));
181	if (hpage_pincount_available(page))
182	hpage_pincount_sub(page, refs);
183	else	142	else
184	refs *= GUP_PIN_COUNTING_BIAS;	143	refs *= GUP_PIN_COUNTING_BIAS;
185	}	144	}
186		145
187	put_page_refs(page, refs);	146	folio_put_refs(folio, refs);
188	}	147	}
189		148
190	/**	149	/**
191	* try_grab_page() - elevate a page's refcount by a flag-dependent amount	150	* try_grab_page() - elevate a page's refcount by a flag-dependent amount
		151	* @page: pointer to page to be grabbed
		152	* @flags: gup flags: these are the FOLL_* flag values.
192	*	153	*
193	* This might not do anything at all, depending on the flags argument.	154	* This might not do anything at all, depending on the flags argument.
194	*	155	*
195	* "grab" names in this file mean, "look at flags to decide whether to use	156	* "grab" names in this file mean, "look at flags to decide whether to use
196	* FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.	157	* FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
197	*	158	*
198	* @page: pointer to page to be grabbed
199	* @flags: gup flags: these are the FOLL_* flag values.
200	*
201	* Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same	159	* Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
202	* time. Cases: please see the try_grab_compound_head() documentation, with	160	* time. Cases: please see the try_grab_folio() documentation, with
203	* "refs=1".	161	* "refs=1".
204	*	162	*
205	* Return: true for success, or if no action was required (if neither FOLL_PIN	163	* Return: true for success, or if no action was required (if neither FOLL_PIN
@@ -208,32 +166,28 @@ static void put_compound_head(struct page *page, int refs, unsigned int flags) · 2 unmodified lines
208	*/	166	*/
209	bool __must_check try_grab_page(struct page *page, unsigned int flags)	167	bool __must_check try_grab_page(struct page *page, unsigned int flags)
210	{	168	{
		169	struct folio *folio = page_folio(page);
		170
211	WARN_ON_ONCE((flags & (FOLL_GET \| FOLL_PIN)) == (FOLL_GET \| FOLL_PIN));	171	WARN_ON_ONCE((flags & (FOLL_GET \| FOLL_PIN)) == (FOLL_GET \| FOLL_PIN));
		172	if (WARN_ON_ONCE(folio_ref_count(folio) <= 0))
		173	return false;
212		174
213	if (flags & FOLL_GET)	175	if (flags & FOLL_GET)
214	return try_get_page(page);	176	folio_ref_inc(folio);
215	else if (flags & FOLL_PIN) {	177	else if (flags & FOLL_PIN) {
216	int refs = 1;
217
218	page = compound_head(page);
219
220	if (WARN_ON_ONCE(page_ref_count(page) <= 0))
221	return false;
222
223	if (hpage_pincount_available(page))
224	hpage_pincount_add(page, 1);
225	else
226	refs = GUP_PIN_COUNTING_BIAS;
227
228	/*	178	/*
229	* Similar to try_grab_compound_head(): even if using the	179	* Similar to try_grab_folio(): be sure to also
230	* hpage_pincount_add/_sub() routines, be sure to	180	* increment the normal page refcount field at least once,
231	* also increment the normal page refcount field at least	181	* so that the page really is pinned.
232	* once, so that the page really is pinned.
233	*/	182	*/
234	page_ref_add(page, refs);	183	if (folio_test_large(folio)) {
		184	folio_ref_add(folio, 1);
		185	atomic_add(1, folio_pincount_ptr(folio));
		186	} else {
		187	folio_ref_add(folio, GUP_PIN_COUNTING_BIAS);
		188	}
235		189
236	mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);	190	node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, 1);
237	}	191	}
238		192
239	return true;	193	return true;
@@ -250,62 +204,40 @@ bool __must_check try_grab_page(struct page *page, unsigned int flags) · 10 unmodified lines
250	*/	204	*/
251	void unpin_user_page(struct page *page)	205	void unpin_user_page(struct page *page)
252	{	206	{
253	put_compound_head(compound_head(page), 1, FOLL_PIN);	207	gup_put_folio(page_folio(page), 1, FOLL_PIN);
254	}	208	}
255	EXPORT_SYMBOL(unpin_user_page);	209	EXPORT_SYMBOL(unpin_user_page);
256		210
257	static inline void compound_range_next(unsigned long i, unsigned long npages,	211	static inline struct folio gup_folio_range_next(struct page start,
258	struct page list, struct page head,	212	unsigned long npages, unsigned long i, unsigned int *ntails)
259	unsigned int *ntails)
260	{	213	{
261	struct page next, page;	214	struct page *next = nth_page(start, i);
		215	struct folio *folio = page_folio(next);
262	unsigned int nr = 1;	216	unsigned int nr = 1;
263		217
264	if (i >= npages)	218	if (folio_test_large(folio))
265	return;	219	nr = min_t(unsigned int, npages - i,
266		220	folio_nr_pages(folio) - folio_page_idx(folio, next));
267	next = *list + i;
268	page = compound_head(next);
269	if (PageCompound(page) && compound_order(page) >= 1)
270	nr = min_t(unsigned int,
271	page + compound_nr(page) - next, npages - i);
272		221
273	*head = page;
274	*ntails = nr;	222	*ntails = nr;
		223	return folio;
275	}	224	}
276		225
277	#define for_each_compound_range(__i, __list, __npages, __head, __ntails) \	226	static inline struct folio gup_folio_next(struct page *list,
278	for (__i = 0, \	227	unsigned long npages, unsigned long i, unsigned int *ntails)
279	compound_range_next(__i, __npages, __list, &(__head), &(__ntails)); \
280	__i < __npages; __i += __ntails, \
281	compound_range_next(__i, __npages, __list, &(__head), &(__ntails)))
282
283	static inline void compound_next(unsigned long i, unsigned long npages,
284	struct page list, struct page head,
285	unsigned int *ntails)
286	{	228	{
287	struct page *page;	229	struct folio *folio = page_folio(list[i]);
288	unsigned int nr;	230	unsigned int nr;
289		231
290	if (i >= npages)
291	return;
292
293	page = compound_head(list[i]);
294	for (nr = i + 1; nr < npages; nr++) {	232	for (nr = i + 1; nr < npages; nr++) {
295	if (compound_head(list[nr]) != page)	233	if (page_folio(list[nr]) != folio)
296	break;	234	break;
297	}	235	}
298		236
299	*head = page;
300	*ntails = nr - i;	237	*ntails = nr - i;
		238	return folio;
301	}	239	}
302		240
303	#define for_each_compound_head(__i, __list, __npages, __head, __ntails) \
304	for (__i = 0, \
305	compound_next(__i, __npages, __list, &(__head), &(__ntails)); \
306	__i < __npages; __i += __ntails, \
307	compound_next(__i, __npages, __list, &(__head), &(__ntails)))
308
309	/**	241	/**
310	* unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages	242	* unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
311	* @pages: array of pages to be maybe marked dirty, and definitely released.	243	* @pages: array of pages to be maybe marked dirty, and definitely released.
@@ -331,16 +263,17 @@ static inline void compound_next(unsigned long i, unsigned long npages, · 19 unmodified lines
331	void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,	263	void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
332	bool make_dirty)	264	bool make_dirty)
333	{	265	{
334	unsigned long index;	266	unsigned long i;
335	struct page *head;	267	struct folio *folio;
336	unsigned int ntails;	268	unsigned int nr;
337		269
338	if (!make_dirty) {	270	if (!make_dirty) {
339	unpin_user_pages(pages, npages);	271	unpin_user_pages(pages, npages);
340	return;	272	return;
341	}	273	}
342		274
343	for_each_compound_head(index, pages, npages, head, ntails) {	275	for (i = 0; i < npages; i += nr) {
		276	folio = gup_folio_next(pages, npages, i, &nr);
344	/*	277	/*
345	* Checking PageDirty at this point may race with	278	* Checking PageDirty at this point may race with
346	* clear_page_dirty_for_io(), but that's OK. Two key	279	* clear_page_dirty_for_io(), but that's OK. Two key
@@ -361,9 +294,12 @@ void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, · 14 unmodified lines
361	* written back, so it gets written back again in the	294	* written back, so it gets written back again in the
362	* next writeback cycle. This is harmless.	295	* next writeback cycle. This is harmless.
363	*/	296	*/
364	if (!PageDirty(head))	297	if (!folio_test_dirty(folio)) {
365	set_page_dirty_lock(head);	298	folio_lock(folio);
366	put_compound_head(head, ntails, FOLL_PIN);	299	folio_mark_dirty(folio);
		300	folio_unlock(folio);
		301	}
		302	gup_put_folio(folio, nr, FOLL_PIN);
367	}	303	}
368	}	304	}
369	EXPORT_SYMBOL(unpin_user_pages_dirty_lock);	305	EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
@@ -392,14 +328,18 @@ EXPORT_SYMBOL(unpin_user_pages_dirty_lock); · 22 unmodified lines
392	void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,	328	void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
393	bool make_dirty)	329	bool make_dirty)
394	{	330	{
395	unsigned long index;	331	unsigned long i;
396	struct page *head;	332	struct folio *folio;
397	unsigned int ntails;	333	unsigned int nr;
398		334
399	for_each_compound_range(index, &page, npages, head, ntails) {	335	for (i = 0; i < npages; i += nr) {
400	if (make_dirty && !PageDirty(head))	336	folio = gup_folio_range_next(page, npages, i, &nr);
401	set_page_dirty_lock(head);	337	if (make_dirty && !folio_test_dirty(folio)) {
402	put_compound_head(head, ntails, FOLL_PIN);	338	folio_lock(folio);
		339	folio_mark_dirty(folio);
		340	folio_unlock(folio);
		341	}
		342	gup_put_folio(folio, nr, FOLL_PIN);
403	}	343	}
404	}	344	}
405	EXPORT_SYMBOL(unpin_user_page_range_dirty_lock);	345	EXPORT_SYMBOL(unpin_user_page_range_dirty_lock);
@@ -415,9 +355,9 @@ EXPORT_SYMBOL(unpin_user_page_range_dirty_lock); · 9 unmodified lines
415	*/	355	*/
416	void unpin_user_pages(struct page **pages, unsigned long npages)	356	void unpin_user_pages(struct page **pages, unsigned long npages)
417	{	357	{
418	unsigned long index;	358	unsigned long i;
419	struct page *head;	359	struct folio *folio;
420	unsigned int ntails;	360	unsigned int nr;
421		361
422	/*	362	/*
423	* If this WARN_ON() fires, then the system might be leaking pages (by	363	* If this WARN_ON() fires, then the system might be leaking pages (by
@@ -427,8 +367,10 @@ void unpin_user_pages(struct page **pages, unsigned long npages) · 3 unmodified lines
427	if (WARN_ON(IS_ERR_VALUE(npages)))	367	if (WARN_ON(IS_ERR_VALUE(npages)))
428	return;	368	return;
429		369
430	for_each_compound_head(index, pages, npages, head, ntails)	370	for (i = 0; i < npages; i += nr) {
431	put_compound_head(head, ntails, FOLL_PIN);	371	folio = gup_folio_next(pages, npages, i, &nr);
		372	gup_put_folio(folio, nr, FOLL_PIN);
		373	}
432	}	374	}
433	EXPORT_SYMBOL(unpin_user_pages);	375	EXPORT_SYMBOL(unpin_user_pages);
434		376
@@ -593,32 +535,6 @@ retry: · 158 unmodified lines
593	*/	535	*/
594	mark_page_accessed(page);	536	mark_page_accessed(page);
595	}	537	}
596	if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
597	/* Do not mlock pte-mapped THP */
598	if (PageTransCompound(page))
599	goto out;
600
601	/*
602	* The preliminary mapping check is mainly to avoid the
603	* pointless overhead of lock_page on the ZERO_PAGE
604	* which might bounce very badly if there is contention.
605	*
606	* If the page is already locked, we don't need to
607	* handle it now - vmscan will handle it later if and
608	* when it attempts to reclaim the page.
609	*/
610	if (page->mapping && trylock_page(page)) {
611	lru_add_drain(); /* push cached pages to LRU */
612	/*
613	* Because we lock page here, and migration is
614	* blocked by the pte's page reference, and we
615	* know the page is still mapped, we don't even
616	* need to check for file-cache page truncation.
617	*/
618	mlock_vma_page(page);
619	unlock_page(page);
620	}
621	}
622	out:	538	out:
623	pte_unmap_unlock(ptep, ptl);	539	pte_unmap_unlock(ptep, ptl);
624	return page;	540	return page;
@@ -941,9 +857,6 @@ static int faultin_page(struct vm_area_struct *vma, · 316 unmodified lines
941	unsigned int fault_flags = 0;	857	unsigned int fault_flags = 0;
942	vm_fault_t ret;	858	vm_fault_t ret;
943		859
944	/* mlock all present pages, but do not fault in new pages */
945	if ((*flags & (FOLL_POPULATE \| FOLL_MLOCK)) == FOLL_MLOCK)
946	return -ENOENT;
947	if (*flags & FOLL_NOFAULT)	860	if (*flags & FOLL_NOFAULT)
948	return -EFAULT;	861	return -EFAULT;
949	if (*flags & FOLL_WRITE)	862	if (*flags & FOLL_WRITE)
@@ -1194,8 +1107,6 @@ retry: · 244 unmodified lines
1194	case -ENOMEM:	1107	case -ENOMEM:
1195	case -EHWPOISON:	1108	case -EHWPOISON:
1196	goto out;	1109	goto out;
1197	case -ENOENT:
1198	goto next_page;
1199	}	1110	}
1200	BUG();	1111	BUG();
1201	} else if (PTR_ERR(page) == -EEXIST) {	1112	} else if (PTR_ERR(page) == -EEXIST) {
@@ -1500,9 +1411,14 @@ long populate_vma_page_range(struct vm_area_struct *vma, · 298 unmodified lines
1500	VM_BUG_ON_VMA(end > vma->vm_end, vma);	1411	VM_BUG_ON_VMA(end > vma->vm_end, vma);
1501	mmap_assert_locked(mm);	1412	mmap_assert_locked(mm);
1502		1413
1503	gup_flags = FOLL_TOUCH \| FOLL_POPULATE \| FOLL_MLOCK;	1414	/*
		1415	* Rightly or wrongly, the VM_LOCKONFAULT case has never used
		1416	* faultin_page() to break COW, so it has no work to do here.
		1417	*/
1504	if (vma->vm_flags & VM_LOCKONFAULT)	1418	if (vma->vm_flags & VM_LOCKONFAULT)
1505	gup_flags &= ~FOLL_POPULATE;	1419	return nr_pages;
		1420
		1421	gup_flags = FOLL_TOUCH;
1506	/*	1422	/*
1507	* We want to touch writable mappings with a write fault in order	1423	* We want to touch writable mappings with a write fault in order
1508	* to break COW, except for shared mappings because these don't COW	1424	* to break COW, except for shared mappings because these don't COW
@@ -1569,10 +1485,9 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, · 60 unmodified lines
1569	* in the page table.	1485	* in the page table.
1570	* FOLL_HWPOISON: Return -EHWPOISON instead of -EFAULT when we hit	1486	* FOLL_HWPOISON: Return -EHWPOISON instead of -EFAULT when we hit
1571	* a poisoned page.	1487	* a poisoned page.
1572	* FOLL_POPULATE: Always populate memory with VM_LOCKONFAULT.
1573	* !FOLL_FORCE: Require proper access permissions.	1488	* !FOLL_FORCE: Require proper access permissions.
1574	*/	1489	*/
1575	gup_flags = FOLL_TOUCH \| FOLL_POPULATE \| FOLL_MLOCK \| FOLL_HWPOISON;	1490	gup_flags = FOLL_TOUCH \| FOLL_HWPOISON;
1576	if (write)	1491	if (write)
1577	gup_flags \|= FOLL_WRITE;	1492	gup_flags \|= FOLL_WRITE;
1578		1493
@@ -1852,72 +1767,80 @@ static long check_and_migrate_movable_pages(unsigned long nr_pages, · 273 unmodified lines
1852	struct page **pages,	1767	struct page **pages,
1853	unsigned int gup_flags)	1768	unsigned int gup_flags)
1854	{	1769	{
1855	unsigned long i;	1770	unsigned long isolation_error_count = 0, i;
1856	unsigned long isolation_error_count = 0;	1771	struct folio *prev_folio = NULL;
1857	bool drain_allow = true;
1858	LIST_HEAD(movable_page_list);	1772	LIST_HEAD(movable_page_list);
1859	long ret = 0;	1773	bool drain_allow = true;
1860	struct page *prev_head = NULL;	1774	int ret = 0;
1861	struct page *head;
1862	struct migration_target_control mtc = {
1863	.nid = NUMA_NO_NODE,
1864	.gfp_mask = GFP_USER \| __GFP_NOWARN,
1865	};
1866		1775
1867	for (i = 0; i < nr_pages; i++) {	1776	for (i = 0; i < nr_pages; i++) {
1868	head = compound_head(pages[i]);	1777	struct folio *folio = page_folio(pages[i]);
1869	if (head == prev_head)	1778
		1779	if (folio == prev_folio)
1870	continue;	1780	continue;
1871	prev_head = head;	1781	prev_folio = folio;
		1782
		1783	if (folio_is_pinnable(folio))
		1784	continue;
		1785
1872	/*	1786	/*
1873	* If we get a movable page, since we are going to be pinning	1787	* Try to move out any movable page before pinning the range.
1874	* these entries, try to move them out if possible.
1875	*/	1788	*/
1876	if (!is_pinnable_page(head)) {	1789	if (folio_test_hugetlb(folio)) {
1877	if (PageHuge(head)) {	1790	if (!isolate_huge_page(&folio->page,
1878	if (!isolate_huge_page(head, &movable_page_list))	1791	&movable_page_list))
1879	isolation_error_count++;	1792	isolation_error_count++;
1880	} else {	1793	continue;
1881	if (!PageLRU(head) && drain_allow) {	1794	}
1882	lru_add_drain_all();
1883	drain_allow = false;
1884	}
1885		1795
1886	if (isolate_lru_page(head)) {	1796	if (!folio_test_lru(folio) && drain_allow) {
1887	isolation_error_count++;	1797	lru_add_drain_all();
1888	continue;	1798	drain_allow = false;
1889	}	1799	}
1890	list_add_tail(&head->lru, &movable_page_list);	1800
1891	mod_node_page_state(page_pgdat(head),	1801	if (folio_isolate_lru(folio)) {
1892	NR_ISOLATED_ANON +	1802	isolation_error_count++;
1893	page_is_file_lru(head),	1803	continue;
1894	thp_nr_pages(head));
1895	}
1896	}	1804	}
		1805	list_add_tail(&folio->lru, &movable_page_list);
		1806	node_stat_mod_folio(folio,
		1807	NR_ISOLATED_ANON + folio_is_file_lru(folio),
		1808	folio_nr_pages(folio));
1897	}	1809	}
1898		1810
		1811	if (!list_empty(&movable_page_list) \|\| isolation_error_count)
		1812	goto unpin_pages;
		1813
1899	/*	1814	/*
1900	* If list is empty, and no isolation errors, means that all pages are	1815	* If list is empty, and no isolation errors, means that all pages are
1901	* in the correct zone.	1816	* in the correct zone.
1902	*/	1817	*/
1903	if (list_empty(&movable_page_list) && !isolation_error_count)	1818	return nr_pages;
1904	return nr_pages;
1905		1819
		1820	unpin_pages:
1906	if (gup_flags & FOLL_PIN) {	1821	if (gup_flags & FOLL_PIN) {
1907	unpin_user_pages(pages, nr_pages);	1822	unpin_user_pages(pages, nr_pages);
1908	} else {	1823	} else {
1909	for (i = 0; i < nr_pages; i++)	1824	for (i = 0; i < nr_pages; i++)
1910	put_page(pages[i]);	1825	put_page(pages[i]);
1911	}	1826	}
		1827
1912	if (!list_empty(&movable_page_list)) {	1828	if (!list_empty(&movable_page_list)) {
		1829	struct migration_target_control mtc = {
		1830	.nid = NUMA_NO_NODE,
		1831	.gfp_mask = GFP_USER \| __GFP_NOWARN,
		1832	};
		1833
1913	ret = migrate_pages(&movable_page_list, alloc_migration_target,	1834	ret = migrate_pages(&movable_page_list, alloc_migration_target,
1914	NULL, (unsigned long)&mtc, MIGRATE_SYNC,	1835	NULL, (unsigned long)&mtc, MIGRATE_SYNC,
1915	MR_LONGTERM_PIN, NULL);	1836	MR_LONGTERM_PIN, NULL);
1916	if (ret && !list_empty(&movable_page_list))	1837	if (ret > 0) /* number of pages not migrated */
1917	putback_movable_pages(&movable_page_list);	1838	ret = -ENOMEM;
1918	}	1839	}
1919		1840
1920	return ret > 0 ? -ENOMEM : ret;	1841	if (ret && !list_empty(&movable_page_list))
		1842	putback_movable_pages(&movable_page_list);
		1843	return ret;
1921	}	1844	}
1922	#else	1845	#else
1923	static long check_and_migrate_movable_pages(unsigned long nr_pages,	1846	static long check_and_migrate_movable_pages(unsigned long nr_pages,
@@ -2227,7 +2150,8 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, · 303 unmodified lines
2227	ptem = ptep = pte_offset_map(&pmd, addr);	2150	ptem = ptep = pte_offset_map(&pmd, addr);
2228	do {	2151	do {
2229	pte_t pte = ptep_get_lockless(ptep);	2152	pte_t pte = ptep_get_lockless(ptep);
2230	struct page head, page;	2153	struct page *page;
		2154	struct folio *folio;
2231		2155
2232	/*	2156	/*
2233	* Similar to the PMD case below, NUMA hinting must take slow	2157	* Similar to the PMD case below, NUMA hinting must take slow
@@ -2254,22 +2178,20 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, · 20 unmodified lines
2254	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));	2178	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2255	page = pte_page(pte);	2179	page = pte_page(pte);
2256		2180
2257	head = try_grab_compound_head(page, 1, flags);	2181	folio = try_grab_folio(page, 1, flags);
2258	if (!head)	2182	if (!folio)
2259	goto pte_unmap;	2183	goto pte_unmap;
2260		2184
2261	if (unlikely(page_is_secretmem(page))) {	2185	if (unlikely(page_is_secretmem(page))) {
2262	put_compound_head(head, 1, flags);	2186	gup_put_folio(folio, 1, flags);
2263	goto pte_unmap;	2187	goto pte_unmap;
2264	}	2188	}
2265		2189
2266	if (unlikely(pte_val(pte) != pte_val(*ptep))) {	2190	if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2267	put_compound_head(head, 1, flags);	2191	gup_put_folio(folio, 1, flags);
2268	goto pte_unmap;	2192	goto pte_unmap;
2269	}	2193	}
2270		2194
2271	VM_BUG_ON_PAGE(compound_head(page) != head, page);
2272
2273	/*	2195	/*
2274	* We need to make the page accessible if and only if we are	2196	* We need to make the page accessible if and only if we are
2275	* going to access its content (the FOLL_PIN case). Please	2197	* going to access its content (the FOLL_PIN case). Please
@@ -2279,14 +2201,13 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, · 3 unmodified lines
2279	if (flags & FOLL_PIN) {	2201	if (flags & FOLL_PIN) {
2280	ret = arch_make_page_accessible(page);	2202	ret = arch_make_page_accessible(page);
2281	if (ret) {	2203	if (ret) {
2282	unpin_user_page(page);	2204	gup_put_folio(folio, 1, flags);
2283	goto pte_unmap;	2205	goto pte_unmap;
2284	}	2206	}
2285	}	2207	}
2286	SetPageReferenced(page);	2208	folio_set_referenced(folio);
2287	pages[*nr] = page;	2209	pages[*nr] = page;
2288	(*nr)++;	2210	(*nr)++;
2289
2290	} while (ptep++, addr += PAGE_SIZE, addr != end);	2211	} while (ptep++, addr += PAGE_SIZE, addr != end);
2291		2212
2292	ret = 1;	2213	ret = 1;
@@ -2403,8 +2324,8 @@ static int record_subpages(struct page *page, unsigned long addr, · 110 unmodified lines
2403	{	2324	{
2404	int nr;	2325	int nr;
2405		2326
2406	for (nr = 0; addr != end; addr += PAGE_SIZE)	2327	for (nr = 0; addr != end; nr++, addr += PAGE_SIZE)
2407	pages[nr++] = page++;	2328	pages[nr] = nth_page(page, nr);
2408		2329
2409	return nr;	2330	return nr;
2410	}	2331	}
@@ -2422,7 +2343,8 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, · 11 unmodified lines
2422	struct page *pages, int nr)	2343	struct page *pages, int nr)
2423	{	2344	{
2424	unsigned long pte_end;	2345	unsigned long pte_end;
2425	struct page head, page;	2346	struct page *page;
		2347	struct folio *folio;
2426	pte_t pte;	2348	pte_t pte;
2427	int refs;	2349	int refs;
2428		2350
@@ -2438,21 +2360,20 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, · 9 unmodified lines
2438	/* hugepages are never "special" */	2360	/* hugepages are never "special" */
2439	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));	2361	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2440		2362
2441	head = pte_page(pte);	2363	page = nth_page(pte_page(pte), (addr & (sz - 1)) >> PAGE_SHIFT);
2442	page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2443	refs = record_subpages(page, addr, end, pages + *nr);	2364	refs = record_subpages(page, addr, end, pages + *nr);
2444		2365
2445	head = try_grab_compound_head(head, refs, flags);	2366	folio = try_grab_folio(page, refs, flags);
2446	if (!head)	2367	if (!folio)
2447	return 0;	2368	return 0;
2448		2369
2449	if (unlikely(pte_val(pte) != pte_val(*ptep))) {	2370	if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2450	put_compound_head(head, refs, flags);	2371	gup_put_folio(folio, refs, flags);
2451	return 0;	2372	return 0;
2452	}	2373	}
2453		2374
2454	*nr += refs;	2375	*nr += refs;
2455	SetPageReferenced(head);	2376	folio_set_referenced(folio);
2456	return 1;	2377	return 1;
2457	}	2378	}
2458		2379
@@ -2486,7 +2407,8 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, · 27 unmodified lines
2486	unsigned long end, unsigned int flags,	2407	unsigned long end, unsigned int flags,
2487	struct page *pages, int nr)	2408	struct page *pages, int nr)
2488	{	2409	{
2489	struct page head, page;	2410	struct page *page;
		2411	struct folio *folio;
2490	int refs;	2412	int refs;
2491		2413
2492	if (!pmd_access_permitted(orig, flags & FOLL_WRITE))	2414	if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
@@ -2499,20 +2421,20 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, · 6 unmodified lines
2499	pages, nr);	2421	pages, nr);
2500	}	2422	}
2501		2423
2502	page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);	2424	page = nth_page(pmd_page(orig), (addr & ~PMD_MASK) >> PAGE_SHIFT);
2503	refs = record_subpages(page, addr, end, pages + *nr);	2425	refs = record_subpages(page, addr, end, pages + *nr);
2504		2426
2505	head = try_grab_compound_head(pmd_page(orig), refs, flags);	2427	folio = try_grab_folio(page, refs, flags);
2506	if (!head)	2428	if (!folio)
2507	return 0;	2429	return 0;
2508		2430
2509	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {	2431	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2510	put_compound_head(head, refs, flags);	2432	gup_put_folio(folio, refs, flags);
2511	return 0;	2433	return 0;
2512	}	2434	}
2513		2435
2514	*nr += refs;	2436	*nr += refs;
2515	SetPageReferenced(head);	2437	folio_set_referenced(folio);
2516	return 1;	2438	return 1;
2517	}	2439	}
2518		2440
@@ -2520,7 +2442,8 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, · 1 unmodified line
2520	unsigned long end, unsigned int flags,	2442	unsigned long end, unsigned int flags,
2521	struct page *pages, int nr)	2443	struct page *pages, int nr)
2522	{	2444	{
2523	struct page head, page;	2445	struct page *page;
		2446	struct folio *folio;
2524	int refs;	2447	int refs;
2525		2448
2526	if (!pud_access_permitted(orig, flags & FOLL_WRITE))	2449	if (!pud_access_permitted(orig, flags & FOLL_WRITE))
@@ -2533,20 +2456,20 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, · 6 unmodified lines
2533	pages, nr);	2456	pages, nr);
2534	}	2457	}
2535		2458
2536	page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);	2459	page = nth_page(pud_page(orig), (addr & ~PUD_MASK) >> PAGE_SHIFT);
2537	refs = record_subpages(page, addr, end, pages + *nr);	2460	refs = record_subpages(page, addr, end, pages + *nr);
2538		2461
2539	head = try_grab_compound_head(pud_page(orig), refs, flags);	2462	folio = try_grab_folio(page, refs, flags);
2540	if (!head)	2463	if (!folio)
2541	return 0;	2464	return 0;
2542		2465
2543	if (unlikely(pud_val(orig) != pud_val(*pudp))) {	2466	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2544	put_compound_head(head, refs, flags);	2467	gup_put_folio(folio, refs, flags);
2545	return 0;	2468	return 0;
2546	}	2469	}
2547		2470
2548	*nr += refs;	2471	*nr += refs;
2549	SetPageReferenced(head);	2472	folio_set_referenced(folio);
2550	return 1;	2473	return 1;
2551	}	2474	}
2552		2475
@@ -2555,27 +2478,28 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, · 2 unmodified lines
2555	struct page *pages, int nr)	2478	struct page *pages, int nr)
2556	{	2479	{
2557	int refs;	2480	int refs;
2558	struct page head, page;	2481	struct page *page;
		2482	struct folio *folio;
2559		2483
2560	if (!pgd_access_permitted(orig, flags & FOLL_WRITE))	2484	if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
2561	return 0;	2485	return 0;
2562		2486
2563	BUILD_BUG_ON(pgd_devmap(orig));	2487	BUILD_BUG_ON(pgd_devmap(orig));
2564		2488
2565	page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);	2489	page = nth_page(pgd_page(orig), (addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2566	refs = record_subpages(page, addr, end, pages + *nr);	2490	refs = record_subpages(page, addr, end, pages + *nr);
2567		2491
2568	head = try_grab_compound_head(pgd_page(orig), refs, flags);	2492	folio = try_grab_folio(page, refs, flags);
2569	if (!head)	2493	if (!folio)
2570	return 0;	2494	return 0;
2571		2495
2572	if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {	2496	if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
2573	put_compound_head(head, refs, flags);	2497	gup_put_folio(folio, refs, flags);
2574	return 0;	2498	return 0;
2575	}	2499	}
2576		2500
2577	*nr += refs;	2501	*nr += refs;
2578	SetPageReferenced(head);	2502	folio_set_referenced(folio);
2579	return 1;	2503	return 1;
2580	}	2504	}
2581		2505

mm/huge_memory.c+62−116

@@ -583,13 +583,10 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, · 582 unmodified lines
583	unsigned long ret;	583	unsigned long ret;
584	loff_t off = (loff_t)pgoff << PAGE_SHIFT;	584	loff_t off = (loff_t)pgoff << PAGE_SHIFT;
585		585
586	if (!IS_DAX(filp->f_mapping->host) \|\| !IS_ENABLED(CONFIG_FS_DAX_PMD))
587	goto out;
588
589	ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);	586	ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);
590	if (ret)	587	if (ret)
591	return ret;	588	return ret;
592	out:	589
593	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);	590	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
594	}	591	}
595	EXPORT_SYMBOL_GPL(thp_get_unmapped_area);	592	EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
@@ -1381,39 +1378,6 @@ struct page follow_trans_huge_pmd(struct vm_area_struct vma, · 785 unmodified lines
1381	if (flags & FOLL_TOUCH)	1378	if (flags & FOLL_TOUCH)
1382	touch_pmd(vma, addr, pmd, flags);	1379	touch_pmd(vma, addr, pmd, flags);
1383		1380
1384	if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
1385	/*
1386	* We don't mlock() pte-mapped THPs. This way we can avoid
1387	* leaking mlocked pages into non-VM_LOCKED VMAs.
1388	*
1389	* For anon THP:
1390	*
1391	* In most cases the pmd is the only mapping of the page as we
1392	* break COW for the mlock() -- see gup_flags \|= FOLL_WRITE for
1393	* writable private mappings in populate_vma_page_range().
1394	*
1395	* The only scenario when we have the page shared here is if we
1396	* mlocking read-only mapping shared over fork(). We skip
1397	* mlocking such pages.
1398	*
1399	* For file THP:
1400	*
1401	* We can expect PageDoubleMap() to be stable under page lock:
1402	* for file pages we set it in page_add_file_rmap(), which
1403	* requires page to be locked.
1404	*/
1405
1406	if (PageAnon(page) && compound_mapcount(page) != 1)
1407	goto skip_mlock;
1408	if (PageDoubleMap(page) \|\| !page->mapping)
1409	goto skip_mlock;
1410	if (!trylock_page(page))
1411	goto skip_mlock;
1412	if (page->mapping && !PageDoubleMap(page))
1413	mlock_vma_page(page);
1414	unlock_page(page);
1415	}
1416	skip_mlock:
1417	page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;	1381	page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1418	VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);	1382	VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
1419		1383
@@ -1611,7 +1575,7 @@ int zap_huge_pmd(struct mmu_gather tlb, struct vm_area_struct vma, · 191 unmodified lines
1611		1575
1612	if (pmd_present(orig_pmd)) {	1576	if (pmd_present(orig_pmd)) {
1613	page = pmd_page(orig_pmd);	1577	page = pmd_page(orig_pmd);
1614	page_remove_rmap(page, true);	1578	page_remove_rmap(page, vma, true);
1615	VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);	1579	VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1616	VM_BUG_ON_PAGE(!PageHead(page), page);	1580	VM_BUG_ON_PAGE(!PageHead(page), page);
1617	} else if (thp_migration_supported()) {	1581	} else if (thp_migration_supported()) {
@@ -2007,7 +1971,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct vma, pmd_t pmd, · 389 unmodified lines
2007	set_page_dirty(page);	1971	set_page_dirty(page);
2008	if (!PageReferenced(page) && pmd_young(old_pmd))	1972	if (!PageReferenced(page) && pmd_young(old_pmd))
2009	SetPageReferenced(page);	1973	SetPageReferenced(page);
2010	page_remove_rmap(page, true);	1974	page_remove_rmap(page, vma, true);
2011	put_page(page);	1975	put_page(page);
2012	}	1976	}
2013	add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);	1977	add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);
@@ -2141,6 +2105,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct vma, pmd_t pmd, · 127 unmodified lines
2141	}	2105	}
2142	}	2106	}
2143	unlock_page_memcg(page);	2107	unlock_page_memcg(page);
		2108
		2109	/* Above is effectively page_remove_rmap(page, vma, true) */
		2110	munlock_vma_page(page, vma, true);
2144	}	2111	}
2145		2112
2146	smp_wmb(); /* make pte visible before pmd */	2113	smp_wmb(); /* make pte visible before pmd */
@@ -2148,18 +2115,18 @@ static void __split_huge_pmd_locked(struct vm_area_struct vma, pmd_t pmd, · 1 unmodified line
2148		2115
2149	if (freeze) {	2116	if (freeze) {
2150	for (i = 0; i < HPAGE_PMD_NR; i++) {	2117	for (i = 0; i < HPAGE_PMD_NR; i++) {
2151	page_remove_rmap(page + i, false);	2118	page_remove_rmap(page + i, vma, false);
2152	put_page(page + i);	2119	put_page(page + i);
2153	}	2120	}
2154	}	2121	}
2155	}	2122	}
2156		2123
2157	void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,	2124	void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
2158	unsigned long address, bool freeze, struct page *page)	2125	unsigned long address, bool freeze, struct folio *folio)
2159	{	2126	{
2160	spinlock_t *ptl;	2127	spinlock_t *ptl;
2161	struct mmu_notifier_range range;	2128	struct mmu_notifier_range range;
2162	bool do_unlock_page = false;	2129	bool do_unlock_folio = false;
2163	pmd_t _pmd;	2130	pmd_t _pmd;
2164		2131
2165	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,	2132	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
@@ -2169,20 +2136,20 @@ void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd, · 3 unmodified lines
2169	ptl = pmd_lock(vma->vm_mm, pmd);	2136	ptl = pmd_lock(vma->vm_mm, pmd);
2170		2137
2171	/*	2138	/*
2172	* If caller asks to setup a migration entries, we need a page to check	2139	* If caller asks to setup a migration entry, we need a folio to check
2173	* pmd against. Otherwise we can end up replacing wrong page.	2140	* pmd against. Otherwise we can end up replacing wrong folio.
2174	*/	2141	*/
2175	VM_BUG_ON(freeze && !page);	2142	VM_BUG_ON(freeze && !folio);
2176	if (page) {	2143	if (folio) {
2177	VM_WARN_ON_ONCE(!PageLocked(page));	2144	VM_WARN_ON_ONCE(!folio_test_locked(folio));
2178	if (page != pmd_page(*pmd))	2145	if (folio != page_folio(pmd_page(*pmd)))
2179	goto out;	2146	goto out;
2180	}	2147	}
2181		2148
2182	repeat:	2149	repeat:
2183	if (pmd_trans_huge(*pmd)) {	2150	if (pmd_trans_huge(*pmd)) {
2184	if (!page) {	2151	if (!folio) {
2185	page = pmd_page(*pmd);	2152	folio = page_folio(pmd_page(*pmd));
2186	/*	2153	/*
2187	* An anonymous page must be locked, to ensure that a	2154	* An anonymous page must be locked, to ensure that a
2188	* concurrent reuse_swap_page() sees stable mapcount;	2155	* concurrent reuse_swap_page() sees stable mapcount;
@@ -2190,33 +2157,31 @@ repeat: · 1 unmodified line
2190	* and page lock must not be taken when zap_pmd_range()	2157	* and page lock must not be taken when zap_pmd_range()
2191	* calls __split_huge_pmd() while i_mmap_lock is held.	2158	* calls __split_huge_pmd() while i_mmap_lock is held.
2192	*/	2159	*/
2193	if (PageAnon(page)) {	2160	if (folio_test_anon(folio)) {
2194	if (unlikely(!trylock_page(page))) {	2161	if (unlikely(!folio_trylock(folio))) {
2195	get_page(page);	2162	folio_get(folio);
2196	_pmd = *pmd;	2163	_pmd = *pmd;
2197	spin_unlock(ptl);	2164	spin_unlock(ptl);
2198	lock_page(page);	2165	folio_lock(folio);
2199	spin_lock(ptl);	2166	spin_lock(ptl);
2200	if (unlikely(!pmd_same(*pmd, _pmd))) {	2167	if (unlikely(!pmd_same(*pmd, _pmd))) {
2201	unlock_page(page);	2168	folio_unlock(folio);
2202	put_page(page);	2169	folio_put(folio);
2203	page = NULL;	2170	folio = NULL;
2204	goto repeat;	2171	goto repeat;
2205	}	2172	}
2206	put_page(page);	2173	folio_put(folio);
2207	}	2174	}
2208	do_unlock_page = true;	2175	do_unlock_folio = true;
2209	}	2176	}
2210	}	2177	}
2211	if (PageMlocked(page))
2212	clear_page_mlock(page);
2213	} else if (!(pmd_devmap(pmd) \|\| is_pmd_migration_entry(pmd)))	2178	} else if (!(pmd_devmap(pmd) \|\| is_pmd_migration_entry(pmd)))
2214	goto out;	2179	goto out;
2215	__split_huge_pmd_locked(vma, pmd, range.start, freeze);	2180	__split_huge_pmd_locked(vma, pmd, range.start, freeze);
2216	out:	2181	out:
2217	spin_unlock(ptl);	2182	spin_unlock(ptl);
2218	if (do_unlock_page)	2183	if (do_unlock_folio)
2219	unlock_page(page);	2184	folio_unlock(folio);
2220	/*	2185	/*
2221	* No need to double call mmu_notifier->invalidate_range() callback.	2186	* No need to double call mmu_notifier->invalidate_range() callback.
2222	* They are 3 cases to consider inside __split_huge_pmd_locked():	2187	* They are 3 cases to consider inside __split_huge_pmd_locked():
@@ -2234,7 +2199,7 @@ out: · 11 unmodified lines
2234	}	2199	}
2235		2200
2236	void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,	2201	void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
2237	bool freeze, struct page *page)	2202	bool freeze, struct folio *folio)
2238	{	2203	{
2239	pgd_t *pgd;	2204	pgd_t *pgd;
2240	p4d_t *p4d;	2205	p4d_t *p4d;
@@ -2255,7 +2220,7 @@ void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, · 14 unmodified lines
2255		2220
2256	pmd = pmd_offset(pud, address);	2221	pmd = pmd_offset(pud, address);
2257		2222
2258	__split_huge_pmd(vma, pmd, address, freeze, page);	2223	__split_huge_pmd(vma, pmd, address, freeze, folio);
2259	}	2224	}
2260		2225
2261	static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)	2226	static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)
@@ -2295,6 +2260,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, · 33 unmodified lines
2295		2260
2296	static void unmap_page(struct page *page)	2261	static void unmap_page(struct page *page)
2297	{	2262	{
		2263	struct folio *folio = page_folio(page);
2298	enum ttu_flags ttu_flags = TTU_RMAP_LOCKED \| TTU_SPLIT_HUGE_PMD \|	2264	enum ttu_flags ttu_flags = TTU_RMAP_LOCKED \| TTU_SPLIT_HUGE_PMD \|
2299	TTU_SYNC;	2265	TTU_SYNC;
2300		2266
@@ -2305,26 +2271,27 @@ static void unmap_page(struct page *page) · 4 unmodified lines
2305	* pages can simply be left unmapped, then faulted back on demand.	2271	* pages can simply be left unmapped, then faulted back on demand.
2306	* If that is ever changed (perhaps for mlock), update remap_page().	2272	* If that is ever changed (perhaps for mlock), update remap_page().
2307	*/	2273	*/
2308	if (PageAnon(page))	2274	if (folio_test_anon(folio))
2309	try_to_migrate(page, ttu_flags);	2275	try_to_migrate(folio, ttu_flags);
2310	else	2276	else
2311	try_to_unmap(page, ttu_flags \| TTU_IGNORE_MLOCK);	2277	try_to_unmap(folio, ttu_flags \| TTU_IGNORE_MLOCK);
2312		2278
2313	VM_WARN_ON_ONCE_PAGE(page_mapped(page), page);	2279	VM_WARN_ON_ONCE_PAGE(page_mapped(page), page);
2314	}	2280	}
2315		2281
2316	static void remap_page(struct page *page, unsigned int nr)	2282	static void remap_page(struct folio *folio, unsigned long nr)
2317	{	2283	{
2318	int i;	2284	int i = 0;
2319		2285
2320	/* If unmap_page() uses try_to_migrate() on file, remove this check */	2286	/* If unmap_page() uses try_to_migrate() on file, remove this check */
2321	if (!PageAnon(page))	2287	if (!folio_test_anon(folio))
2322	return;	2288	return;
2323	if (PageTransHuge(page)) {	2289	for (;;) {
2324	remove_migration_ptes(page, page, true);	2290	remove_migration_ptes(folio, folio, true);
2325	} else {	2291	i += folio_nr_pages(folio);
2326	for (i = 0; i < nr; i++)	2292	if (i >= nr)
2327	remove_migration_ptes(page + i, page + i, true);	2293	break;
		2294	folio = folio_next(folio);
2328	}	2295	}
2329	}	2296	}
2330		2297
@@ -2344,8 +2311,11 @@ static void lru_add_page_tail(struct page head, struct page tail, · 13 unmodified lines
2344	} else {	2311	} else {
2345	/* head is still on lru (and we have it frozen) */	2312	/* head is still on lru (and we have it frozen) */
2346	VM_WARN_ON(!PageLRU(head));	2313	VM_WARN_ON(!PageLRU(head));
		2314	if (PageUnevictable(tail))
		2315	tail->mlock_count = 0;
		2316	else
		2317	list_add_tail(&tail->lru, &head->lru);
2347	SetPageLRU(tail);	2318	SetPageLRU(tail);
2348	list_add_tail(&tail->lru, &head->lru);
2349	}	2319	}
2350	}	2320	}
2351		2321
@@ -2481,7 +2451,7 @@ static void __split_huge_page(struct page page, struct list_head list, · 129 unmodified lines
2481	}	2451	}
2482	local_irq_enable();	2452	local_irq_enable();
2483		2453
2484	remap_page(head, nr);	2454	remap_page(folio, nr);
2485		2455
2486	if (PageSwapCache(head)) {	2456	if (PageSwapCache(head)) {
2487	swp_entry_t entry = { .val = page_private(head) };	2457	swp_entry_t entry = { .val = page_private(head) };
@@ -2506,30 +2476,6 @@ static void __split_huge_page(struct page page, struct list_head list, · 18 unmodified lines
2506	}	2476	}
2507	}	2477	}
2508		2478
2509	int total_mapcount(struct page *page)
2510	{
2511	int i, compound, nr, ret;
2512
2513	VM_BUG_ON_PAGE(PageTail(page), page);
2514
2515	if (likely(!PageCompound(page)))
2516	return atomic_read(&page->_mapcount) + 1;
2517
2518	compound = compound_mapcount(page);
2519	nr = compound_nr(page);
2520	if (PageHuge(page))
2521	return compound;
2522	ret = compound;
2523	for (i = 0; i < nr; i++)
2524	ret += atomic_read(&page[i]._mapcount) + 1;
2525	/* File pages has compound_mapcount included in _mapcount */
2526	if (!PageAnon(page))
2527	return ret - compound * nr;
2528	if (PageDoubleMap(page))
2529	ret -= nr;
2530	return ret;
2531	}
2532
2533	/*	2479	/*
2534	* This calculates accurately how many mappings a transparent hugepage	2480	* This calculates accurately how many mappings a transparent hugepage
2535	* has (unlike page_mapcount() which isn't fully accurate). This full	2481	* has (unlike page_mapcount() which isn't fully accurate). This full
@@ -2579,18 +2525,19 @@ int page_trans_huge_mapcount(struct page *page) · 43 unmodified lines
2579	}	2525	}
2580		2526
2581	/* Racy check whether the huge page can be split */	2527	/* Racy check whether the huge page can be split */
2582	bool can_split_huge_page(struct page page, int pextra_pins)	2528	bool can_split_folio(struct folio folio, int pextra_pins)
2583	{	2529	{
2584	int extra_pins;	2530	int extra_pins;
2585		2531
2586	/* Additional pins from page cache */	2532	/* Additional pins from page cache */
2587	if (PageAnon(page))	2533	if (folio_test_anon(folio))
2588	extra_pins = PageSwapCache(page) ? thp_nr_pages(page) : 0;	2534	extra_pins = folio_test_swapcache(folio) ?
		2535	folio_nr_pages(folio) : 0;
2589	else	2536	else
2590	extra_pins = thp_nr_pages(page);	2537	extra_pins = folio_nr_pages(folio);
2591	if (pextra_pins)	2538	if (pextra_pins)
2592	*pextra_pins = extra_pins;	2539	*pextra_pins = extra_pins;
2593	return total_mapcount(page) == page_count(page) - extra_pins - 1;	2540	return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 1;
2594	}	2541	}
2595		2542
2596	/*	2543	/*
@@ -2614,7 +2561,8 @@ bool can_split_huge_page(struct page page, int pextra_pins) · 17 unmodified lines
2614	*/	2561	*/
2615	int split_huge_page_to_list(struct page page, struct list_head list)	2562	int split_huge_page_to_list(struct page page, struct list_head list)
2616	{	2563	{
2617	struct page *head = compound_head(page);	2564	struct folio *folio = page_folio(page);
		2565	struct page *head = &folio->page;
2618	struct deferred_split *ds_queue = get_deferred_split_queue(head);	2566	struct deferred_split *ds_queue = get_deferred_split_queue(head);
2619	XA_STATE(xas, &head->mapping->i_pages, head->index);	2567	XA_STATE(xas, &head->mapping->i_pages, head->index);
2620	struct anon_vma *anon_vma = NULL;	2568	struct anon_vma *anon_vma = NULL;
@@ -2634,7 +2582,7 @@ int split_huge_page_to_list(struct page page, struct list_head list) · 13 unmodified lines
2634	* The caller does not necessarily hold an mmap_lock that would	2582	* The caller does not necessarily hold an mmap_lock that would
2635	* prevent the anon_vma disappearing so we first we take a	2583	* prevent the anon_vma disappearing so we first we take a
2636	* reference to it and then lock the anon_vma for write. This	2584	* reference to it and then lock the anon_vma for write. This
2637	* is similar to page_lock_anon_vma_read except the write lock	2585	* is similar to folio_lock_anon_vma_read except the write lock
2638	* is taken to serialise against parallel split or collapse	2586	* is taken to serialise against parallel split or collapse
2639	* operations.	2587	* operations.
2640	*/	2588	*/
@@ -2681,7 +2629,7 @@ int split_huge_page_to_list(struct page page, struct list_head list) · 40 unmodified lines
2681	* Racy check if we can split the page, before unmap_page() will	2629	* Racy check if we can split the page, before unmap_page() will
2682	* split PMDs	2630	* split PMDs
2683	*/	2631	*/
2684	if (!can_split_huge_page(head, &extra_pins)) {	2632	if (!can_split_folio(folio, &extra_pins)) {
2685	ret = -EBUSY;	2633	ret = -EBUSY;
2686	goto out_unlock;	2634	goto out_unlock;
2687	}	2635	}
@@ -2731,7 +2679,7 @@ fail: · 43 unmodified lines
2731	if (mapping)	2679	if (mapping)
2732	xas_unlock(&xas);	2680	xas_unlock(&xas);
2733	local_irq_enable();	2681	local_irq_enable();
2734	remap_page(head, thp_nr_pages(head));	2682	remap_page(folio, folio_nr_pages(folio));
2735	ret = -EBUSY;	2683	ret = -EBUSY;
2736	}	2684	}
2737		2685
@@ -2988,7 +2936,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start, · 250 unmodified lines
2988	goto next;	2936	goto next;
2989		2937
2990	total++;	2938	total++;
2991	if (!can_split_huge_page(compound_head(page), NULL))	2939	if (!can_split_folio(page_folio(page), NULL))
2992	goto next;	2940	goto next;
2993		2941
2994	if (!trylock_page(page))	2942	if (!trylock_page(page))
@@ -3181,7 +3129,7 @@ void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, · 186 unmodified lines
3181	if (pmd_soft_dirty(pmdval))	3129	if (pmd_soft_dirty(pmdval))
3182	pmdswp = pmd_swp_mksoft_dirty(pmdswp);	3130	pmdswp = pmd_swp_mksoft_dirty(pmdswp);
3183	set_pmd_at(mm, address, pvmw->pmd, pmdswp);	3131	set_pmd_at(mm, address, pvmw->pmd, pmdswp);
3184	page_remove_rmap(page, true);	3132	page_remove_rmap(page, vma, true);
3185	put_page(page);	3133	put_page(page);
3186	}	3134	}
3187		3135
@@ -3210,10 +3158,8 @@ void remove_migration_pmd(struct page_vma_mapped_walk pvmw, struct page new) · 22 unmodified lines
3210	if (PageAnon(new))	3158	if (PageAnon(new))
3211	page_add_anon_rmap(new, vma, mmun_start, true);	3159	page_add_anon_rmap(new, vma, mmun_start, true);
3212	else	3160	else
3213	page_add_file_rmap(new, true);	3161	page_add_file_rmap(new, vma, true);
3214	set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);	3162	set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);
3215	if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new))
3216	mlock_vma_page(new);
3217		3163
3218	/* No need to invalidate - it was non-present before */	3164	/* No need to invalidate - it was non-present before */
3219	update_mmu_cache_pmd(vma, address, pvmw->pmd);	3165	update_mmu_cache_pmd(vma, address, pvmw->pmd);

mm/hugetlb.c+9−6

 	}
 	set_compound_order(page, 0);
+#ifdef CONFIG_64BIT
 	page[1].compound_nr = 0;
+#endif
 	__ClearPageHead(page);
 }
 	for (; j < nr_pages; j++, p = mem_map_next(p, page, j))
 		__ClearPageReserved(p);
 	set_compound_order(page, 0);
+#ifdef CONFIG_64BIT
 	page[1].compound_nr = 0;
+#endif
 	__ClearPageHead(page);
 	return false;
 }
 			set_page_dirty(page);
 		hugetlb_count_sub(pages_per_huge_page(h), mm);
-		page_remove_rmap(page, true);
+		page_remove_rmap(page, vma, true);
 		spin_unlock(ptl);
 		tlb_remove_page_size(tlb, page, huge_page_size(h));
 		/* Break COW */
 		huge_ptep_clear_flush(vma, haddr, ptep);
 		mmu_notifier_invalidate_range(mm, range.start, range.end);
-		page_remove_rmap(old_page, true);
+		page_remove_rmap(old_page, vma, true);
 		hugepage_add_new_anon_rmap(new_page, vma, haddr);
 		set_huge_pte_at(mm, haddr, ptep,
 				make_huge_pte(vma, new_page, 1));
 		if (pages) {
 			/*
-			 * try_grab_compound_head() should always succeed here,
+			 * try_grab_folio() should always succeed here,
 			 * because: a) we hold the ptl lock, and b) we've just
 			 * checked that the huge page is present in the page
 			 * tables. If the huge page is present, then the tail
 			 * any way. So this page must be available at this
 			 * point, unless the page refcount overflowed:
 			 */
-			if (WARN_ON_ONCE(!try_grab_compound_head(pages[i],
+			if (WARN_ON_ONCE(!try_grab_folio(pages[i], refs,
-								 refs,
+							 flags))) {
-								 flags))) {
 				spin_unlock(ptl);
 				remainder = 0;
 				err = -ENOMEM;

mm/internal.h+79−38

 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/pagemap.h>
+#include <linux/rmap.h>
 #include <linux/tracepoint-defs.h>
 struct folio_batch;
 vm_fault_t do_swap_page(struct vm_fault *vmf);
 void folio_rotate_reclaimable(struct folio *folio);
 bool __folio_end_writeback(struct folio *folio);
+void deactivate_file_folio(struct folio *folio);
 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
 		unsigned long floor, unsigned long ceiling);
 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);
-static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
-{
-	return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP));
-}
 struct zap_details;
 void unmap_page_range(struct mmu_gather *tlb,
 			     struct vm_area_struct *vma,
 			     unsigned long addr, unsigned long end,
 			     struct zap_details *details);
-void do_page_cache_ra(struct readahead_control *, unsigned long nr_to_read,
+void page_cache_ra_order(struct readahead_control *, struct file_ra_state *,
-		unsigned long lookahead_size);
+		unsigned int order);
 void force_page_cache_ra(struct readahead_control *, unsigned long nr);
 static inline void force_page_cache_readahead(struct address_space *mapping,
 		struct file *file, pgoff_t index, unsigned long nr_to_read)
 int truncate_inode_folio(struct address_space *mapping, struct folio *folio);
 bool truncate_inode_partial_folio(struct folio *folio, loff_t start,
 		loff_t end);
+long invalidate_inode_page(struct page *page);
+unsigned long invalidate_mapping_pagevec(struct address_space *mapping,
+		pgoff_t start, pgoff_t end, unsigned long *nr_pagevec);
 /**
  * folio_evictable - Test whether a folio is evictable.
 /*
  * in mm/vmscan.c:
  */
-extern int isolate_lru_page(struct page *page);
+int isolate_lru_page(struct page *page);
-extern void putback_lru_page(struct page *page);
+int folio_isolate_lru(struct folio *folio);
+void putback_lru_page(struct page *page);
+void folio_putback_lru(struct folio *folio);
 extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason);
 /*
 void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
 		struct vm_area_struct *prev);
 void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma);
+struct anon_vma *folio_anon_vma(struct folio *folio);
 #ifdef CONFIG_MMU
 void unmap_mapping_folio(struct folio *folio);
 extern long faultin_vma_page_range(struct vm_area_struct *vma,
 				   unsigned long start, unsigned long end,
 				   bool write, int *locked);
-extern void munlock_vma_pages_range(struct vm_area_struct *vma,
-			unsigned long start, unsigned long end);
-static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
-{
-	munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
-}
-/*
- * must be called with vma's mmap_lock held for read or write, and page locked.
- */
-extern void mlock_vma_page(struct page *page);
-extern unsigned int munlock_vma_page(struct page *page);
 extern int mlock_future_check(struct mm_struct *mm, unsigned long flags,
 			      unsigned long len);
 /*
- * Clear the page's PageMlocked().  This can be useful in a situation where
+ * mlock_vma_page() and munlock_vma_page():
- * we want to unconditionally remove a page from the pagecache -- e.g.,
+ * should be called with vma's mmap_lock held for read or write,
- * on truncation or freeing.
+ * under page table lock for the pte/pmd being added or removed.
  *
- * It is legal to call this function for any page, mlocked or not.
+ * mlock is usually called at the end of page_add_*_rmap(),
- * If called for a page that is still mapped by mlocked vmas, all we do
+ * munlock at the end of page_remove_rmap(); but new anon
- * is revert to lazy LRU behaviour -- semantics are not broken.
+ * pages are managed by lru_cache_add_inactive_or_unevictable()
+ * calling mlock_new_page().
+ *
+ * @compound is used to include pmd mappings of THPs, but filter out
+ * pte mappings of THPs, which cannot be consistently counted: a pte
+ * mapping of the THP head cannot be distinguished by the page alone.
  */
-extern void clear_page_mlock(struct page *page);
+void mlock_folio(struct folio *folio);
+static inline void mlock_vma_folio(struct folio *folio,
+			struct vm_area_struct *vma, bool compound)
+{
+	/*
+	 * The VM_SPECIAL check here serves two purposes.
+	 * 1) VM_IO check prevents migration from double-counting during mlock.
+	 * 2) Although mmap_region() and mlock_fixup() take care that VM_LOCKED
+	 *    is never left set on a VM_SPECIAL vma, there is an interval while
+	 *    file->f_op->mmap() is using vm_insert_page(s), when VM_LOCKED may
+	 *    still be set while VM_SPECIAL bits are added: so ignore it then.
+	 */
+	if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED) &&
+	    (compound || !folio_test_large(folio)))
+		mlock_folio(folio);
+}
+static inline void mlock_vma_page(struct page *page,
+			struct vm_area_struct *vma, bool compound)
+{
+	mlock_vma_folio(page_folio(page), vma, compound);
+}
+void munlock_page(struct page *page);
+static inline void munlock_vma_page(struct page *page,
+			struct vm_area_struct *vma, bool compound)
+{
+	if (unlikely(vma->vm_flags & VM_LOCKED) &&
+	    (compound || !PageTransCompound(page)))
+		munlock_page(page);
+}
+void mlock_new_page(struct page *page);
+bool need_mlock_page_drain(int cpu);
+void mlock_page_drain(int cpu);
 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
 }
 /*
- * Then at what user virtual address will none of the page be found in vma?
+ * Then at what user virtual address will none of the range be found in vma?
  * Assumes that vma_address() already returned a good starting address.
- * If page is a compound head, the entire compound page is considered.
  */
-static inline unsigned long
+static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw)
-vma_address_end(struct page *page, struct vm_area_struct *vma)
 {
+	struct vm_area_struct *vma = pvmw->vma;
 	pgoff_t pgoff;
 	unsigned long address;
-	VM_BUG_ON_PAGE(PageKsm(page), page);	/* KSM page->index unusable */
+	/* Common case, plus ->pgoff is invalid for KSM */
-	pgoff = page_to_pgoff(page) + compound_nr(page);
+	if (pvmw->nr_pages == 1)
+		return pvmw->address + PAGE_SIZE;
+	pgoff = pvmw->pgoff + pvmw->nr_pages;
 	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
 	/* Check for address beyond vma (or wrapped through 0?) */
 	if (address < vma->vm_start || address > vma->vm_end)
 }
 #else /* !CONFIG_MMU */
 static inline void unmap_mapping_folio(struct folio *folio) { }
-static inline void clear_page_mlock(struct page *page) { }
+static inline void mlock_vma_page(struct page *page,
-static inline void mlock_vma_page(struct page *page) { }
+			struct vm_area_struct *vma, bool compound) { }
+static inline void munlock_vma_page(struct page *page,
+			struct vm_area_struct *vma, bool compound) { }
+static inline void mlock_new_page(struct page *page) { }
+static inline bool need_mlock_page_drain(int cpu) { return false; }
+static inline void mlock_page_drain(int cpu) { }
 static inline void vunmap_range_noflush(unsigned long start, unsigned long end)
 {
 }
 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
 		      unsigned long addr, int page_nid, int *flags);
+void free_zone_device_page(struct page *page);
+/*
+ * mm/gup.c
+ */
+struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags);
 DECLARE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
 #endif	/* __MM_INTERNAL_H */

mm/khugepaged.c+11−4

 			 */
 			spin_lock(ptl);
 			ptep_clear(vma->vm_mm, address, _pte);
-			page_remove_rmap(src_page, false);
+			page_remove_rmap(src_page, vma, false);
 			spin_unlock(ptl);
 			free_page_and_swap_cache(src_page);
 		}
 		if (pte_none(*pte))
 			continue;
 		page = vm_normal_page(vma, addr, *pte);
-		page_remove_rmap(page, false);
+		page_remove_rmap(page, vma, false);
 	}
 	pte_unmap_unlock(start_pte, ptl);
 		}
 		if (page_mapped(page))
-			unmap_mapping_pages(mapping, index, 1, false);
+			try_to_unmap(page_folio(page),
+					TTU_IGNORE_MLOCK | TTU_BATCH_FLUSH);
 		xas_lock_irq(&xas);
 		xas_set(&xas, index);
 		VM_BUG_ON_PAGE(page != xas_load(&xas), page);
-		VM_BUG_ON_PAGE(page_mapped(page), page);
 		/*
 		 * The page is expected to have page_count() == 3:
 	xas_unlock_irq(&xas);
 xa_unlocked:
+	/*
+	 * If collapse is successful, flush must be done now before copying.
+	 * If collapse is unsuccessful, does flush actually need to be done?
+	 * Do it anyway, to clear the state.
+	 */
+	try_to_unmap_flush();
 	if (result == SCAN_SUCCEED) {
 		struct page *page, *tmp;

mm/ksm.c+10−22

 			      pte_t *orig_pte)
 {
 	struct mm_struct *mm = vma->vm_mm;
-	struct page_vma_mapped_walk pvmw = {
+	DEFINE_PAGE_VMA_WALK(pvmw, page, vma, 0, 0);
-		.page = page,
-		.vma = vma,
-	};
 	int swapped;
 	int err = -EFAULT;
 	struct mmu_notifier_range range;
 	ptep_clear_flush(vma, addr, ptep);
 	set_pte_at_notify(mm, addr, ptep, newpte);
-	page_remove_rmap(page, false);
+	page_remove_rmap(page, vma, false);
 	if (!page_mapped(page))
 		try_to_free_swap(page);
 	put_page(page);
 			err = replace_page(vma, page, kpage, orig_pte);
 	}
-	if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {
-		munlock_vma_page(page);
-		if (!PageMlocked(kpage)) {
-			unlock_page(page);
-			lock_page(kpage);
-			mlock_vma_page(kpage);
-			page = kpage;		/* for final unlock */
-		}
-	}
 out_unlock:
 	unlock_page(page);
 out:
 struct page *ksm_might_need_to_copy(struct page *page,
 			struct vm_area_struct *vma, unsigned long address)
 {
-	struct anon_vma *anon_vma = page_anon_vma(page);
+	struct folio *folio = page_folio(page);
+	struct anon_vma *anon_vma = folio_anon_vma(folio);
 	struct page *new_page;
 	if (PageKsm(page)) {
 	return new_page;
 }
-void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)
+void rmap_walk_ksm(struct folio *folio, const struct rmap_walk_control *rwc)
 {
 	struct stable_node *stable_node;
 	struct rmap_item *rmap_item;
 	int search_new_forks = 0;
-	VM_BUG_ON_PAGE(!PageKsm(page), page);
+	VM_BUG_ON_FOLIO(!folio_test_ksm(folio), folio);
 	/*
 	 * Rely on the page lock to protect against concurrent modifications
 	 * to that page's node of the stable tree.
 	 */
-	VM_BUG_ON_PAGE(!PageLocked(page), page);
+	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
-	stable_node = page_stable_node(page);
+	stable_node = folio_stable_node(folio);
 	if (!stable_node)
 		return;
 again:
 			if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
 				continue;
-			if (!rwc->rmap_one(page, vma, addr, rwc->arg)) {
+			if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) {
 				anon_vma_unlock_read(anon_vma);
 				return;
 			}
-			if (rwc->done && rwc->done(page)) {
+			if (rwc->done && rwc->done(folio)) {
 				anon_vma_unlock_read(anon_vma);
 				return;
 			}

mm/madvise.c+5−0

 	tlb_end_vma(tlb, vma);
 }
+static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
+{
+	return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP));
+}
 static long madvise_cold(struct vm_area_struct *vma,
 			struct vm_area_struct **prev,
 			unsigned long start_addr, unsigned long end_addr)

mm/memcontrol.c+16−21

 #include <linux/fs.h>
 #include <linux/seq_file.h>
 #include <linux/vmpressure.h>
+#include <linux/memremap.h>
 #include <linux/mm_inline.h>
 #include <linux/swap_cgroup.h>
 #include <linux/cpu.h>
  * @nr_pages: positive when adding or negative when removing
  *
  * This function must be called under lru_lock, just before a page is added
- * to or just after a page is removed from an lru list (that ordering being
+ * to or just after a page is removed from an lru list.
- * so as to allow it to check that lru_size 0 is consistent with list_empty).
  */
 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
 				int zid, int nr_pages)
 		return NULL;
 	/*
-	 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
+	 * Handle device private pages that are not accessible by the CPU, but
-	 * a device and because they are not accessible by CPU they are store
+	 * stored as special swap entries in the page table.
-	 * as special swap entry in the CPU page table.
 	 */
 	if (is_device_private_entry(ent)) {
 		page = pfn_swap_entry_to_page(ent);
-		/*
+		if (!get_page_unless_zero(page))
-		 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
-		 * a refcount of 1 when free (unlike normal page)
-		 */
-		if (!page_ref_add_unless(page, 1, 1))
 			return NULL;
 		return page;
 	}
 /**
  * mem_cgroup_swapout - transfer a memsw charge to swap
- * @page: page whose memsw charge to transfer
+ * @folio: folio whose memsw charge to transfer
  * @entry: swap entry to move the charge to
  *
- * Transfer the memsw charge of @page to @entry.
+ * Transfer the memsw charge of @folio to @entry.
  */
-void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
+void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
 {
 	struct mem_cgroup *memcg, *swap_memcg;
 	unsigned int nr_entries;
 	unsigned short oldid;
-	VM_BUG_ON_PAGE(PageLRU(page), page);
+	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
-	VM_BUG_ON_PAGE(page_count(page), page);
+	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
 	if (mem_cgroup_disabled())
 		return;
 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
 		return;
-	memcg = page_memcg(page);
+	memcg = folio_memcg(folio);
-	VM_WARN_ON_ONCE_PAGE(!memcg, page);
+	VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
 	if (!memcg)
 		return;
 	 * ancestor for the swap instead and transfer the memory+swap charge.
 	 */
 	swap_memcg = mem_cgroup_id_get_online(memcg);
-	nr_entries = thp_nr_pages(page);
+	nr_entries = folio_nr_pages(folio);
 	/* Get references for the tail pages, too */
 	if (nr_entries > 1)
 		mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
 	oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
 				   nr_entries);
-	VM_BUG_ON_PAGE(oldid, page);
+	VM_BUG_ON_FOLIO(oldid, folio);
 	mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
-	page->memcg_data = 0;
+	folio->memcg_data = 0;
 	if (!mem_cgroup_is_root(memcg))
 		page_counter_uncharge(&memcg->memory, nr_entries);
 	memcg_stats_lock();
 	mem_cgroup_charge_statistics(memcg, -nr_entries);
 	memcg_stats_unlock();
-	memcg_check_events(memcg, page_to_nid(page));
+	memcg_check_events(memcg, folio_nid(folio));
 	css_put(&memcg->css);
 }

mm/memory-failure.c+7−5

 static void collect_procs_anon(struct page *page, struct list_head *to_kill,
 				int force_early)
 {
+	struct folio *folio = page_folio(page);
 	struct vm_area_struct *vma;
 	struct task_struct *tsk;
 	struct anon_vma *av;
 	pgoff_t pgoff;
-	av = page_lock_anon_vma_read(page);
+	av = folio_lock_anon_vma_read(folio);
 	if (av == NULL)	/* Not actually mapped anymore */
 		return;
 static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
 				  int flags, struct page *hpage)
 {
+	struct folio *folio = page_folio(hpage);
 	enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_SYNC;
 	struct address_space *mapping;
 	LIST_HEAD(tokill);
 		 */
 		mapping = hugetlb_page_mapping_lock_write(hpage);
 		if (mapping) {
-			try_to_unmap(hpage, ttu|TTU_RMAP_LOCKED);
+			try_to_unmap(folio, ttu|TTU_RMAP_LOCKED);
 			i_mmap_unlock_write(mapping);
 		} else
 			pr_info("Memory failure: %#lx: could not lock mapping for mapped huge page\n", pfn);
 	} else {
-		try_to_unmap(hpage, ttu);
+		try_to_unmap(folio, ttu);
 	}
 	unmap_success = !page_mapped(hpage);
  */
 static int __soft_offline_page(struct page *page)
 {
-	int ret = 0;
+	long ret = 0;
 	unsigned long pfn = page_to_pfn(page);
 	struct page *hpage = compound_head(page);
 	char const *msg_page[] = {"page", "hugepage"};
 			if (!list_empty(&pagelist))
 				putback_movable_pages(&pagelist);
-			pr_info("soft offline: %#lx: %s migration failed %d, type %pGp\n",
+			pr_info("soft offline: %#lx: %s migration failed %ld, type %pGp\n",
 				pfn, msg_page[huge], ret, &page->flags);
 			if (ret > 0)
 				ret = -EBUSY;

mm/memory.c+15−28

 	set_pte_at(vma->vm_mm, address, ptep, pte);
-	if (vma->vm_flags & VM_LOCKED)
-		mlock_vma_page(page);
 	/*
 	 * No need to invalidate - it was non-present before. However
 	 * secondary CPUs may have mappings that need invalidating.
 					mark_page_accessed(page);
 			}
 			rss[mm_counter(page)]--;
-			page_remove_rmap(page, false);
+			page_remove_rmap(page, vma, false);
 			if (unlikely(page_mapcount(page) < 0))
 				print_bad_pte(vma, addr, ptent, page);
 			if (unlikely(__tlb_remove_page(tlb, page))) {
 				continue;
 			rss[mm_counter(page)]--;
 			if (is_device_private_entry(entry))
-				page_remove_rmap(page, false);
+				page_remove_rmap(page, vma, false);
 			put_page(page);
 		} else if (!non_swap_entry(entry)) {
 			/* Genuine swap entry, hence a private anon page */
 	return 0;
 }
-static int insert_page_into_pte_locked(struct mm_struct *mm, pte_t *pte,
+static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
 			unsigned long addr, struct page *page, pgprot_t prot)
 {
 	if (!pte_none(*pte))
 		return -EBUSY;
 	/* Ok, finally just insert the thing.. */
 	get_page(page);
-	inc_mm_counter_fast(mm, mm_counter_file(page));
+	inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
-	page_add_file_rmap(page, false);
+	page_add_file_rmap(page, vma, false);
-	set_pte_at(mm, addr, pte, mk_pte(page, prot));
+	set_pte_at(vma->vm_mm, addr, pte, mk_pte(page, prot));
 	return 0;
 }
 static int insert_page(struct vm_area_struct *vma, unsigned long addr,
 			struct page *page, pgprot_t prot)
 {
-	struct mm_struct *mm = vma->vm_mm;
 	int retval;
 	pte_t *pte;
 	spinlock_t *ptl;
 	if (retval)
 		goto out;
 	retval = -ENOMEM;
-	pte = get_locked_pte(mm, addr, &ptl);
+	pte = get_locked_pte(vma->vm_mm, addr, &ptl);
 	if (!pte)
 		goto out;
-	retval = insert_page_into_pte_locked(mm, pte, addr, page, prot);
+	retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
 	pte_unmap_unlock(pte, ptl);
 out:
 	return retval;
 }
 #ifdef pte_index
-static int insert_page_in_batch_locked(struct mm_struct *mm, pte_t *pte,
+static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
 			unsigned long addr, struct page *page, pgprot_t prot)
 {
 	int err;
 	err = validate_page_before_insert(page);
 	if (err)
 		return err;
-	return insert_page_into_pte_locked(mm, pte, addr, page, prot);
+	return insert_page_into_pte_locked(vma, pte, addr, page, prot);
 }
 /* insert_pages() amortizes the cost of spinlock operations
 		start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
 		for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
-			int err = insert_page_in_batch_locked(mm, pte,
+			int err = insert_page_in_batch_locked(vma, pte,
 				addr, pages[curr_page_idx], prot);
 			if (unlikely(err)) {
 				pte_unmap_unlock(start_pte, pte_lock);
 			 * mapcount is visible. So transitively, TLBs to
 			 * old page will be flushed before it can be reused.
 			 */
-			page_remove_rmap(old_page, false);
+			page_remove_rmap(old_page, vma, false);
 		}
 		/* Free the old page.. */
 	 */
 	mmu_notifier_invalidate_range_only_end(&range);
 	if (old_page) {
-		/*
-		 * Don't let another task, with possibly unlocked vma,
-		 * keep the mlocked page.
-		 */
-		if (page_copied && (vma->vm_flags & VM_LOCKED)) {
-			lock_page(old_page);	/* LRU manipulation */
-			if (PageMlocked(old_page))
-				munlock_vma_page(old_page);
-			unlock_page(old_page);
-		}
 		if (page_copied)
 			free_swap_cache(old_page);
 		put_page(old_page);
 		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
 	add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
-	page_add_file_rmap(page, true);
+	page_add_file_rmap(page, vma, true);
 	/*
 	 * deposit and withdraw with pmd lock held
 	 */
 		lru_cache_add_inactive_or_unevictable(page, vma);
 	} else {
 		inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
-		page_add_file_rmap(page, false);
+		page_add_file_rmap(page, vma, false);
 	}
 	set_pte_at(vma->vm_mm, addr, vmf->pte, entry);
 }

mm/memory_hotplug.c+8−5

 				      DEFAULT_RATELIMIT_BURST);
 	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
+		struct folio *folio;
 		if (!pfn_valid(pfn))
 			continue;
 		page = pfn_to_page(pfn);
-		head = compound_head(page);
+		folio = page_folio(page);
+		head = &folio->page;
 		if (PageHuge(page)) {
 			pfn = page_to_pfn(head) + compound_nr(head) - 1;
 		 * the unmap as the catch all safety net).
 		 */
 		if (PageHWPoison(page)) {
-			if (WARN_ON(PageLRU(page)))
+			if (WARN_ON(folio_test_lru(folio)))
-				isolate_lru_page(page);
+				folio_isolate_lru(folio);
-			if (page_mapped(page))
+			if (folio_mapped(folio))
-				try_to_unmap(page, TTU_IGNORE_MLOCK);
+				try_to_unmap(folio, TTU_IGNORE_MLOCK);
 			continue;
 		}

mm/memremap.c+34−31

 #include <linux/io.h>
 #include <linux/kasan.h>
 #include <linux/memory_hotplug.h>
-#include <linux/mm.h>
+#include <linux/memremap.h>
 #include <linux/pfn_t.h>
 #include <linux/swap.h>
 #include <linux/mmzone.h>
 #include <linux/types.h>
 #include <linux/wait_bit.h>
 #include <linux/xarray.h>
+#include "internal.h"
 static DEFINE_XARRAY(pgmap_array);
 EXPORT_SYMBOL_GPL(memremap_compat_align);
 #endif
-#ifdef CONFIG_DEV_PAGEMAP_OPS
+#ifdef CONFIG_FS_DAX
 DEFINE_STATIC_KEY_FALSE(devmap_managed_key);
 EXPORT_SYMBOL(devmap_managed_key);
 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
 {
-	if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
+	if (pgmap->type == MEMORY_DEVICE_FS_DAX)
-	    pgmap->type == MEMORY_DEVICE_FS_DAX)
 		static_branch_dec(&devmap_managed_key);
 }
 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)
 {
-	if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
+	if (pgmap->type == MEMORY_DEVICE_FS_DAX)
-	    pgmap->type == MEMORY_DEVICE_FS_DAX)
 		static_branch_inc(&devmap_managed_key);
 }
 #else
 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
 {
 }
-#endif /* CONFIG_DEV_PAGEMAP_OPS */
+#endif /* CONFIG_FS_DAX */
 static void pgmap_array_delete(struct range *range)
 {
 	return (range->start + range_len(range)) >> PAGE_SHIFT;
 }
-static unsigned long pfn_next(struct dev_pagemap *pgmap, unsigned long pfn)
-{
-	if (pfn % (1024 << pgmap->vmemmap_shift))
-		cond_resched();
-	return pfn + pgmap_vmemmap_nr(pgmap);
-}
 static unsigned long pfn_len(struct dev_pagemap *pgmap, unsigned long range_id)
 {
 	return (pfn_end(pgmap, range_id) -
 		pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift;
 }
-#define for_each_device_pfn(pfn, map, i) \
-	for (pfn = pfn_first(map, i); pfn < pfn_end(map, i); \
-	     pfn = pfn_next(map, pfn))
 static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
 {
 	struct range *range = &pgmap->ranges[range_id];
 void memunmap_pages(struct dev_pagemap *pgmap)
 {
-	unsigned long pfn;
 	int i;
 	percpu_ref_kill(&pgmap->ref);
 	for (i = 0; i < pgmap->nr_range; i++)
-		for_each_device_pfn(pfn, pgmap, i)
+		percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i));
-			put_page(pfn_to_page(pfn));
 	wait_for_completion(&pgmap->done);
 	percpu_ref_exit(&pgmap->ref);
 		}
 		break;
 	case MEMORY_DEVICE_FS_DAX:
-		if (!IS_ENABLED(CONFIG_ZONE_DEVICE) ||
+		if (IS_ENABLED(CONFIG_FS_DAX_LIMITED)) {
-		    IS_ENABLED(CONFIG_FS_DAX_LIMITED)) {
 			WARN(1, "File system DAX not supported\n");
 			return ERR_PTR(-EINVAL);
 		}
 }
 EXPORT_SYMBOL_GPL(get_dev_pagemap);
-#ifdef CONFIG_DEV_PAGEMAP_OPS
+void free_zone_device_page(struct page *page)
-void free_devmap_managed_page(struct page *page)
 {
-	/* notify page idle for dax */
+	if (WARN_ON_ONCE(!page->pgmap->ops || !page->pgmap->ops->page_free))
-	if (!is_device_private_page(page)) {
-		wake_up_var(&page->_refcount);
 		return;
-	}
 	__ClearPageWaiters(page);
 	mem_cgroup_uncharge(page_folio(page));
 	/*
-	 * When a device_private page is freed, the page->mapping field
+	 * When a device managed page is freed, the page->mapping field
 	 * may still contain a (stale) mapping value. For example, the
 	 * lower bits of page->mapping may still identify the page as an
 	 * anonymous page. Ultimately, this entire field is just stale
 	 */
 	page->mapping = NULL;
 	page->pgmap->ops->page_free(page);
+	/*
+	 * Reset the page count to 1 to prepare for handing out the page again.
+	 */
+	set_page_count(page, 1);
+}
+#ifdef CONFIG_FS_DAX
+bool __put_devmap_managed_page(struct page *page)
+{
+	if (page->pgmap->type != MEMORY_DEVICE_FS_DAX)
+		return false;
+	/*
+	 * fsdax page refcounts are 1-based, rather than 0-based: if
+	 * refcount is 1, then the page is free and the refcount is
+	 * stable because nobody holds a reference on the page.
+	 */
+	if (page_ref_dec_return(page) == 1)
+		wake_up_var(&page->_refcount);
+	return true;
 }
-#endif /* CONFIG_DEV_PAGEMAP_OPS */
+EXPORT_SYMBOL(__put_devmap_managed_page);
+#endif /* CONFIG_FS_DAX */

mm/migrate.c+56−814

@@ -38,12 +38,10 @@ · 37 unmodified lines
38	#include <linux/hugetlb.h>	38	#include <linux/hugetlb.h>
39	#include <linux/hugetlb_cgroup.h>	39	#include <linux/hugetlb_cgroup.h>
40	#include <linux/gfp.h>	40	#include <linux/gfp.h>
41	#include <linux/pagewalk.h>
42	#include <linux/pfn_t.h>	41	#include <linux/pfn_t.h>
43	#include <linux/memremap.h>	42	#include <linux/memremap.h>
44	#include <linux/userfaultfd_k.h>	43	#include <linux/userfaultfd_k.h>
45	#include <linux/balloon_compaction.h>	44	#include <linux/balloon_compaction.h>
46	#include <linux/mmu_notifier.h>
47	#include <linux/page_idle.h>	45	#include <linux/page_idle.h>
48	#include <linux/page_owner.h>	46	#include <linux/page_owner.h>
49	#include <linux/sched/mm.h>	47	#include <linux/sched/mm.h>
@@ -174,37 +172,33 @@ void putback_movable_pages(struct list_head *l) · 124 unmodified lines
174	/*	172	/*
175	* Restore a potential migration pte to a working pte entry	173	* Restore a potential migration pte to a working pte entry
176	*/	174	*/
177	static bool remove_migration_pte(struct page page, struct vm_area_struct vma,	175	static bool remove_migration_pte(struct folio *folio,
178	unsigned long addr, void *old)	176	struct vm_area_struct vma, unsigned long addr, void old)
179	{	177	{
180	struct page_vma_mapped_walk pvmw = {	178	DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC \| PVMW_MIGRATION);
181	.page = old,
182	.vma = vma,
183	.address = addr,
184	.flags = PVMW_SYNC \| PVMW_MIGRATION,
185	};
186	struct page *new;
187	pte_t pte;
188	swp_entry_t entry;
189		179
190	VM_BUG_ON_PAGE(PageTail(page), page);
191	while (page_vma_mapped_walk(&pvmw)) {	180	while (page_vma_mapped_walk(&pvmw)) {
192	if (PageKsm(page))	181	pte_t pte;
193	new = page;	182	swp_entry_t entry;
194	else	183	struct page *new;
195	new = page - pvmw.page->index +	184	unsigned long idx = 0;
196	linear_page_index(vma, pvmw.address);	185
		186	/* pgoff is invalid for ksm pages, but they are never large */
		187	if (folio_test_large(folio) && !folio_test_hugetlb(folio))
		188	idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff;
		189	new = folio_page(folio, idx);
197		190
198	#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION	191	#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
199	/* PMD-mapped THP migration entry */	192	/* PMD-mapped THP migration entry */
200	if (!pvmw.pte) {	193	if (!pvmw.pte) {
201	VM_BUG_ON_PAGE(PageHuge(page) \|\| !PageTransCompound(page), page);	194	VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) \|\|
		195	!folio_test_pmd_mappable(folio), folio);
202	remove_migration_pmd(&pvmw, new);	196	remove_migration_pmd(&pvmw, new);
203	continue;	197	continue;
204	}	198	}
205	#endif	199	#endif
206		200
207	get_page(new);	201	folio_get(folio);
208	pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));	202	pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
209	if (pte_swp_soft_dirty(*pvmw.pte))	203	if (pte_swp_soft_dirty(*pvmw.pte))
210	pte = pte_mksoft_dirty(pte);	204	pte = pte_mksoft_dirty(pte);
@@ -233,12 +227,12 @@ static bool remove_migration_pte(struct page page, struct vm_area_struct vma, · 22 unmodified lines
233	}	227	}
234		228
235	#ifdef CONFIG_HUGETLB_PAGE	229	#ifdef CONFIG_HUGETLB_PAGE
236	if (PageHuge(new)) {	230	if (folio_test_hugetlb(folio)) {
237	unsigned int shift = huge_page_shift(hstate_vma(vma));	231	unsigned int shift = huge_page_shift(hstate_vma(vma));
238		232
239	pte = pte_mkhuge(pte);	233	pte = pte_mkhuge(pte);
240	pte = arch_make_huge_pte(pte, shift, vma->vm_flags);	234	pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
241	if (PageAnon(new))	235	if (folio_test_anon(folio))
242	hugepage_add_anon_rmap(new, vma, pvmw.address);	236	hugepage_add_anon_rmap(new, vma, pvmw.address);
243	else	237	else
244	page_dup_rmap(new, true);	238	page_dup_rmap(new, true);
@@ -246,17 +240,14 @@ static bool remove_migration_pte(struct page page, struct vm_area_struct vma, · 1 unmodified line
246	} else	240	} else
247	#endif	241	#endif
248	{	242	{
249	if (PageAnon(new))	243	if (folio_test_anon(folio))
250	page_add_anon_rmap(new, vma, pvmw.address, false);	244	page_add_anon_rmap(new, vma, pvmw.address, false);
251	else	245	else
252	page_add_file_rmap(new, false);	246	page_add_file_rmap(new, vma, false);
253	set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);	247	set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
254	}	248	}
255	if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))	249	if (vma->vm_flags & VM_LOCKED)
256	mlock_vma_page(new);	250	mlock_page_drain(smp_processor_id());
257
258	if (PageTransHuge(page) && PageMlocked(page))
259	clear_page_mlock(page);
260		251
261	/* No need to invalidate - it was non-present before */	252	/* No need to invalidate - it was non-present before */
262	update_mmu_cache(vma, pvmw.address, pvmw.pte);	253	update_mmu_cache(vma, pvmw.address, pvmw.pte);
@@ -269,17 +260,17 @@ static bool remove_migration_pte(struct page page, struct vm_area_struct vma, · 6 unmodified lines
269	* Get rid of all migration entries and replace them by	260	* Get rid of all migration entries and replace them by
270	* references to the indicated page.	261	* references to the indicated page.
271	*/	262	*/
272	void remove_migration_ptes(struct page old, struct page new, bool locked)	263	void remove_migration_ptes(struct folio src, struct folio dst, bool locked)
273	{	264	{
274	struct rmap_walk_control rwc = {	265	struct rmap_walk_control rwc = {
275	.rmap_one = remove_migration_pte,	266	.rmap_one = remove_migration_pte,
276	.arg = old,	267	.arg = src,
277	};	268	};
278		269
279	if (locked)	270	if (locked)
280	rmap_walk_locked(new, &rwc);	271	rmap_walk_locked(dst, &rwc);
281	else	272	else
282	rmap_walk(new, &rwc);	273	rmap_walk(dst, &rwc);
283	}	274	}
284		275
285	/*	276	/*
@@ -342,14 +333,8 @@ static int expected_page_refs(struct address_space mapping, struct page page) · 56 unmodified lines
342	{	333	{
343	int expected_count = 1;	334	int expected_count = 1;
344		335
345	/*
346	* Device private pages have an extra refcount as they are
347	* ZONE_DEVICE pages.
348	*/
349	expected_count += is_device_private_page(page);
350	if (mapping)	336	if (mapping)
351	expected_count += compound_nr(page) + page_has_private(page);	337	expected_count += compound_nr(page) + page_has_private(page);
352
353	return expected_count;	338	return expected_count;
354	}	339	}
355		340
@@ -772,6 +757,7 @@ int buffer_migrate_page_norefs(struct address_space *mapping, · 416 unmodified lines
772	*/	757	*/
773	static int writeout(struct address_space mapping, struct page page)	758	static int writeout(struct address_space mapping, struct page page)
774	{	759	{
		760	struct folio *folio = page_folio(page);
775	struct writeback_control wbc = {	761	struct writeback_control wbc = {
776	.sync_mode = WB_SYNC_NONE,	762	.sync_mode = WB_SYNC_NONE,
777	.nr_to_write = 1,	763	.nr_to_write = 1,
@@ -797,7 +783,7 @@ static int writeout(struct address_space mapping, struct page page) · 19 unmodified lines
797	* At this point we know that the migration attempt cannot	783	* At this point we know that the migration attempt cannot
798	* be successful.	784	* be successful.
799	*/	785	*/
800	remove_migration_ptes(page, page, false);	786	remove_migration_ptes(folio, folio, false);
801		787
802	rc = mapping->a_ops->writepage(page, &wbc);	788	rc = mapping->a_ops->writepage(page, &wbc);
803		789
@@ -927,6 +913,8 @@ out: · 123 unmodified lines
927	static int __unmap_and_move(struct page page, struct page newpage,	913	static int __unmap_and_move(struct page page, struct page newpage,
928	int force, enum migrate_mode mode)	914	int force, enum migrate_mode mode)
929	{	915	{
		916	struct folio *folio = page_folio(page);
		917	struct folio *dst = page_folio(newpage);
930	int rc = -EAGAIN;	918	int rc = -EAGAIN;
931	bool page_was_mapped = false;	919	bool page_was_mapped = false;
932	struct anon_vma *anon_vma = NULL;	920	struct anon_vma *anon_vma = NULL;
@@ -1030,16 +1018,31 @@ static int __unmap_and_move(struct page page, struct page newpage, · 97 unmodified lines
1030	/* Establish migration ptes */	1018	/* Establish migration ptes */
1031	VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,	1019	VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
1032	page);	1020	page);
1033	try_to_migrate(page, 0);	1021	try_to_migrate(folio, 0);
1034	page_was_mapped = true;	1022	page_was_mapped = true;
1035	}	1023	}
1036		1024
1037	if (!page_mapped(page))	1025	if (!page_mapped(page))
1038	rc = move_to_new_page(newpage, page, mode);	1026	rc = move_to_new_page(newpage, page, mode);
1039		1027
		1028	/*
		1029	* When successful, push newpage to LRU immediately: so that if it
		1030	* turns out to be an mlocked page, remove_migration_ptes() will
		1031	* automatically build up the correct newpage->mlock_count for it.
		1032	*
		1033	* We would like to do something similar for the old page, when
		1034	* unsuccessful, and other cases when a page has been temporarily
		1035	* isolated from the unevictable LRU: but this case is the easiest.
		1036	*/
		1037	if (rc == MIGRATEPAGE_SUCCESS) {
		1038	lru_cache_add(newpage);
		1039	if (page_was_mapped)
		1040	lru_add_drain();
		1041	}
		1042
1040	if (page_was_mapped)	1043	if (page_was_mapped)
1041	remove_migration_ptes(page,	1044	remove_migration_ptes(folio,
1042	rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);	1045	rc == MIGRATEPAGE_SUCCESS ? dst : folio, false);
1043		1046
1044	out_unlock_both:	1047	out_unlock_both:
1045	unlock_page(newpage);	1048	unlock_page(newpage);
@@ -1050,20 +1053,12 @@ out_unlock: · 4 unmodified lines
1050	unlock_page(page);	1053	unlock_page(page);
1051	out:	1054	out:
1052	/*	1055	/*
1053	* If migration is successful, decrease refcount of the newpage	1056	* If migration is successful, decrease refcount of the newpage,
1054	* which will not free the page because new page owner increased	1057	* which will not free the page because new page owner increased
1055	* refcounter. As well, if it is LRU page, add the page to LRU	1058	* refcounter.
1056	* list in here. Use the old state of the isolated source page to
1057	* determine if we migrated a LRU page. newpage was already unlocked
1058	* and possibly modified by its owner - don't rely on the page
1059	* state.
1060	*/	1059	*/
1061	if (rc == MIGRATEPAGE_SUCCESS) {	1060	if (rc == MIGRATEPAGE_SUCCESS)
1062	if (unlikely(!is_lru))	1061	put_page(newpage);
1063	put_page(newpage);
1064	else
1065	putback_lru_page(newpage);
1066	}
1067		1062
1068	return rc;	1063	return rc;
1069	}	1064	}
@@ -1173,6 +1168,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 103 unmodified lines
1173	enum migrate_mode mode, int reason,	1168	enum migrate_mode mode, int reason,
1174	struct list_head *ret)	1169	struct list_head *ret)
1175	{	1170	{
		1171	struct folio dst, src = page_folio(hpage);
1176	int rc = -EAGAIN;	1172	int rc = -EAGAIN;
1177	int page_was_mapped = 0;	1173	int page_was_mapped = 0;
1178	struct page *new_hpage;	1174	struct page *new_hpage;
@@ -1200,6 +1196,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 21 unmodified lines
1200	new_hpage = get_new_page(hpage, private);	1196	new_hpage = get_new_page(hpage, private);
1201	if (!new_hpage)	1197	if (!new_hpage)
1202	return -ENOMEM;	1198	return -ENOMEM;
		1199	dst = page_folio(new_hpage);
1203		1200
1204	if (!trylock_page(hpage)) {	1201	if (!trylock_page(hpage)) {
1205	if (!force)	1202	if (!force)
@@ -1249,7 +1246,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 43 unmodified lines
1249	ttu \|= TTU_RMAP_LOCKED;	1246	ttu \|= TTU_RMAP_LOCKED;
1250	}	1247	}
1251		1248
1252	try_to_migrate(hpage, ttu);	1249	try_to_migrate(src, ttu);
1253	page_was_mapped = 1;	1250	page_was_mapped = 1;
1254		1251
1255	if (mapping_locked)	1252	if (mapping_locked)
@@ -1260,8 +1257,8 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 4 unmodified lines
1260	rc = move_to_new_page(new_hpage, hpage, mode);	1257	rc = move_to_new_page(new_hpage, hpage, mode);
1261		1258
1262	if (page_was_mapped)	1259	if (page_was_mapped)
1263	remove_migration_ptes(hpage,	1260	remove_migration_ptes(src,
1264	rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);	1261	rc == MIGRATEPAGE_SUCCESS ? dst : src, false);
1265		1262
1266	unlock_put_anon:	1263	unlock_put_anon:
1267	unlock_page(new_hpage);	1264	unlock_page(new_hpage);
@@ -2147,761 +2144,6 @@ out: · 879 unmodified lines
2147	#endif /* CONFIG_NUMA_BALANCING */	2144	#endif /* CONFIG_NUMA_BALANCING */
2148	#endif /* CONFIG_NUMA */	2145	#endif /* CONFIG_NUMA */
2149		2146
2150	#ifdef CONFIG_DEVICE_PRIVATE
2151	static int migrate_vma_collect_skip(unsigned long start,
2152	unsigned long end,
2153	struct mm_walk *walk)
2154	{
2155	struct migrate_vma *migrate = walk->private;
2156	unsigned long addr;
2157
2158	for (addr = start; addr < end; addr += PAGE_SIZE) {
2159	migrate->dst[migrate->npages] = 0;
2160	migrate->src[migrate->npages++] = 0;
2161	}
2162
2163	return 0;
2164	}
2165
2166	static int migrate_vma_collect_hole(unsigned long start,
2167	unsigned long end,
2168	__always_unused int depth,
2169	struct mm_walk *walk)
2170	{
2171	struct migrate_vma *migrate = walk->private;
2172	unsigned long addr;
2173
2174	/* Only allow populating anonymous memory. */
2175	if (!vma_is_anonymous(walk->vma))
2176	return migrate_vma_collect_skip(start, end, walk);
2177
2178	for (addr = start; addr < end; addr += PAGE_SIZE) {
2179	migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
2180	migrate->dst[migrate->npages] = 0;
2181	migrate->npages++;
2182	migrate->cpages++;
2183	}
2184
2185	return 0;
2186	}
2187
2188	static int migrate_vma_collect_pmd(pmd_t *pmdp,
2189	unsigned long start,
2190	unsigned long end,
2191	struct mm_walk *walk)
2192	{
2193	struct migrate_vma *migrate = walk->private;
2194	struct vm_area_struct *vma = walk->vma;
2195	struct mm_struct *mm = vma->vm_mm;
2196	unsigned long addr = start, unmapped = 0;
2197	spinlock_t *ptl;
2198	pte_t *ptep;
2199
2200	again:
2201	if (pmd_none(*pmdp))
2202	return migrate_vma_collect_hole(start, end, -1, walk);
2203
2204	if (pmd_trans_huge(*pmdp)) {
2205	struct page *page;
2206
2207	ptl = pmd_lock(mm, pmdp);
2208	if (unlikely(!pmd_trans_huge(*pmdp))) {
2209	spin_unlock(ptl);
2210	goto again;
2211	}
2212
2213	page = pmd_page(*pmdp);
2214	if (is_huge_zero_page(page)) {
2215	spin_unlock(ptl);
2216	split_huge_pmd(vma, pmdp, addr);
2217	if (pmd_trans_unstable(pmdp))
2218	return migrate_vma_collect_skip(start, end,
2219	walk);
2220	} else {
2221	int ret;
2222
2223	get_page(page);
2224	spin_unlock(ptl);
2225	if (unlikely(!trylock_page(page)))
2226	return migrate_vma_collect_skip(start, end,
2227	walk);
2228	ret = split_huge_page(page);
2229	unlock_page(page);
2230	put_page(page);
2231	if (ret)
2232	return migrate_vma_collect_skip(start, end,
2233	walk);
2234	if (pmd_none(*pmdp))
2235	return migrate_vma_collect_hole(start, end, -1,
2236	walk);
2237	}
2238	}
2239
2240	if (unlikely(pmd_bad(*pmdp)))
2241	return migrate_vma_collect_skip(start, end, walk);
2242
2243	ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2244	arch_enter_lazy_mmu_mode();
2245
2246	for (; addr < end; addr += PAGE_SIZE, ptep++) {
2247	unsigned long mpfn = 0, pfn;
2248	struct page *page;
2249	swp_entry_t entry;
2250	pte_t pte;
2251
2252	pte = *ptep;
2253
2254	if (pte_none(pte)) {
2255	if (vma_is_anonymous(vma)) {
2256	mpfn = MIGRATE_PFN_MIGRATE;
2257	migrate->cpages++;
2258	}
2259	goto next;
2260	}
2261
2262	if (!pte_present(pte)) {
2263	/*
2264	* Only care about unaddressable device page special
2265	* page table entry. Other special swap entries are not
2266	* migratable, and we ignore regular swapped page.
2267	*/
2268	entry = pte_to_swp_entry(pte);
2269	if (!is_device_private_entry(entry))
2270	goto next;
2271
2272	page = pfn_swap_entry_to_page(entry);
2273	if (!(migrate->flags &
2274	MIGRATE_VMA_SELECT_DEVICE_PRIVATE) \|\|
2275	page->pgmap->owner != migrate->pgmap_owner)
2276	goto next;
2277
2278	mpfn = migrate_pfn(page_to_pfn(page)) \|
2279	MIGRATE_PFN_MIGRATE;
2280	if (is_writable_device_private_entry(entry))
2281	mpfn \|= MIGRATE_PFN_WRITE;
2282	} else {
2283	if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
2284	goto next;
2285	pfn = pte_pfn(pte);
2286	if (is_zero_pfn(pfn)) {
2287	mpfn = MIGRATE_PFN_MIGRATE;
2288	migrate->cpages++;
2289	goto next;
2290	}
2291	page = vm_normal_page(migrate->vma, addr, pte);
2292	mpfn = migrate_pfn(pfn) \| MIGRATE_PFN_MIGRATE;
2293	mpfn \|= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
2294	}
2295
2296	/* FIXME support THP */
2297	if (!page \|\| !page->mapping \|\| PageTransCompound(page)) {
2298	mpfn = 0;
2299	goto next;
2300	}
2301
2302	/*
2303	* By getting a reference on the page we pin it and that blocks
2304	* any kind of migration. Side effect is that it "freezes" the
2305	* pte.
2306	*
2307	* We drop this reference after isolating the page from the lru
2308	* for non device page (device page are not on the lru and thus
2309	* can't be dropped from it).
2310	*/
2311	get_page(page);
2312
2313	/*
2314	* Optimize for the common case where page is only mapped once
2315	* in one process. If we can lock the page, then we can safely
2316	* set up a special migration page table entry now.
2317	*/
2318	if (trylock_page(page)) {
2319	pte_t swp_pte;
2320
2321	migrate->cpages++;
2322	ptep_get_and_clear(mm, addr, ptep);
2323
2324	/* Setup special migration page table entry */
2325	if (mpfn & MIGRATE_PFN_WRITE)
2326	entry = make_writable_migration_entry(
2327	page_to_pfn(page));
2328	else
2329	entry = make_readable_migration_entry(
2330	page_to_pfn(page));
2331	swp_pte = swp_entry_to_pte(entry);
2332	if (pte_present(pte)) {
2333	if (pte_soft_dirty(pte))
2334	swp_pte = pte_swp_mksoft_dirty(swp_pte);
2335	if (pte_uffd_wp(pte))
2336	swp_pte = pte_swp_mkuffd_wp(swp_pte);
2337	} else {
2338	if (pte_swp_soft_dirty(pte))
2339	swp_pte = pte_swp_mksoft_dirty(swp_pte);
2340	if (pte_swp_uffd_wp(pte))
2341	swp_pte = pte_swp_mkuffd_wp(swp_pte);
2342	}
2343	set_pte_at(mm, addr, ptep, swp_pte);
2344
2345	/*
2346	* This is like regular unmap: we remove the rmap and
2347	* drop page refcount. Page won't be freed, as we took
2348	* a reference just above.
2349	*/
2350	page_remove_rmap(page, false);
2351	put_page(page);
2352
2353	if (pte_present(pte))
2354	unmapped++;
2355	} else {
2356	put_page(page);
2357	mpfn = 0;
2358	}
2359
2360	next:
2361	migrate->dst[migrate->npages] = 0;
2362	migrate->src[migrate->npages++] = mpfn;
2363	}
2364	arch_leave_lazy_mmu_mode();
2365	pte_unmap_unlock(ptep - 1, ptl);
2366
2367	/* Only flush the TLB if we actually modified any entries */
2368	if (unmapped)
2369	flush_tlb_range(walk->vma, start, end);
2370
2371	return 0;
2372	}
2373
2374	static const struct mm_walk_ops migrate_vma_walk_ops = {
2375	.pmd_entry = migrate_vma_collect_pmd,
2376	.pte_hole = migrate_vma_collect_hole,
2377	};
2378
2379	/*
2380	* migrate_vma_collect() - collect pages over a range of virtual addresses
2381	* @migrate: migrate struct containing all migration information
2382	*
2383	* This will walk the CPU page table. For each virtual address backed by a
2384	* valid page, it updates the src array and takes a reference on the page, in
2385	* order to pin the page until we lock it and unmap it.
2386	*/
2387	static void migrate_vma_collect(struct migrate_vma *migrate)
2388	{
2389	struct mmu_notifier_range range;
2390
2391	/*
2392	* Note that the pgmap_owner is passed to the mmu notifier callback so
2393	* that the registered device driver can skip invalidating device
2394	* private page mappings that won't be migrated.
2395	*/
2396	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_MIGRATE, 0,
2397	migrate->vma, migrate->vma->vm_mm, migrate->start, migrate->end,
2398	migrate->pgmap_owner);
2399	mmu_notifier_invalidate_range_start(&range);
2400
2401	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
2402	&migrate_vma_walk_ops, migrate);
2403
2404	mmu_notifier_invalidate_range_end(&range);
2405	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
2406	}
2407
2408	/*
2409	* migrate_vma_check_page() - check if page is pinned or not
2410	* @page: struct page to check
2411	*
2412	* Pinned pages cannot be migrated. This is the same test as in
2413	* folio_migrate_mapping(), except that here we allow migration of a
2414	* ZONE_DEVICE page.
2415	*/
2416	static bool migrate_vma_check_page(struct page *page)
2417	{
2418	/*
2419	* One extra ref because caller holds an extra reference, either from
2420	* isolate_lru_page() for a regular page, or migrate_vma_collect() for
2421	* a device page.
2422	*/
2423	int extra = 1;
2424
2425	/*
2426	* FIXME support THP (transparent huge page), it is bit more complex to
2427	* check them than regular pages, because they can be mapped with a pmd
2428	* or with a pte (split pte mapping).
2429	*/
2430	if (PageCompound(page))
2431	return false;
2432
2433	/* Page from ZONE_DEVICE have one extra reference */
2434	if (is_zone_device_page(page))
2435	extra++;
2436
2437	/* For file back page */
2438	if (page_mapping(page))
2439	extra += 1 + page_has_private(page);
2440
2441	if ((page_count(page) - extra) > page_mapcount(page))
2442	return false;
2443
2444	return true;
2445	}
2446
2447	/*
2448	* migrate_vma_unmap() - replace page mapping with special migration pte entry
2449	* @migrate: migrate struct containing all migration information
2450	*
2451	* Isolate pages from the LRU and replace mappings (CPU page table pte) with a
2452	* special migration pte entry and check if it has been pinned. Pinned pages are
2453	* restored because we cannot migrate them.
2454	*
2455	* This is the last step before we call the device driver callback to allocate
2456	* destination memory and copy contents of original page over to new page.
2457	*/
2458	static void migrate_vma_unmap(struct migrate_vma *migrate)
2459	{
2460	const unsigned long npages = migrate->npages;
2461	unsigned long i, restore = 0;
2462	bool allow_drain = true;
2463
2464	lru_add_drain();
2465
2466	for (i = 0; i < npages; i++) {
2467	struct page *page = migrate_pfn_to_page(migrate->src[i]);
2468
2469	if (!page)
2470	continue;
2471
2472	/* ZONE_DEVICE pages are not on LRU */
2473	if (!is_zone_device_page(page)) {
2474	if (!PageLRU(page) && allow_drain) {
2475	/* Drain CPU's pagevec */
2476	lru_add_drain_all();
2477	allow_drain = false;
2478	}
2479
2480	if (isolate_lru_page(page)) {
2481	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2482	migrate->cpages--;
2483	restore++;
2484	continue;
2485	}
2486
2487	/* Drop the reference we took in collect */
2488	put_page(page);
2489	}
2490
2491	if (page_mapped(page))
2492	try_to_migrate(page, 0);
2493
2494	if (page_mapped(page) \|\| !migrate_vma_check_page(page)) {
2495	if (!is_zone_device_page(page)) {
2496	get_page(page);
2497	putback_lru_page(page);
2498	}
2499
2500	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2501	migrate->cpages--;
2502	restore++;
2503	continue;
2504	}
2505	}
2506
2507	for (i = 0; i < npages && restore; i++) {
2508	struct page *page = migrate_pfn_to_page(migrate->src[i]);
2509
2510	if (!page \|\| (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2511	continue;
2512
2513	remove_migration_ptes(page, page, false);
2514
2515	migrate->src[i] = 0;
2516	unlock_page(page);
2517	put_page(page);
2518	restore--;
2519	}
2520	}
2521
2522	/**
2523	* migrate_vma_setup() - prepare to migrate a range of memory
2524	* @args: contains the vma, start, and pfns arrays for the migration
2525	*
2526	* Returns: negative errno on failures, 0 when 0 or more pages were migrated
2527	* without an error.
2528	*
2529	* Prepare to migrate a range of memory virtual address range by collecting all
2530	* the pages backing each virtual address in the range, saving them inside the
2531	* src array. Then lock those pages and unmap them. Once the pages are locked
2532	* and unmapped, check whether each page is pinned or not. Pages that aren't
2533	* pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
2534	* corresponding src array entry. Then restores any pages that are pinned, by
2535	* remapping and unlocking those pages.
2536	*
2537	* The caller should then allocate destination memory and copy source memory to
2538	* it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
2539	* flag set). Once these are allocated and copied, the caller must update each
2540	* corresponding entry in the dst array with the pfn value of the destination
2541	* page and with MIGRATE_PFN_VALID. Destination pages must be locked via
2542	* lock_page().
2543	*
2544	* Note that the caller does not have to migrate all the pages that are marked
2545	* with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
2546	* device memory to system memory. If the caller cannot migrate a device page
2547	* back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
2548	* consequences for the userspace process, so it must be avoided if at all
2549	* possible.
2550	*
2551	* For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
2552	* do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
2553	* allowing the caller to allocate device memory for those unbacked virtual
2554	* addresses. For this the caller simply has to allocate device memory and
2555	* properly set the destination entry like for regular migration. Note that
2556	* this can still fail, and thus inside the device driver you must check if the
2557	* migration was successful for those entries after calling migrate_vma_pages(),
2558	* just like for regular migration.
2559	*
2560	* After that, the callers must call migrate_vma_pages() to go over each entry
2561	* in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
2562	* set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
2563	* then migrate_vma_pages() to migrate struct page information from the source
2564	* struct page to the destination struct page. If it fails to migrate the
2565	* struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
2566	* src array.
2567	*
2568	* At this point all successfully migrated pages have an entry in the src
2569	* array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
2570	* array entry with MIGRATE_PFN_VALID flag set.
2571	*
2572	* Once migrate_vma_pages() returns the caller may inspect which pages were
2573	* successfully migrated, and which were not. Successfully migrated pages will
2574	* have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
2575	*
2576	* It is safe to update device page table after migrate_vma_pages() because
2577	* both destination and source page are still locked, and the mmap_lock is held
2578	* in read mode (hence no one can unmap the range being migrated).
2579	*
2580	* Once the caller is done cleaning up things and updating its page table (if it
2581	* chose to do so, this is not an obligation) it finally calls
2582	* migrate_vma_finalize() to update the CPU page table to point to new pages
2583	* for successfully migrated pages or otherwise restore the CPU page table to
2584	* point to the original source pages.
2585	*/
2586	int migrate_vma_setup(struct migrate_vma *args)
2587	{
2588	long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
2589
2590	args->start &= PAGE_MASK;
2591	args->end &= PAGE_MASK;
2592	if (!args->vma \|\| is_vm_hugetlb_page(args->vma) \|\|
2593	(args->vma->vm_flags & VM_SPECIAL) \|\| vma_is_dax(args->vma))
2594	return -EINVAL;
2595	if (nr_pages <= 0)
2596	return -EINVAL;
2597	if (args->start < args->vma->vm_start \|\|
2598	args->start >= args->vma->vm_end)
2599	return -EINVAL;
2600	if (args->end <= args->vma->vm_start \|\| args->end > args->vma->vm_end)
2601	return -EINVAL;
2602	if (!args->src \|\| !args->dst)
2603	return -EINVAL;
2604
2605	memset(args->src, 0, sizeof(args->src) nr_pages);
2606	args->cpages = 0;
2607	args->npages = 0;
2608
2609	migrate_vma_collect(args);
2610
2611	if (args->cpages)
2612	migrate_vma_unmap(args);
2613
2614	/*
2615	* At this point pages are locked and unmapped, and thus they have
2616	* stable content and can safely be copied to destination memory that
2617	* is allocated by the drivers.
2618	*/
2619	return 0;
2620
2621	}
2622	EXPORT_SYMBOL(migrate_vma_setup);
2623
2624	/*
2625	* This code closely matches the code in:
2626	* __handle_mm_fault()
2627	* handle_pte_fault()
2628	* do_anonymous_page()
2629	* to map in an anonymous zero page but the struct page will be a ZONE_DEVICE
2630	* private page.
2631	*/
2632	static void migrate_vma_insert_page(struct migrate_vma *migrate,
2633	unsigned long addr,
2634	struct page *page,
2635	unsigned long *src)
2636	{
2637	struct vm_area_struct *vma = migrate->vma;
2638	struct mm_struct *mm = vma->vm_mm;
2639	bool flush = false;
2640	spinlock_t *ptl;
2641	pte_t entry;
2642	pgd_t *pgdp;
2643	p4d_t *p4dp;
2644	pud_t *pudp;
2645	pmd_t *pmdp;
2646	pte_t *ptep;
2647
2648	/* Only allow populating anonymous memory */
2649	if (!vma_is_anonymous(vma))
2650	goto abort;
2651
2652	pgdp = pgd_offset(mm, addr);
2653	p4dp = p4d_alloc(mm, pgdp, addr);
2654	if (!p4dp)
2655	goto abort;
2656	pudp = pud_alloc(mm, p4dp, addr);
2657	if (!pudp)
2658	goto abort;
2659	pmdp = pmd_alloc(mm, pudp, addr);
2660	if (!pmdp)
2661	goto abort;
2662
2663	if (pmd_trans_huge(pmdp) \|\| pmd_devmap(pmdp))
2664	goto abort;
2665
2666	/*
2667	* Use pte_alloc() instead of pte_alloc_map(). We can't run
2668	* pte_offset_map() on pmds where a huge pmd might be created
2669	* from a different thread.
2670	*
2671	* pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
2672	* parallel threads are excluded by other means.
2673	*
2674	* Here we only have mmap_read_lock(mm).
2675	*/
2676	if (pte_alloc(mm, pmdp))
2677	goto abort;
2678
2679	/* See the comment in pte_alloc_one_map() */
2680	if (unlikely(pmd_trans_unstable(pmdp)))
2681	goto abort;
2682
2683	if (unlikely(anon_vma_prepare(vma)))
2684	goto abort;
2685	if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))
2686	goto abort;
2687
2688	/*
2689	* The memory barrier inside __SetPageUptodate makes sure that
2690	* preceding stores to the page contents become visible before
2691	* the set_pte_at() write.
2692	*/
2693	__SetPageUptodate(page);
2694
2695	if (is_zone_device_page(page)) {
2696	if (is_device_private_page(page)) {
2697	swp_entry_t swp_entry;
2698
2699	if (vma->vm_flags & VM_WRITE)
2700	swp_entry = make_writable_device_private_entry(
2701	page_to_pfn(page));
2702	else
2703	swp_entry = make_readable_device_private_entry(
2704	page_to_pfn(page));
2705	entry = swp_entry_to_pte(swp_entry);
2706	} else {
2707	/*
2708	* For now we only support migrating to un-addressable
2709	* device memory.
2710	*/
2711	pr_warn_once("Unsupported ZONE_DEVICE page type.\n");
2712	goto abort;
2713	}
2714	} else {
2715	entry = mk_pte(page, vma->vm_page_prot);
2716	if (vma->vm_flags & VM_WRITE)
2717	entry = pte_mkwrite(pte_mkdirty(entry));
2718	}
2719
2720	ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2721
2722	if (check_stable_address_space(mm))
2723	goto unlock_abort;
2724
2725	if (pte_present(*ptep)) {
2726	unsigned long pfn = pte_pfn(*ptep);
2727
2728	if (!is_zero_pfn(pfn))
2729	goto unlock_abort;
2730	flush = true;
2731	} else if (!pte_none(*ptep))
2732	goto unlock_abort;
2733
2734	/*
2735	* Check for userfaultfd but do not deliver the fault. Instead,
2736	* just back off.
2737	*/
2738	if (userfaultfd_missing(vma))
2739	goto unlock_abort;
2740
2741	inc_mm_counter(mm, MM_ANONPAGES);
2742	page_add_new_anon_rmap(page, vma, addr, false);
2743	if (!is_zone_device_page(page))
2744	lru_cache_add_inactive_or_unevictable(page, vma);
2745	get_page(page);
2746
2747	if (flush) {
2748	flush_cache_page(vma, addr, pte_pfn(*ptep));
2749	ptep_clear_flush_notify(vma, addr, ptep);
2750	set_pte_at_notify(mm, addr, ptep, entry);
2751	update_mmu_cache(vma, addr, ptep);
2752	} else {
2753	/* No need to invalidate - it was non-present before */
2754	set_pte_at(mm, addr, ptep, entry);
2755	update_mmu_cache(vma, addr, ptep);
2756	}
2757
2758	pte_unmap_unlock(ptep, ptl);
2759	*src = MIGRATE_PFN_MIGRATE;
2760	return;
2761
2762	unlock_abort:
2763	pte_unmap_unlock(ptep, ptl);
2764	abort:
2765	*src &= ~MIGRATE_PFN_MIGRATE;
2766	}
2767
2768	/**
2769	* migrate_vma_pages() - migrate meta-data from src page to dst page
2770	* @migrate: migrate struct containing all migration information
2771	*
2772	* This migrates struct page meta-data from source struct page to destination
2773	* struct page. This effectively finishes the migration from source page to the
2774	* destination page.
2775	*/
2776	void migrate_vma_pages(struct migrate_vma *migrate)
2777	{
2778	const unsigned long npages = migrate->npages;
2779	const unsigned long start = migrate->start;
2780	struct mmu_notifier_range range;
2781	unsigned long addr, i;
2782	bool notified = false;
2783
2784	for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
2785	struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2786	struct page *page = migrate_pfn_to_page(migrate->src[i]);
2787	struct address_space *mapping;
2788	int r;
2789
2790	if (!newpage) {
2791	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2792	continue;
2793	}
2794
2795	if (!page) {
2796	if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2797	continue;
2798	if (!notified) {
2799	notified = true;
2800
2801	mmu_notifier_range_init_owner(&range,
2802	MMU_NOTIFY_MIGRATE, 0, migrate->vma,
2803	migrate->vma->vm_mm, addr, migrate->end,
2804	migrate->pgmap_owner);
2805	mmu_notifier_invalidate_range_start(&range);
2806	}
2807	migrate_vma_insert_page(migrate, addr, newpage,
2808	&migrate->src[i]);
2809	continue;
2810	}
2811
2812	mapping = page_mapping(page);
2813
2814	if (is_zone_device_page(newpage)) {
2815	if (is_device_private_page(newpage)) {
2816	/*
2817	* For now only support private anonymous when
2818	* migrating to un-addressable device memory.
2819	*/
2820	if (mapping) {
2821	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2822	continue;
2823	}
2824	} else {
2825	/*
2826	* Other types of ZONE_DEVICE page are not
2827	* supported.
2828	*/
2829	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2830	continue;
2831	}
2832	}
2833
2834	r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
2835	if (r != MIGRATEPAGE_SUCCESS)
2836	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2837	}
2838
2839	/*
2840	* No need to double call mmu_notifier->invalidate_range() callback as
2841	* the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
2842	* did already call it.
2843	*/
2844	if (notified)
2845	mmu_notifier_invalidate_range_only_end(&range);
2846	}
2847	EXPORT_SYMBOL(migrate_vma_pages);
2848
2849	/**
2850	* migrate_vma_finalize() - restore CPU page table entry
2851	* @migrate: migrate struct containing all migration information
2852	*
2853	* This replaces the special migration pte entry with either a mapping to the
2854	* new page if migration was successful for that page, or to the original page
2855	* otherwise.
2856	*
2857	* This also unlocks the pages and puts them back on the lru, or drops the extra
2858	* refcount, for device pages.
2859	*/
2860	void migrate_vma_finalize(struct migrate_vma *migrate)
2861	{
2862	const unsigned long npages = migrate->npages;
2863	unsigned long i;
2864
2865	for (i = 0; i < npages; i++) {
2866	struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2867	struct page *page = migrate_pfn_to_page(migrate->src[i]);
2868
2869	if (!page) {
2870	if (newpage) {
2871	unlock_page(newpage);
2872	put_page(newpage);
2873	}
2874	continue;
2875	}
2876
2877	if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) \|\| !newpage) {
2878	if (newpage) {
2879	unlock_page(newpage);
2880	put_page(newpage);
2881	}
2882	newpage = page;
2883	}
2884
2885	remove_migration_ptes(page, newpage, false);
2886	unlock_page(page);
2887
2888	if (is_zone_device_page(page))
2889	put_page(page);
2890	else
2891	putback_lru_page(page);
2892
2893	if (newpage != page) {
2894	unlock_page(newpage);
2895	if (is_zone_device_page(newpage))
2896	put_page(newpage);
2897	else
2898	putback_lru_page(newpage);
2899	}
2900	}
2901	}
2902	EXPORT_SYMBOL(migrate_vma_finalize);
2903	#endif /* CONFIG_DEVICE_PRIVATE */
2904
2905	/*	2147	/*
2906	* node_demotion[] example:	2148	* node_demotion[] example:
2907	*	2149	*

mm/migrate_device.c+773−0

@@ -0,0 +1,773 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Device Memory Migration functionality.
	4	*
	5	* Originally written by Jérôme Glisse.
	6	*/
	7	#include <linux/export.h>
	8	#include <linux/memremap.h>
	9	#include <linux/migrate.h>
	10	#include <linux/mm_inline.h>
	11	#include <linux/mmu_notifier.h>
	12	#include <linux/oom.h>
	13	#include <linux/pagewalk.h>
	14	#include <linux/rmap.h>
	15	#include <linux/swapops.h>
	16	#include <asm/tlbflush.h>
	17	#include "internal.h"
	18
	19	static int migrate_vma_collect_skip(unsigned long start,
	20	unsigned long end,
	21	struct mm_walk *walk)
	22	{
	23	struct migrate_vma *migrate = walk->private;
	24	unsigned long addr;
	25
	26	for (addr = start; addr < end; addr += PAGE_SIZE) {
	27	migrate->dst[migrate->npages] = 0;
	28	migrate->src[migrate->npages++] = 0;
	29	}
	30
	31	return 0;
	32	}
	33
	34	static int migrate_vma_collect_hole(unsigned long start,
	35	unsigned long end,
	36	__always_unused int depth,
	37	struct mm_walk *walk)
	38	{
	39	struct migrate_vma *migrate = walk->private;
	40	unsigned long addr;
	41
	42	/* Only allow populating anonymous memory. */
	43	if (!vma_is_anonymous(walk->vma))
	44	return migrate_vma_collect_skip(start, end, walk);
	45
	46	for (addr = start; addr < end; addr += PAGE_SIZE) {
	47	migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
	48	migrate->dst[migrate->npages] = 0;
	49	migrate->npages++;
	50	migrate->cpages++;
	51	}
	52
	53	return 0;
	54	}
	55
	56	static int migrate_vma_collect_pmd(pmd_t *pmdp,
	57	unsigned long start,
	58	unsigned long end,
	59	struct mm_walk *walk)
	60	{
	61	struct migrate_vma *migrate = walk->private;
	62	struct vm_area_struct *vma = walk->vma;
	63	struct mm_struct *mm = vma->vm_mm;
	64	unsigned long addr = start, unmapped = 0;
	65	spinlock_t *ptl;
	66	pte_t *ptep;
	67
	68	again:
	69	if (pmd_none(*pmdp))
	70	return migrate_vma_collect_hole(start, end, -1, walk);
	71
	72	if (pmd_trans_huge(*pmdp)) {
	73	struct page *page;
	74
	75	ptl = pmd_lock(mm, pmdp);
	76	if (unlikely(!pmd_trans_huge(*pmdp))) {
	77	spin_unlock(ptl);
	78	goto again;
	79	}
	80
	81	page = pmd_page(*pmdp);
	82	if (is_huge_zero_page(page)) {
	83	spin_unlock(ptl);
	84	split_huge_pmd(vma, pmdp, addr);
	85	if (pmd_trans_unstable(pmdp))
	86	return migrate_vma_collect_skip(start, end,
	87	walk);
	88	} else {
	89	int ret;
	90
	91	get_page(page);
	92	spin_unlock(ptl);
	93	if (unlikely(!trylock_page(page)))
	94	return migrate_vma_collect_skip(start, end,
	95	walk);
	96	ret = split_huge_page(page);
	97	unlock_page(page);
	98	put_page(page);
	99	if (ret)
	100	return migrate_vma_collect_skip(start, end,
	101	walk);
	102	if (pmd_none(*pmdp))
	103	return migrate_vma_collect_hole(start, end, -1,
	104	walk);
	105	}
	106	}
	107
	108	if (unlikely(pmd_bad(*pmdp)))
	109	return migrate_vma_collect_skip(start, end, walk);
	110
	111	ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
	112	arch_enter_lazy_mmu_mode();
	113
	114	for (; addr < end; addr += PAGE_SIZE, ptep++) {
	115	unsigned long mpfn = 0, pfn;
	116	struct page *page;
	117	swp_entry_t entry;
	118	pte_t pte;
	119
	120	pte = *ptep;
	121
	122	if (pte_none(pte)) {
	123	if (vma_is_anonymous(vma)) {
	124	mpfn = MIGRATE_PFN_MIGRATE;
	125	migrate->cpages++;
	126	}
	127	goto next;
	128	}
	129
	130	if (!pte_present(pte)) {
	131	/*
	132	* Only care about unaddressable device page special
	133	* page table entry. Other special swap entries are not
	134	* migratable, and we ignore regular swapped page.
	135	*/
	136	entry = pte_to_swp_entry(pte);
	137	if (!is_device_private_entry(entry))
	138	goto next;
	139
	140	page = pfn_swap_entry_to_page(entry);
	141	if (!(migrate->flags &
	142	MIGRATE_VMA_SELECT_DEVICE_PRIVATE) \|\|
	143	page->pgmap->owner != migrate->pgmap_owner)
	144	goto next;
	145
	146	mpfn = migrate_pfn(page_to_pfn(page)) \|
	147	MIGRATE_PFN_MIGRATE;
	148	if (is_writable_device_private_entry(entry))
	149	mpfn \|= MIGRATE_PFN_WRITE;
	150	} else {
	151	if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
	152	goto next;
	153	pfn = pte_pfn(pte);
	154	if (is_zero_pfn(pfn)) {
	155	mpfn = MIGRATE_PFN_MIGRATE;
	156	migrate->cpages++;
	157	goto next;
	158	}
	159	page = vm_normal_page(migrate->vma, addr, pte);
	160	mpfn = migrate_pfn(pfn) \| MIGRATE_PFN_MIGRATE;
	161	mpfn \|= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
	162	}
	163
	164	/* FIXME support THP */
	165	if (!page \|\| !page->mapping \|\| PageTransCompound(page)) {
	166	mpfn = 0;
	167	goto next;
	168	}
	169
	170	/*
	171	* By getting a reference on the page we pin it and that blocks
	172	* any kind of migration. Side effect is that it "freezes" the
	173	* pte.
	174	*
	175	* We drop this reference after isolating the page from the lru
	176	* for non device page (device page are not on the lru and thus
	177	* can't be dropped from it).
	178	*/
	179	get_page(page);
	180
	181	/*
	182	* Optimize for the common case where page is only mapped once
	183	* in one process. If we can lock the page, then we can safely
	184	* set up a special migration page table entry now.
	185	*/
	186	if (trylock_page(page)) {
	187	pte_t swp_pte;
	188
	189	migrate->cpages++;
	190	ptep_get_and_clear(mm, addr, ptep);
	191
	192	/* Setup special migration page table entry */
	193	if (mpfn & MIGRATE_PFN_WRITE)
	194	entry = make_writable_migration_entry(
	195	page_to_pfn(page));
	196	else
	197	entry = make_readable_migration_entry(
	198	page_to_pfn(page));
	199	swp_pte = swp_entry_to_pte(entry);
	200	if (pte_present(pte)) {
	201	if (pte_soft_dirty(pte))
	202	swp_pte = pte_swp_mksoft_dirty(swp_pte);
	203	if (pte_uffd_wp(pte))
	204	swp_pte = pte_swp_mkuffd_wp(swp_pte);
	205	} else {
	206	if (pte_swp_soft_dirty(pte))
	207	swp_pte = pte_swp_mksoft_dirty(swp_pte);
	208	if (pte_swp_uffd_wp(pte))
	209	swp_pte = pte_swp_mkuffd_wp(swp_pte);
	210	}
	211	set_pte_at(mm, addr, ptep, swp_pte);
	212
	213	/*
	214	* This is like regular unmap: we remove the rmap and
	215	* drop page refcount. Page won't be freed, as we took
	216	* a reference just above.
	217	*/
	218	page_remove_rmap(page, vma, false);
	219	put_page(page);
	220
	221	if (pte_present(pte))
	222	unmapped++;
	223	} else {
	224	put_page(page);
	225	mpfn = 0;
	226	}
	227
	228	next:
	229	migrate->dst[migrate->npages] = 0;
	230	migrate->src[migrate->npages++] = mpfn;
	231	}
	232	arch_leave_lazy_mmu_mode();
	233	pte_unmap_unlock(ptep - 1, ptl);
	234
	235	/* Only flush the TLB if we actually modified any entries */
	236	if (unmapped)
	237	flush_tlb_range(walk->vma, start, end);
	238
	239	return 0;
	240	}
	241
	242	static const struct mm_walk_ops migrate_vma_walk_ops = {
	243	.pmd_entry = migrate_vma_collect_pmd,
	244	.pte_hole = migrate_vma_collect_hole,
	245	};
	246
	247	/*
	248	* migrate_vma_collect() - collect pages over a range of virtual addresses
	249	* @migrate: migrate struct containing all migration information
	250	*
	251	* This will walk the CPU page table. For each virtual address backed by a
	252	* valid page, it updates the src array and takes a reference on the page, in
	253	* order to pin the page until we lock it and unmap it.
	254	*/
	255	static void migrate_vma_collect(struct migrate_vma *migrate)
	256	{
	257	struct mmu_notifier_range range;
	258
	259	/*
	260	* Note that the pgmap_owner is passed to the mmu notifier callback so
	261	* that the registered device driver can skip invalidating device
	262	* private page mappings that won't be migrated.
	263	*/
	264	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_MIGRATE, 0,
	265	migrate->vma, migrate->vma->vm_mm, migrate->start, migrate->end,
	266	migrate->pgmap_owner);
	267	mmu_notifier_invalidate_range_start(&range);
	268
	269	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
	270	&migrate_vma_walk_ops, migrate);
	271
	272	mmu_notifier_invalidate_range_end(&range);
	273	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
	274	}
	275
	276	/*
	277	* migrate_vma_check_page() - check if page is pinned or not
	278	* @page: struct page to check
	279	*
	280	* Pinned pages cannot be migrated. This is the same test as in
	281	* folio_migrate_mapping(), except that here we allow migration of a
	282	* ZONE_DEVICE page.
	283	*/
	284	static bool migrate_vma_check_page(struct page *page)
	285	{
	286	/*
	287	* One extra ref because caller holds an extra reference, either from
	288	* isolate_lru_page() for a regular page, or migrate_vma_collect() for
	289	* a device page.
	290	*/
	291	int extra = 1;
	292
	293	/*
	294	* FIXME support THP (transparent huge page), it is bit more complex to
	295	* check them than regular pages, because they can be mapped with a pmd
	296	* or with a pte (split pte mapping).
	297	*/
	298	if (PageCompound(page))
	299	return false;
	300
	301	/* Page from ZONE_DEVICE have one extra reference */
	302	if (is_zone_device_page(page))
	303	extra++;
	304
	305	/* For file back page */
	306	if (page_mapping(page))
	307	extra += 1 + page_has_private(page);
	308
	309	if ((page_count(page) - extra) > page_mapcount(page))
	310	return false;
	311
	312	return true;
	313	}
	314
	315	/*
	316	* migrate_vma_unmap() - replace page mapping with special migration pte entry
	317	* @migrate: migrate struct containing all migration information
	318	*
	319	* Isolate pages from the LRU and replace mappings (CPU page table pte) with a
	320	* special migration pte entry and check if it has been pinned. Pinned pages are
	321	* restored because we cannot migrate them.
	322	*
	323	* This is the last step before we call the device driver callback to allocate
	324	* destination memory and copy contents of original page over to new page.
	325	*/
	326	static void migrate_vma_unmap(struct migrate_vma *migrate)
	327	{
	328	const unsigned long npages = migrate->npages;
	329	unsigned long i, restore = 0;
	330	bool allow_drain = true;
	331
	332	lru_add_drain();
	333
	334	for (i = 0; i < npages; i++) {
	335	struct page *page = migrate_pfn_to_page(migrate->src[i]);
	336	struct folio *folio;
	337
	338	if (!page)
	339	continue;
	340
	341	/* ZONE_DEVICE pages are not on LRU */
	342	if (!is_zone_device_page(page)) {
	343	if (!PageLRU(page) && allow_drain) {
	344	/* Drain CPU's pagevec */
	345	lru_add_drain_all();
	346	allow_drain = false;
	347	}
	348
	349	if (isolate_lru_page(page)) {
	350	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
	351	migrate->cpages--;
	352	restore++;
	353	continue;
	354	}
	355
	356	/* Drop the reference we took in collect */
	357	put_page(page);
	358	}
	359
	360	folio = page_folio(page);
	361	if (folio_mapped(folio))
	362	try_to_migrate(folio, 0);
	363
	364	if (page_mapped(page) \|\| !migrate_vma_check_page(page)) {
	365	if (!is_zone_device_page(page)) {
	366	get_page(page);
	367	putback_lru_page(page);
	368	}
	369
	370	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
	371	migrate->cpages--;
	372	restore++;
	373	continue;
	374	}
	375	}
	376
	377	for (i = 0; i < npages && restore; i++) {
	378	struct page *page = migrate_pfn_to_page(migrate->src[i]);
	379	struct folio *folio;
	380
	381	if (!page \|\| (migrate->src[i] & MIGRATE_PFN_MIGRATE))
	382	continue;
	383
	384	folio = page_folio(page);
	385	remove_migration_ptes(folio, folio, false);
	386
	387	migrate->src[i] = 0;
	388	folio_unlock(folio);
	389	folio_put(folio);
	390	restore--;
	391	}
	392	}
	393
	394	/**
	395	* migrate_vma_setup() - prepare to migrate a range of memory
	396	* @args: contains the vma, start, and pfns arrays for the migration
	397	*
	398	* Returns: negative errno on failures, 0 when 0 or more pages were migrated
	399	* without an error.
	400	*
	401	* Prepare to migrate a range of memory virtual address range by collecting all
	402	* the pages backing each virtual address in the range, saving them inside the
	403	* src array. Then lock those pages and unmap them. Once the pages are locked
	404	* and unmapped, check whether each page is pinned or not. Pages that aren't
	405	* pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
	406	* corresponding src array entry. Then restores any pages that are pinned, by
	407	* remapping and unlocking those pages.
	408	*
	409	* The caller should then allocate destination memory and copy source memory to
	410	* it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
	411	* flag set). Once these are allocated and copied, the caller must update each
	412	* corresponding entry in the dst array with the pfn value of the destination
	413	* page and with MIGRATE_PFN_VALID. Destination pages must be locked via
	414	* lock_page().
	415	*
	416	* Note that the caller does not have to migrate all the pages that are marked
	417	* with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
	418	* device memory to system memory. If the caller cannot migrate a device page
	419	* back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
	420	* consequences for the userspace process, so it must be avoided if at all
	421	* possible.
	422	*
	423	* For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
	424	* do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
	425	* allowing the caller to allocate device memory for those unbacked virtual
	426	* addresses. For this the caller simply has to allocate device memory and
	427	* properly set the destination entry like for regular migration. Note that
	428	* this can still fail, and thus inside the device driver you must check if the
	429	* migration was successful for those entries after calling migrate_vma_pages(),
	430	* just like for regular migration.
	431	*
	432	* After that, the callers must call migrate_vma_pages() to go over each entry
	433	* in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
	434	* set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
	435	* then migrate_vma_pages() to migrate struct page information from the source
	436	* struct page to the destination struct page. If it fails to migrate the
	437	* struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
	438	* src array.
	439	*
	440	* At this point all successfully migrated pages have an entry in the src
	441	* array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
	442	* array entry with MIGRATE_PFN_VALID flag set.
	443	*
	444	* Once migrate_vma_pages() returns the caller may inspect which pages were
	445	* successfully migrated, and which were not. Successfully migrated pages will
	446	* have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
	447	*
	448	* It is safe to update device page table after migrate_vma_pages() because
	449	* both destination and source page are still locked, and the mmap_lock is held
	450	* in read mode (hence no one can unmap the range being migrated).
	451	*
	452	* Once the caller is done cleaning up things and updating its page table (if it
	453	* chose to do so, this is not an obligation) it finally calls
	454	* migrate_vma_finalize() to update the CPU page table to point to new pages
	455	* for successfully migrated pages or otherwise restore the CPU page table to
	456	* point to the original source pages.
	457	*/
	458	int migrate_vma_setup(struct migrate_vma *args)
	459	{
	460	long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
	461
	462	args->start &= PAGE_MASK;
	463	args->end &= PAGE_MASK;
	464	if (!args->vma \|\| is_vm_hugetlb_page(args->vma) \|\|
	465	(args->vma->vm_flags & VM_SPECIAL) \|\| vma_is_dax(args->vma))
	466	return -EINVAL;
	467	if (nr_pages <= 0)
	468	return -EINVAL;
	469	if (args->start < args->vma->vm_start \|\|
	470	args->start >= args->vma->vm_end)
	471	return -EINVAL;
	472	if (args->end <= args->vma->vm_start \|\| args->end > args->vma->vm_end)
	473	return -EINVAL;
	474	if (!args->src \|\| !args->dst)
	475	return -EINVAL;
	476
	477	memset(args->src, 0, sizeof(args->src) nr_pages);
	478	args->cpages = 0;
	479	args->npages = 0;
	480
	481	migrate_vma_collect(args);
	482
	483	if (args->cpages)
	484	migrate_vma_unmap(args);
	485
	486	/*
	487	* At this point pages are locked and unmapped, and thus they have
	488	* stable content and can safely be copied to destination memory that
	489	* is allocated by the drivers.
	490	*/
	491	return 0;
	492
	493	}
	494	EXPORT_SYMBOL(migrate_vma_setup);
	495
	496	/*
	497	* This code closely matches the code in:
	498	* __handle_mm_fault()
	499	* handle_pte_fault()
	500	* do_anonymous_page()
	501	* to map in an anonymous zero page but the struct page will be a ZONE_DEVICE
	502	* private page.
	503	*/
	504	static void migrate_vma_insert_page(struct migrate_vma *migrate,
	505	unsigned long addr,
	506	struct page *page,
	507	unsigned long *src)
	508	{
	509	struct vm_area_struct *vma = migrate->vma;
	510	struct mm_struct *mm = vma->vm_mm;
	511	bool flush = false;
	512	spinlock_t *ptl;
	513	pte_t entry;
	514	pgd_t *pgdp;
	515	p4d_t *p4dp;
	516	pud_t *pudp;
	517	pmd_t *pmdp;
	518	pte_t *ptep;
	519
	520	/* Only allow populating anonymous memory */
	521	if (!vma_is_anonymous(vma))
	522	goto abort;
	523
	524	pgdp = pgd_offset(mm, addr);
	525	p4dp = p4d_alloc(mm, pgdp, addr);
	526	if (!p4dp)
	527	goto abort;
	528	pudp = pud_alloc(mm, p4dp, addr);
	529	if (!pudp)
	530	goto abort;
	531	pmdp = pmd_alloc(mm, pudp, addr);
	532	if (!pmdp)
	533	goto abort;
	534
	535	if (pmd_trans_huge(pmdp) \|\| pmd_devmap(pmdp))
	536	goto abort;
	537
	538	/*
	539	* Use pte_alloc() instead of pte_alloc_map(). We can't run
	540	* pte_offset_map() on pmds where a huge pmd might be created
	541	* from a different thread.
	542	*
	543	* pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
	544	* parallel threads are excluded by other means.
	545	*
	546	* Here we only have mmap_read_lock(mm).
	547	*/
	548	if (pte_alloc(mm, pmdp))
	549	goto abort;
	550
	551	/* See the comment in pte_alloc_one_map() */
	552	if (unlikely(pmd_trans_unstable(pmdp)))
	553	goto abort;
	554
	555	if (unlikely(anon_vma_prepare(vma)))
	556	goto abort;
	557	if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))
	558	goto abort;
	559
	560	/*
	561	* The memory barrier inside __SetPageUptodate makes sure that
	562	* preceding stores to the page contents become visible before
	563	* the set_pte_at() write.
	564	*/
	565	__SetPageUptodate(page);
	566
	567	if (is_device_private_page(page)) {
	568	swp_entry_t swp_entry;
	569
	570	if (vma->vm_flags & VM_WRITE)
	571	swp_entry = make_writable_device_private_entry(
	572	page_to_pfn(page));
	573	else
	574	swp_entry = make_readable_device_private_entry(
	575	page_to_pfn(page));
	576	entry = swp_entry_to_pte(swp_entry);
	577	} else {
	578	/*
	579	* For now we only support migrating to un-addressable device
	580	* memory.
	581	*/
	582	if (is_zone_device_page(page)) {
	583	pr_warn_once("Unsupported ZONE_DEVICE page type.\n");
	584	goto abort;
	585	}
	586	entry = mk_pte(page, vma->vm_page_prot);
	587	if (vma->vm_flags & VM_WRITE)
	588	entry = pte_mkwrite(pte_mkdirty(entry));
	589	}
	590
	591	ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
	592
	593	if (check_stable_address_space(mm))
	594	goto unlock_abort;
	595
	596	if (pte_present(*ptep)) {
	597	unsigned long pfn = pte_pfn(*ptep);
	598
	599	if (!is_zero_pfn(pfn))
	600	goto unlock_abort;
	601	flush = true;
	602	} else if (!pte_none(*ptep))
	603	goto unlock_abort;
	604
	605	/*
	606	* Check for userfaultfd but do not deliver the fault. Instead,
	607	* just back off.
	608	*/
	609	if (userfaultfd_missing(vma))
	610	goto unlock_abort;
	611
	612	inc_mm_counter(mm, MM_ANONPAGES);
	613	page_add_new_anon_rmap(page, vma, addr, false);
	614	if (!is_zone_device_page(page))
	615	lru_cache_add_inactive_or_unevictable(page, vma);
	616	get_page(page);
	617
	618	if (flush) {
	619	flush_cache_page(vma, addr, pte_pfn(*ptep));
	620	ptep_clear_flush_notify(vma, addr, ptep);
	621	set_pte_at_notify(mm, addr, ptep, entry);
	622	update_mmu_cache(vma, addr, ptep);
	623	} else {
	624	/* No need to invalidate - it was non-present before */
	625	set_pte_at(mm, addr, ptep, entry);
	626	update_mmu_cache(vma, addr, ptep);
	627	}
	628
	629	pte_unmap_unlock(ptep, ptl);
	630	*src = MIGRATE_PFN_MIGRATE;
	631	return;
	632
	633	unlock_abort:
	634	pte_unmap_unlock(ptep, ptl);
	635	abort:
	636	*src &= ~MIGRATE_PFN_MIGRATE;
	637	}
	638
	639	/**
	640	* migrate_vma_pages() - migrate meta-data from src page to dst page
	641	* @migrate: migrate struct containing all migration information
	642	*
	643	* This migrates struct page meta-data from source struct page to destination
	644	* struct page. This effectively finishes the migration from source page to the
	645	* destination page.
	646	*/
	647	void migrate_vma_pages(struct migrate_vma *migrate)
	648	{
	649	const unsigned long npages = migrate->npages;
	650	const unsigned long start = migrate->start;
	651	struct mmu_notifier_range range;
	652	unsigned long addr, i;
	653	bool notified = false;
	654
	655	for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
	656	struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
	657	struct page *page = migrate_pfn_to_page(migrate->src[i]);
	658	struct address_space *mapping;
	659	int r;
	660
	661	if (!newpage) {
	662	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
	663	continue;
	664	}
	665
	666	if (!page) {
	667	if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))
	668	continue;
	669	if (!notified) {
	670	notified = true;
	671
	672	mmu_notifier_range_init_owner(&range,
	673	MMU_NOTIFY_MIGRATE, 0, migrate->vma,
	674	migrate->vma->vm_mm, addr, migrate->end,
	675	migrate->pgmap_owner);
	676	mmu_notifier_invalidate_range_start(&range);
	677	}
	678	migrate_vma_insert_page(migrate, addr, newpage,
	679	&migrate->src[i]);
	680	continue;
	681	}
	682
	683	mapping = page_mapping(page);
	684
	685	if (is_device_private_page(newpage)) {
	686	/*
	687	* For now only support private anonymous when migrating
	688	* to un-addressable device memory.
	689	*/
	690	if (mapping) {
	691	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
	692	continue;
	693	}
	694	} else if (is_zone_device_page(newpage)) {
	695	/*
	696	* Other types of ZONE_DEVICE page are not supported.
	697	*/
	698	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
	699	continue;
	700	}
	701
	702	r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
	703	if (r != MIGRATEPAGE_SUCCESS)
	704	migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
	705	}
	706
	707	/*
	708	* No need to double call mmu_notifier->invalidate_range() callback as
	709	* the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
	710	* did already call it.
	711	*/
	712	if (notified)
	713	mmu_notifier_invalidate_range_only_end(&range);
	714	}
	715	EXPORT_SYMBOL(migrate_vma_pages);
	716
	717	/**
	718	* migrate_vma_finalize() - restore CPU page table entry
	719	* @migrate: migrate struct containing all migration information
	720	*
	721	* This replaces the special migration pte entry with either a mapping to the
	722	* new page if migration was successful for that page, or to the original page
	723	* otherwise.
	724	*
	725	* This also unlocks the pages and puts them back on the lru, or drops the extra
	726	* refcount, for device pages.
	727	*/
	728	void migrate_vma_finalize(struct migrate_vma *migrate)
	729	{
	730	const unsigned long npages = migrate->npages;
	731	unsigned long i;
	732
	733	for (i = 0; i < npages; i++) {
	734	struct folio dst, src;
	735	struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
	736	struct page *page = migrate_pfn_to_page(migrate->src[i]);
	737
	738	if (!page) {
	739	if (newpage) {
	740	unlock_page(newpage);
	741	put_page(newpage);
	742	}
	743	continue;
	744	}
	745
	746	if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) \|\| !newpage) {
	747	if (newpage) {
	748	unlock_page(newpage);
	749	put_page(newpage);
	750	}
	751	newpage = page;
	752	}
	753
	754	src = page_folio(page);
	755	dst = page_folio(newpage);
	756	remove_migration_ptes(src, dst, false);
	757	folio_unlock(src);
	758
	759	if (is_zone_device_page(page))
	760	put_page(page);
	761	else
	762	putback_lru_page(page);
	763
	764	if (newpage != page) {
	765	unlock_page(newpage);
	766	if (is_zone_device_page(newpage))
	767	put_page(newpage);
	768	else
	769	putback_lru_page(newpage);
	770	}
	771	}
	772	}
	773	EXPORT_SYMBOL(migrate_vma_finalize);

mm/mlock.c+266−372

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	#include <linux/swapops.h>	14	#include <linux/swapops.h>
15	#include <linux/pagemap.h>	15	#include <linux/pagemap.h>
16	#include <linux/pagevec.h>	16	#include <linux/pagevec.h>
		17	#include <linux/pagewalk.h>
17	#include <linux/mempolicy.h>	18	#include <linux/mempolicy.h>
18	#include <linux/syscalls.h>	19	#include <linux/syscalls.h>
19	#include <linux/sched.h>	20	#include <linux/sched.h>
@@ -27,6 +28,8 @@ · 7 unmodified lines
27		28
28	#include "internal.h"	29	#include "internal.h"
29		30
		31	static DEFINE_PER_CPU(struct pagevec, mlock_pvec);
		32
30	bool can_do_mlock(void)	33	bool can_do_mlock(void)
31	{	34	{
32	if (rlimit(RLIMIT_MEMLOCK) != 0)	35	if (rlimit(RLIMIT_MEMLOCK) != 0)
@@ -46,441 +49,320 @@ EXPORT_SYMBOL(can_do_mlock); · 13 unmodified lines
46	* be placed on the LRU "unevictable" list, rather than the [in]active lists.	49	* be placed on the LRU "unevictable" list, rather than the [in]active lists.
47	* The unevictable list is an LRU sibling list to the [in]active lists.	50	* The unevictable list is an LRU sibling list to the [in]active lists.
48	* PageUnevictable is set to indicate the unevictable state.	51	* PageUnevictable is set to indicate the unevictable state.
49	*
50	* When lazy mlocking via vmscan, it is important to ensure that the
51	* vma's VM_LOCKED status is not concurrently being modified, otherwise we
52	* may have mlocked a page that is being munlocked. So lazy mlock must take
53	* the mmap_lock for read, and verify that the vma really is locked
54	* (see mm/rmap.c).
55	*/	52	*/
56		53
57	/*	54	static struct lruvec __mlock_page(struct page page, struct lruvec *lruvec)
58	* LRU accounting for clear_page_mlock()
59	*/
60	void clear_page_mlock(struct page *page)
61	{	55	{
62	int nr_pages;	56	/* There is nothing more we can do while it's off LRU */
		57	if (!TestClearPageLRU(page))
		58	return lruvec;
63		59
64	if (!TestClearPageMlocked(page))	60	lruvec = folio_lruvec_relock_irq(page_folio(page), lruvec);
65	return;
66		61
67	nr_pages = thp_nr_pages(page);	62	if (unlikely(page_evictable(page))) {
68	mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
69	count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages);
70	/*
71	* The previous TestClearPageMlocked() corresponds to the smp_mb()
72	* in __pagevec_lru_add_fn().
73	*
74	* See __pagevec_lru_add_fn for more explanation.
75	*/
76	if (!isolate_lru_page(page)) {
77	putback_lru_page(page);
78	} else {
79	/*	63	/*
80	* We lost the race. the page already moved to evictable list.	64	* This is a little surprising, but quite possible:
		65	* PageMlocked must have got cleared already by another CPU.
		66	* Could this page be on the Unevictable LRU? I'm not sure,
		67	* but move it now if so.
81	*/	68	*/
82	if (PageUnevictable(page))	69	if (PageUnevictable(page)) {
83	count_vm_events(UNEVICTABLE_PGSTRANDED, nr_pages);	70	del_page_from_lru_list(page, lruvec);
		71	ClearPageUnevictable(page);
		72	add_page_to_lru_list(page, lruvec);
		73	__count_vm_events(UNEVICTABLE_PGRESCUED,
		74	thp_nr_pages(page));
		75	}
		76	goto out;
84	}	77	}
		78
		79	if (PageUnevictable(page)) {
		80	if (PageMlocked(page))
		81	page->mlock_count++;
		82	goto out;
		83	}
		84
		85	del_page_from_lru_list(page, lruvec);
		86	ClearPageActive(page);
		87	SetPageUnevictable(page);
		88	page->mlock_count = !!PageMlocked(page);
		89	add_page_to_lru_list(page, lruvec);
		90	__count_vm_events(UNEVICTABLE_PGCULLED, thp_nr_pages(page));
		91	out:
		92	SetPageLRU(page);
		93	return lruvec;
85	}	94	}
86		95
87	/*	96	static struct lruvec __mlock_new_page(struct page page, struct lruvec *lruvec)
88	* Mark page as mlocked if not already.
89	* If page on LRU, isolate and putback to move to unevictable list.
90	*/
91	void mlock_vma_page(struct page *page)
92	{	97	{
93	/* Serialize with page migration */	98	VM_BUG_ON_PAGE(PageLRU(page), page);
94	BUG_ON(!PageLocked(page));
95		99
96	VM_BUG_ON_PAGE(PageTail(page), page);	100	lruvec = folio_lruvec_relock_irq(page_folio(page), lruvec);
97	VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page);
98		101
99	if (!TestSetPageMlocked(page)) {	102	/* As above, this is a little surprising, but possible */
100	int nr_pages = thp_nr_pages(page);	103	if (unlikely(page_evictable(page)))
		104	goto out;
101		105
102	mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages);	106	SetPageUnevictable(page);
103	count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);	107	page->mlock_count = !!PageMlocked(page);
104	if (!isolate_lru_page(page))	108	__count_vm_events(UNEVICTABLE_PGCULLED, thp_nr_pages(page));
105	putback_lru_page(page);	109	out:
106	}	110	add_page_to_lru_list(page, lruvec);
		111	SetPageLRU(page);
		112	return lruvec;
107	}	113	}
108		114
109	/*	115	static struct lruvec __munlock_page(struct page page, struct lruvec *lruvec)
110	* Finish munlock after successful page isolation
111	*
112	* Page must be locked. This is a wrapper for page_mlock()
113	* and putback_lru_page() with munlock accounting.
114	*/
115	static void __munlock_isolated_page(struct page *page)
116	{	116	{
117	/*	117	int nr_pages = thp_nr_pages(page);
118	* Optimization: if the page was mapped just once, that's our mapping	118	bool isolated = false;
119	* and we don't need to check all the other vmas.	119
120	*/	120	if (!TestClearPageLRU(page))
121	if (page_mapcount(page) > 1)	121	goto munlock;
122	page_mlock(page);
123		122
124	/* Did try_to_unlock() succeed or punt? */	123	isolated = true;
125	if (!PageMlocked(page))	124	lruvec = folio_lruvec_relock_irq(page_folio(page), lruvec);
126	count_vm_events(UNEVICTABLE_PGMUNLOCKED, thp_nr_pages(page));
127		125
128	putback_lru_page(page);	126	if (PageUnevictable(page)) {
		127	/* Then mlock_count is maintained, but might undercount */
		128	if (page->mlock_count)
		129	page->mlock_count--;
		130	if (page->mlock_count)
		131	goto out;
		132	}
		133	/* else assume that was the last mlock: reclaim will fix it if not */
		134
		135	munlock:
		136	if (TestClearPageMlocked(page)) {
		137	__mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
		138	if (isolated \|\| !PageUnevictable(page))
		139	__count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages);
		140	else
		141	__count_vm_events(UNEVICTABLE_PGSTRANDED, nr_pages);
		142	}
		143
		144	/* page_evictable() has to be checked after clearing Mlocked */
		145	if (isolated && PageUnevictable(page) && page_evictable(page)) {
		146	del_page_from_lru_list(page, lruvec);
		147	ClearPageUnevictable(page);
		148	add_page_to_lru_list(page, lruvec);
		149	__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
		150	}
		151	out:
		152	if (isolated)
		153	SetPageLRU(page);
		154	return lruvec;
129	}	155	}
130		156
131	/*	157	/*
132	* Accounting for page isolation fail during munlock	158	* Flags held in the low bits of a struct page pointer on the mlock_pvec.
133	*
134	* Performs accounting when page isolation fails in munlock. There is nothing
135	* else to do because it means some other task has already removed the page
136	* from the LRU. putback_lru_page() will take care of removing the page from
137	* the unevictable list, if necessary. vmscan [page_referenced()] will move
138	* the page back to the unevictable list if some other vma has it mlocked.
139	*/	159	*/
140	static void __munlock_isolation_failed(struct page *page)	160	#define LRU_PAGE 0x1
		161	#define NEW_PAGE 0x2
		162	static inline struct page mlock_lru(struct page page)
141	{	163	{
142	int nr_pages = thp_nr_pages(page);	164	return (struct page *)((unsigned long)page + LRU_PAGE);
		165	}
143		166
144	if (PageUnevictable(page))	167	static inline struct page mlock_new(struct page page)
145	__count_vm_events(UNEVICTABLE_PGSTRANDED, nr_pages);	168	{
146	else	169	return (struct page *)((unsigned long)page + NEW_PAGE);
147	__count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages);
148	}	170	}
149		171
150	/**	172	/*
151	* munlock_vma_page - munlock a vma page	173	* mlock_pagevec() is derived from pagevec_lru_move_fn():
152	* @page: page to be unlocked, either a normal page or THP page head	174	* perhaps that can make use of such page pointer flags in future,
153	*	175	* but for now just keep it for mlock. We could use three separate
154	* returns the size of the page as a page mask (0 for normal page,	176	* pagevecs instead, but one feels better (munlocking a full pagevec
155	* HPAGE_PMD_NR - 1 for THP head page)	177	* does not need to drain mlocking pagevecs first).
156	*
157	* called from munlock()/munmap() path with page supposedly on the LRU.
158	* When we munlock a page, because the vma where we found the page is being
159	* munlock()ed or munmap()ed, we want to check whether other vmas hold the
160	* page locked so that we can leave it on the unevictable lru list and not
161	* bother vmscan with it. However, to walk the page's rmap list in
162	* page_mlock() we must isolate the page from the LRU. If some other
163	* task has removed the page from the LRU, we won't be able to do that.
164	* So we clear the PageMlocked as we might not get another chance. If we
165	* can't isolate the page, we leave it for putback_lru_page() and vmscan
166	* [page_referenced()/try_to_unmap()] to deal with.
167	*/	178	*/
168	unsigned int munlock_vma_page(struct page *page)	179	static void mlock_pagevec(struct pagevec *pvec)
169	{	180	{
170	int nr_pages;	181	struct lruvec *lruvec = NULL;
171		182	unsigned long mlock;
172	/* For page_mlock() and to serialize with page migration */	183	struct page *page;
173	BUG_ON(!PageLocked(page));	184	int i;
174	VM_BUG_ON_PAGE(PageTail(page), page);
175		185
176	if (!TestClearPageMlocked(page)) {	186	for (i = 0; i < pagevec_count(pvec); i++) {
177	/* Potentially, PTE-mapped THP: do not skip the rest PTEs */	187	page = pvec->pages[i];
178	return 0;	188	mlock = (unsigned long)page & (LRU_PAGE \| NEW_PAGE);
		189	page = (struct page *)((unsigned long)page - mlock);
		190	pvec->pages[i] = page;
		191
		192	if (mlock & LRU_PAGE)
		193	lruvec = __mlock_page(page, lruvec);
		194	else if (mlock & NEW_PAGE)
		195	lruvec = __mlock_new_page(page, lruvec);
		196	else
		197	lruvec = __munlock_page(page, lruvec);
179	}	198	}
180		199
181	nr_pages = thp_nr_pages(page);	200	if (lruvec)
182	mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);	201	unlock_page_lruvec_irq(lruvec);
		202	release_pages(pvec->pages, pvec->nr);
		203	pagevec_reinit(pvec);
		204	}
183		205
184	if (!isolate_lru_page(page))	206	void mlock_page_drain(int cpu)
185	__munlock_isolated_page(page);	207	{
186	else	208	struct pagevec *pvec;
187	__munlock_isolation_failed(page);
188		209
189	return nr_pages - 1;	210	pvec = &per_cpu(mlock_pvec, cpu);
		211	if (pagevec_count(pvec))
		212	mlock_pagevec(pvec);
190	}	213	}
191		214
192	/*	215	bool need_mlock_page_drain(int cpu)
193	* convert get_user_pages() return value to posix mlock() error
194	*/
195	static int __mlock_posix_error_return(long retval)
196	{	216	{
197	if (retval == -EFAULT)	217	return pagevec_count(&per_cpu(mlock_pvec, cpu));
198	retval = -ENOMEM;
199	else if (retval == -ENOMEM)
200	retval = -EAGAIN;
201	return retval;
202	}	218	}
203		219
204	/*	220	/**
205	* Prepare page for fast batched LRU putback via putback_lru_evictable_pagevec()	221	* mlock_folio - mlock a folio already on (or temporarily off) LRU
206	*	222	* @folio: folio to be mlocked.
207	* The fast path is available only for evictable pages with single mapping.
208	* Then we can bypass the per-cpu pvec and get better performance.
209	* when mapcount > 1 we need page_mlock() which can fail.
210	* when !page_evictable(), we need the full redo logic of putback_lru_page to
211	* avoid leaving evictable page in unevictable list.
212	*
213	* In case of success, @page is added to @pvec and @pgrescued is incremented
214	* in case that the page was previously unevictable. @page is also unlocked.
215	*/	223	*/
216	static bool __putback_lru_fast_prepare(struct page page, struct pagevec pvec,	224	void mlock_folio(struct folio *folio)
217	int *pgrescued)
218	{	225	{
219	VM_BUG_ON_PAGE(PageLRU(page), page);	226	struct pagevec *pvec = &get_cpu_var(mlock_pvec);
220	VM_BUG_ON_PAGE(!PageLocked(page), page);
221		227
222	if (page_mapcount(page) <= 1 && page_evictable(page)) {	228	if (!folio_test_set_mlocked(folio)) {
223	pagevec_add(pvec, page);	229	int nr_pages = folio_nr_pages(folio);
224	if (TestClearPageUnevictable(page))	230
225	(*pgrescued)++;	231	zone_stat_mod_folio(folio, NR_MLOCK, nr_pages);
226	unlock_page(page);	232	__count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
227	return true;
228	}	233	}
229		234
230	return false;	235	folio_get(folio);
		236	if (!pagevec_add(pvec, mlock_lru(&folio->page)) \|\|
		237	folio_test_large(folio) \|\| lru_cache_disabled())
		238	mlock_pagevec(pvec);
		239	put_cpu_var(mlock_pvec);
231	}	240	}
232		241
233	/*	242	/**
234	* Putback multiple evictable pages to the LRU	243	* mlock_new_page - mlock a newly allocated page not yet on LRU
235	*	244	* @page: page to be mlocked, either a normal page or a THP head.
236	* Batched putback of evictable pages that bypasses the per-cpu pvec. Some of
237	* the pages might have meanwhile become unevictable but that is OK.
238	*/	245	*/
239	static void __putback_lru_fast(struct pagevec *pvec, int pgrescued)	246	void mlock_new_page(struct page *page)
240	{	247	{
241	count_vm_events(UNEVICTABLE_PGMUNLOCKED, pagevec_count(pvec));	248	struct pagevec *pvec = &get_cpu_var(mlock_pvec);
242	/*	249	int nr_pages = thp_nr_pages(page);
243	*__pagevec_lru_add() calls release_pages() so we don't call	250
244	* put_page() explicitly	251	SetPageMlocked(page);
245	*/	252	mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages);
246	__pagevec_lru_add(pvec);	253	__count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
247	count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);	254
		255	get_page(page);
		256	if (!pagevec_add(pvec, mlock_new(page)) \|\|
		257	PageHead(page) \|\| lru_cache_disabled())
		258	mlock_pagevec(pvec);
		259	put_cpu_var(mlock_pvec);
248	}	260	}
249		261
250	/*	262	/**
251	* Munlock a batch of pages from the same zone	263	* munlock_page - munlock a page
252	*	264	* @page: page to be munlocked, either a normal page or a THP head.
253	* The work is split to two main phases. First phase clears the Mlocked flag
254	* and attempts to isolate the pages, all under a single zone lru lock.
255	* The second phase finishes the munlock only for pages where isolation
256	* succeeded.
257	*
258	* Note that the pagevec may be modified during the process.
259	*/	265	*/
260	static void __munlock_pagevec(struct pagevec pvec, struct zone zone)	266	void munlock_page(struct page *page)
261	{	267	{
262	int i;	268	struct pagevec *pvec = &get_cpu_var(mlock_pvec);
263	int nr = pagevec_count(pvec);
264	int delta_munlocked = -nr;
265	struct pagevec pvec_putback;
266	struct lruvec *lruvec = NULL;
267	int pgrescued = 0;
268
269	pagevec_init(&pvec_putback);
270
271	/* Phase 1: page isolation */
272	for (i = 0; i < nr; i++) {
273	struct page *page = pvec->pages[i];
274	struct folio *folio = page_folio(page);
275
276	if (TestClearPageMlocked(page)) {
277	/*
278	* We already have pin from follow_page_mask()
279	* so we can spare the get_page() here.
280	*/
281	if (TestClearPageLRU(page)) {
282	lruvec = folio_lruvec_relock_irq(folio, lruvec);
283	del_page_from_lru_list(page, lruvec);
284	continue;
285	} else
286	__munlock_isolation_failed(page);
287	} else {
288	delta_munlocked++;
289	}
290
291	/*
292	* We won't be munlocking this page in the next phase
293	* but we still need to release the follow_page_mask()
294	* pin. We cannot do it under lru_lock however. If it's
295	* the last pin, __page_cache_release() would deadlock.
296	*/
297	pagevec_add(&pvec_putback, pvec->pages[i]);
298	pvec->pages[i] = NULL;
299	}
300	if (lruvec) {
301	__mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
302	unlock_page_lruvec_irq(lruvec);
303	} else if (delta_munlocked) {
304	mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
305	}
306
307	/* Now we can release pins of pages that we are not munlocking */
308	pagevec_release(&pvec_putback);
309
310	/* Phase 2: page munlock */
311	for (i = 0; i < nr; i++) {
312	struct page *page = pvec->pages[i];
313
314	if (page) {
315	lock_page(page);
316	if (!__putback_lru_fast_prepare(page, &pvec_putback,
317	&pgrescued)) {
318	/*
319	* Slow path. We don't want to lose the last
320	* pin before unlock_page()
321	*/
322	get_page(page); /* for putback_lru_page() */
323	__munlock_isolated_page(page);
324	unlock_page(page);
325	put_page(page); /* from follow_page_mask() */
326	}
327	}
328	}
329		269
330	/*	270	/*
331	* Phase 3: page putback for pages that qualified for the fast path	271	* TestClearPageMlocked(page) must be left to __munlock_page(),
332	* This will also call put_page() to return pin from follow_page_mask()	272	* which will check whether the page is multiply mlocked.
333	*/	273	*/
334	if (pagevec_count(&pvec_putback))	274
335	__putback_lru_fast(&pvec_putback, pgrescued);	275	get_page(page);
		276	if (!pagevec_add(pvec, page) \|\|
		277	PageHead(page) \|\| lru_cache_disabled())
		278	mlock_pagevec(pvec);
		279	put_cpu_var(mlock_pvec);
336	}	280	}
337		281
338	/*	282	static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
339	* Fill up pagevec for __munlock_pagevec using pte walk	283	unsigned long end, struct mm_walk *walk)
340	*	284
341	* The function expects that the struct page corresponding to @start address is
342	* a non-TPH page already pinned and in the @pvec, and that it belongs to @zone.
343	*
344	* The rest of @pvec is filled by subsequent pages within the same pmd and same
345	* zone, as long as the pte's are present and vm_normal_page() succeeds. These
346	* pages also get pinned.
347	*
348	* Returns the address of the next page that should be scanned. This equals
349	* @start + PAGE_SIZE when no page could be added by the pte walk.
350	*/
351	static unsigned long __munlock_pagevec_fill(struct pagevec *pvec,
352	struct vm_area_struct vma, struct zone zone,
353	unsigned long start, unsigned long end)
354	{	285	{
355	pte_t *pte;	286	struct vm_area_struct *vma = walk->vma;
356	spinlock_t *ptl;	287	spinlock_t *ptl;
		288	pte_t start_pte, pte;
		289	struct page *page;
357		290
358	/*	291	ptl = pmd_trans_huge_lock(pmd, vma);
359	* Initialize pte walk starting at the already pinned page where we	292	if (ptl) {
360	* are sure that there is a pte, as it was pinned under the same	293	if (!pmd_present(*pmd))
361	* mmap_lock write op.	294	goto out;
362	*/	295	if (is_huge_zero_pmd(*pmd))
363	pte = get_locked_pte(vma->vm_mm, start, &ptl);	296	goto out;
364	/* Make sure we do not cross the page table boundary */	297	page = pmd_page(*pmd);
365	end = pgd_addr_end(start, end);	298	if (vma->vm_flags & VM_LOCKED)
366	end = p4d_addr_end(start, end);	299	mlock_folio(page_folio(page));
367	end = pud_addr_end(start, end);	300	else
368	end = pmd_addr_end(start, end);	301	munlock_page(page);
369		302	goto out;
370	/* The page next to the pinned page is the first we will try to get */	303	}
371	start += PAGE_SIZE;
372	while (start < end) {
373	struct page *page = NULL;
374	pte++;
375	if (pte_present(*pte))
376	page = vm_normal_page(vma, start, *pte);
377	/*
378	* Break if page could not be obtained or the page's node+zone does not
379	* match
380	*/
381	if (!page \|\| page_zone(page) != zone)
382	break;
383		304
384	/*	305	start_pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
385	* Do not use pagevec for PTE-mapped THP,	306	for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) {
386	* munlock_vma_pages_range() will handle them.	307	if (!pte_present(*pte))
387	*/	308	continue;
		309	page = vm_normal_page(vma, addr, *pte);
		310	if (!page)
		311	continue;
388	if (PageTransCompound(page))	312	if (PageTransCompound(page))
389	break;	313	continue;
390		314	if (vma->vm_flags & VM_LOCKED)
391	get_page(page);	315	mlock_folio(page_folio(page));
392	/*	316	else
393	* Increase the address that will be returned before the	317	munlock_page(page);
394	* eventual break due to pvec becoming full by adding the page
395	*/
396	start += PAGE_SIZE;
397	if (pagevec_add(pvec, page) == 0)
398	break;
399	}	318	}
400	pte_unmap_unlock(pte, ptl);	319	pte_unmap(start_pte);
401	return start;	320	out:
		321	spin_unlock(ptl);
		322	cond_resched();
		323	return 0;
402	}	324	}
403		325
404	/*	326	/*
405	* munlock_vma_pages_range() - munlock all pages in the vma range.'	327	* mlock_vma_pages_range() - mlock any pages already in the range,
406	* @vma - vma containing range to be munlock()ed.	328	* or munlock all pages in the range.
		329	* @vma - vma containing range to be mlock()ed or munlock()ed
407	* @start - start address in @vma of the range	330	* @start - start address in @vma of the range
408	* @end - end of range in @vma.	331	* @end - end of range in @vma
		332	* @newflags - the new set of flags for @vma.
409	*	333	*
410	* For mremap(), munmap() and exit().	334	* Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED;
411	*	335	* called for munlock() and munlockall(), to clear VM_LOCKED from @vma.
412	* Called with @vma VM_LOCKED.
413	*
414	* Returns with VM_LOCKED cleared. Callers must be prepared to
415	* deal with this.
416	*
417	* We don't save and restore VM_LOCKED here because pages are
418	* still on lru. In unmap path, pages might be scanned by reclaim
419	* and re-mlocked by page_mlock/try_to_unmap before we unmap and
420	* free them. This will result in freeing mlocked pages.
421	*/	336	*/
422	void munlock_vma_pages_range(struct vm_area_struct *vma,	337	static void mlock_vma_pages_range(struct vm_area_struct *vma,
423	unsigned long start, unsigned long end)	338	unsigned long start, unsigned long end, vm_flags_t newflags)
424	{	339	{
425	vma->vm_flags &= VM_LOCKED_CLEAR_MASK;	340	static const struct mm_walk_ops mlock_walk_ops = {
		341	.pmd_entry = mlock_pte_range,
		342	};
426		343
427	while (start < end) {	344	/*
428	struct page *page;	345	* There is a slight chance that concurrent page migration,
429	unsigned int page_mask = 0;	346	* or page reclaim finding a page of this now-VM_LOCKED vma,
430	unsigned long page_increm;	347	* will call mlock_vma_page() and raise page's mlock_count:
431	struct pagevec pvec;	348	* double counting, leaving the page unevictable indefinitely.
432	struct zone *zone;	349	* Communicate this danger to mlock_vma_page() with VM_IO,
		350	* which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas.
		351	* mmap_lock is held in write mode here, so this weird
		352	* combination should not be visible to other mmap_lock users;
		353	* but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED.
		354	*/
		355	if (newflags & VM_LOCKED)
		356	newflags \|= VM_IO;
		357	WRITE_ONCE(vma->vm_flags, newflags);
433		358
434	pagevec_init(&pvec);	359	lru_add_drain();
435	/*	360	walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL);
436	* Although FOLL_DUMP is intended for get_dump_page(),	361	lru_add_drain();
437	* it just so happens that its special treatment of the	362
438	* ZERO_PAGE (returning an error instead of doing get_page)	363	if (newflags & VM_IO) {
439	* suits munlock very well (and if somehow an abnormal page	364	newflags &= ~VM_IO;
440	* has sneaked into the range, we won't oops here: great).	365	WRITE_ONCE(vma->vm_flags, newflags);
441	*/
442	page = follow_page(vma, start, FOLL_GET \| FOLL_DUMP);
443
444	if (page && !IS_ERR(page)) {
445	if (PageTransTail(page)) {
446	VM_BUG_ON_PAGE(PageMlocked(page), page);
447	put_page(page); /* follow_page_mask() */
448	} else if (PageTransHuge(page)) {
449	lock_page(page);
450	/*
451	* Any THP page found by follow_page_mask() may
452	* have gotten split before reaching
453	* munlock_vma_page(), so we need to compute
454	* the page_mask here instead.
455	*/
456	page_mask = munlock_vma_page(page);
457	unlock_page(page);
458	put_page(page); /* follow_page_mask() */
459	} else {
460	/*
461	* Non-huge pages are handled in batches via
462	* pagevec. The pin from follow_page_mask()
463	* prevents them from collapsing by THP.
464	*/
465	pagevec_add(&pvec, page);
466	zone = page_zone(page);
467
468	/*
469	* Try to fill the rest of pagevec using fast
470	* pte walk. This will also update start to
471	* the next page to process. Then munlock the
472	* pagevec.
473	*/
474	start = __munlock_pagevec_fill(&pvec, vma,
475	zone, start, end);
476	__munlock_pagevec(&pvec, zone);
477	goto next;
478	}
479	}
480	page_increm = 1 + page_mask;
481	start += page_increm * PAGE_SIZE;
482	next:
483	cond_resched();
484	}	366	}
485	}	367	}
486		368
@@ -500,10 +382,9 @@ static int mlock_fixup(struct vm_area_struct vma, struct vm_area_struct *prev, · 13 unmodified lines
500	pgoff_t pgoff;	382	pgoff_t pgoff;
501	int nr_pages;	383	int nr_pages;
502	int ret = 0;	384	int ret = 0;
503	int lock = !!(newflags & VM_LOCKED);	385	vm_flags_t oldflags = vma->vm_flags;
504	vm_flags_t old_flags = vma->vm_flags;
505		386
506	if (newflags == vma->vm_flags \|\| (vma->vm_flags & VM_SPECIAL) \|\|	387	if (newflags == oldflags \|\| (oldflags & VM_SPECIAL) \|\|
507	is_vm_hugetlb_page(vma) \|\| vma == get_gate_vma(current->mm) \|\|	388	is_vm_hugetlb_page(vma) \|\| vma == get_gate_vma(current->mm) \|\|
508	vma_is_dax(vma) \|\| vma_is_secretmem(vma))	389	vma_is_dax(vma) \|\| vma_is_secretmem(vma))
509	/* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */	390	/* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */
@@ -535,9 +416,9 @@ success: · 25 unmodified lines
535	* Keep track of amount of locked VM.	416	* Keep track of amount of locked VM.
536	*/	417	*/
537	nr_pages = (end - start) >> PAGE_SHIFT;	418	nr_pages = (end - start) >> PAGE_SHIFT;
538	if (!lock)	419	if (!(newflags & VM_LOCKED))
539	nr_pages = -nr_pages;	420	nr_pages = -nr_pages;
540	else if (old_flags & VM_LOCKED)	421	else if (oldflags & VM_LOCKED)
541	nr_pages = 0;	422	nr_pages = 0;
542	mm->locked_vm += nr_pages;	423	mm->locked_vm += nr_pages;
543		424
@@ -547,11 +428,12 @@ success: · 3 unmodified lines
547	* set VM_LOCKED, populate_vma_page_range will bring it back.	428	* set VM_LOCKED, populate_vma_page_range will bring it back.
548	*/	429	*/
549		430
550	if (lock)	431	if ((newflags & VM_LOCKED) && (oldflags & VM_LOCKED)) {
		432	/* No work to do, and mlocking twice would be wrong */
551	vma->vm_flags = newflags;	433	vma->vm_flags = newflags;
552	else	434	} else {
553	munlock_vma_pages_range(vma, start, end);	435	mlock_vma_pages_range(vma, start, end, newflags);
554		436	}
555	out:	437	out:
556	*prev = vma;	438	*prev = vma;
557	return ret;	439	return ret;
@@ -645,6 +527,18 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, · 87 unmodified lines
645	return count >> PAGE_SHIFT;	527	return count >> PAGE_SHIFT;
646	}	528	}
647		529
		530	/*
		531	* convert get_user_pages() return value to posix mlock() error
		532	*/
		533	static int __mlock_posix_error_return(long retval)
		534	{
		535	if (retval == -EFAULT)
		536	retval = -ENOMEM;
		537	else if (retval == -ENOMEM)
		538	retval = -EAGAIN;
		539	return retval;
		540	}
		541
648	static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)	542	static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
649	{	543	{
650	unsigned long locked;	544	unsigned long locked;

mm/mmap.c+2−30

 	vma->vm_prev = NULL;
 	do {
 		vma_rb_erase(vma, &mm->mm_rb);
+		if (vma->vm_flags & VM_LOCKED)
+			mm->locked_vm -= vma_pages(vma);
 		mm->map_count--;
 		tail_vma = vma;
 		vma = vma->vm_next;
 	return __split_vma(mm, vma, addr, new_below);
 }
-static inline void
-unlock_range(struct vm_area_struct *start, unsigned long limit)
-{
-	struct mm_struct *mm = start->vm_mm;
-	struct vm_area_struct *tmp = start;
-	while (tmp && tmp->vm_start < limit) {
-		if (tmp->vm_flags & VM_LOCKED) {
-			mm->locked_vm -= vma_pages(tmp);
-			munlock_vma_pages_all(tmp);
-		}
-		tmp = tmp->vm_next;
-	}
-}
 /* Munmap is split into 2 main parts -- this part which finds
  * what needs doing, and the areas themselves, which do the
  * work.  This now handles partial unmappings.
 			return error;
 	}
-	/*
-	 * unlock any mlock()ed ranges before detaching vmas
-	 */
-	if (mm->locked_vm)
-		unlock_range(vma, end);
 	/* Detach vmas from rbtree */
 	if (!detach_vmas_to_be_unmapped(mm, vma, prev, end))
 		downgrade = false;
 		 * Nothing can be holding mm->mmap_lock here and the above call
 		 * to mmu_notifier_release(mm) ensures mmu notifier callbacks in
 		 * __oom_reap_task_mm() will not block.
-		 *
-		 * This needs to be done before calling unlock_range(),
-		 * which clears VM_LOCKED, otherwise the oom reaper cannot
-		 * reliably test it.
 		 */
 		(void)__oom_reap_task_mm(mm);
 		set_bit(MMF_OOM_SKIP, &mm->flags);
 	}
 	mmap_write_lock(mm);
-	if (mm->locked_vm)
-		unlock_range(mm->mmap, ULONG_MAX);
 	arch_exit_mmap(mm);
 	vma = mm->mmap;

mm/mmzone.c+7−0

@@ -81,6 +81,13 @@ void lruvec_init(struct lruvec *lruvec) · 80 unmodified lines
81		81
82	for_each_lru(lru)	82	for_each_lru(lru)
83	INIT_LIST_HEAD(&lruvec->lists[lru]);	83	INIT_LIST_HEAD(&lruvec->lists[lru]);
		84	/*
		85	* The "Unevictable LRU" is imaginary: though its size is maintained,
		86	* it is never scanned, and unevictable pages are not threaded on it
		87	* (so that their lru fields can be reused to hold mlock_count).
		88	* Poison its list head, so that any operations on it would crash.
		89	*/
		90	list_del(&lruvec->lists[LRU_UNEVICTABLE]);
84	}	91	}
85		92
86	#if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS)	93	#if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS)

mm/oom_kill.c+1−1

@@ -523,7 +523,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) · 522 unmodified lines
523	set_bit(MMF_UNSTABLE, &mm->flags);	523	set_bit(MMF_UNSTABLE, &mm->flags);
524		524
525	for (vma = mm->mmap ; vma; vma = vma->vm_next) {	525	for (vma = mm->mmap ; vma; vma = vma->vm_next) {
526	if (!can_madv_lru_vma(vma))	526	if (vma->vm_flags & (VM_HUGETLB\|VM_PFNMAP))
527	continue;	527	continue;
528		528
529	/*	529	/*

mm/page_alloc.c+1−2

@@ -734,8 +734,7 @@ static void prep_compound_head(struct page *page, unsigned int order) · 733 unmodified lines
734	set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);	734	set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
735	set_compound_order(page, order);	735	set_compound_order(page, order);
736	atomic_set(compound_mapcount_ptr(page), -1);	736	atomic_set(compound_mapcount_ptr(page), -1);
737	if (hpage_pincount_available(page))	737	atomic_set(compound_pincount_ptr(page), 0);
738	atomic_set(compound_pincount_ptr(page), 0);
739	}	738	}
740		739
741	static void prep_compound_tail(struct page *head, int tail_idx)	740	static void prep_compound_tail(struct page *head, int tail_idx)

mm/page_idle.c+14−16

 #include <linux/page_ext.h>
 #include <linux/page_idle.h>
+#include "internal.h"
 #define BITMAP_CHUNK_SIZE	sizeof(u64)
 #define BITMAP_CHUNK_BITS	(BITMAP_CHUNK_SIZE * BITS_PER_BYTE)
 	return page;
 }
-static bool page_idle_clear_pte_refs_one(struct page *page,
+static bool page_idle_clear_pte_refs_one(struct folio *folio,
 					struct vm_area_struct *vma,
 					unsigned long addr, void *arg)
 {
-	struct page_vma_mapped_walk pvmw = {
+	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
-		.page = page,
-		.vma = vma,
-		.address = addr,
-	};
 	bool referenced = false;
 	while (page_vma_mapped_walk(&pvmw)) {
 	}
 	if (referenced) {
-		clear_page_idle(page);
+		folio_clear_idle(folio);
 		/*
 		 * We cleared the referenced bit in a mapping to this page. To
 		 * avoid interference with page reclaim, mark it young so that
-		 * page_referenced() will return > 0.
+		 * folio_referenced() will return > 0.
 		 */
-		set_page_young(page);
+		folio_set_young(folio);
 	}
 	return true;
 }
 static void page_idle_clear_pte_refs(struct page *page)
 {
+	struct folio *folio = page_folio(page);
 	/*
 	 * Since rwc.arg is unused, rwc is effectively immutable, so we
 	 * can make it static const to save some cycles and stack.
 	 */
 	static const struct rmap_walk_control rwc = {
 		.rmap_one = page_idle_clear_pte_refs_one,
-		.anon_lock = page_lock_anon_vma_read,
+		.anon_lock = folio_lock_anon_vma_read,
 	};
 	bool need_lock;
-	if (!page_mapped(page) ||
+	if (!folio_mapped(folio) || !folio_raw_mapping(folio))
-	    !page_rmapping(page))
 		return;
-	need_lock = !PageAnon(page) || PageKsm(page);
+	need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
-	if (need_lock && !trylock_page(page))
+	if (need_lock && !folio_trylock(folio))
 		return;
-	rmap_walk(page, (struct rmap_walk_control *)&rwc);
+	rmap_walk(folio, &rwc);
 	if (need_lock)
-		unlock_page(page);
+		folio_unlock(folio);
 }
 static ssize_t page_idle_bitmap_read(struct file *file, struct kobject *kobj,

mm/page_vma_mapped.c+26−32

 	return true;
 }
-static inline bool pfn_is_match(struct page *page, unsigned long pfn)
-{
-	unsigned long page_pfn = page_to_pfn(page);
-	/* normal page and hugetlbfs page */
-	if (!PageTransCompound(page) || PageHuge(page))
-		return page_pfn == pfn;
-	/* THP can be referenced by any subpage */
-	return pfn >= page_pfn && pfn - page_pfn < thp_nr_pages(page);
-}
 /**
  * check_pte - check if @pvmw->page is mapped at the @pvmw->pte
  * @pvmw: page_vma_mapped_walk struct, includes a pair pte and page for checking
 		pfn = pte_pfn(*pvmw->pte);
 	}
-	return pfn_is_match(pvmw->page, pfn);
+	return (pfn - pvmw->pfn) < pvmw->nr_pages;
+}
+/* Returns true if the two ranges overlap.  Careful to not overflow. */
+static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw)
+{
+	if ((pfn + HPAGE_PMD_NR - 1) < pvmw->pfn)
+		return false;
+	if (pfn > pvmw->pfn + pvmw->nr_pages - 1)
+		return false;
+	return true;
 }
 static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size)
 }
 /**
- * page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at
+ * page_vma_mapped_walk - check if @pvmw->pfn is mapped in @pvmw->vma at
  * @pvmw->address
  * @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
  * must be set. pmd, pte and ptl must be NULL.
  */
 bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
 {
-	struct mm_struct *mm = pvmw->vma->vm_mm;
+	struct vm_area_struct *vma = pvmw->vma;
-	struct page *page = pvmw->page;
+	struct mm_struct *mm = vma->vm_mm;
 	unsigned long end;
 	pgd_t *pgd;
 	p4d_t *p4d;
 	if (pvmw->pmd && !pvmw->pte)
 		return not_found(pvmw);
-	if (unlikely(PageHuge(page))) {
+	if (unlikely(is_vm_hugetlb_page(vma))) {
+		unsigned long size = pvmw->nr_pages * PAGE_SIZE;
 		/* The only possible mapping was handled on last iteration */
 		if (pvmw->pte)
 			return not_found(pvmw);
 		/* when pud is not present, pte will be NULL */
-		pvmw->pte = huge_pte_offset(mm, pvmw->address, page_size(page));
+		pvmw->pte = huge_pte_offset(mm, pvmw->address, size);
 		if (!pvmw->pte)
 			return false;
-		pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte);
+		pvmw->ptl = huge_pte_lockptr(size_to_hstate(size), mm,
+						pvmw->pte);
 		spin_lock(pvmw->ptl);
 		if (!check_pte(pvmw))
 			return not_found(pvmw);
 		return true;
 	}
-	/*
+	end = vma_address_end(pvmw);
-	 * Seek to next pte only makes sense for THP.
-	 * But more important than that optimization, is to filter out
-	 * any PageKsm page: whose page->index misleads vma_address()
-	 * and vma_address_end() to disaster.
-	 */
-	end = PageTransCompound(page) ?
-		vma_address_end(page, pvmw->vma) :
-		pvmw->address + PAGE_SIZE;
 	if (pvmw->pte)
 		goto next_pte;
 restart:
 			if (likely(pmd_trans_huge(pmde))) {
 				if (pvmw->flags & PVMW_MIGRATION)
 					return not_found(pvmw);
-				if (pmd_page(pmde) != page)
+				if (!check_pmd(pmd_pfn(pmde), pvmw))
 					return not_found(pvmw);
 				return true;
 			}
 					return not_found(pvmw);
 				entry = pmd_to_swp_entry(pmde);
 				if (!is_migration_entry(entry) ||
-				    pfn_swap_entry_to_page(entry) != page)
+				    !check_pmd(swp_offset(entry), pvmw))
 					return not_found(pvmw);
 				return true;
 			}
 			 * cleared *pmd but not decremented compound_mapcount().
 			 */
 			if ((pvmw->flags & PVMW_SYNC) &&
-			    PageTransCompound(page)) {
+			    transparent_hugepage_active(vma) &&
+			    (pvmw->nr_pages >= HPAGE_PMD_NR)) {
 				spinlock_t *ptl = pmd_lock(mm, pvmw->pmd);
 				spin_unlock(ptl);
 int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
 {
 	struct page_vma_mapped_walk pvmw = {
-		.page = page,
+		.pfn = page_to_pfn(page),
+		.nr_pages = 1,
 		.vma = vma,
 		.flags = PVMW_SYNC,
 	};

mm/readahead.c+100−8

@@ -262,7 +262,7 @@ static void read_pages(struct readahead_control rac, struct list_head pages, · 261 unmodified lines
262		262
263	blk_finish_plug(&plug);	263	blk_finish_plug(&plug);
264		264
265	BUG_ON(!list_empty(pages));	265	BUG_ON(pages && !list_empty(pages));
266	BUG_ON(readahead_count(rac));	266	BUG_ON(readahead_count(rac));
267		267
268	out:	268	out:
@@ -361,7 +361,7 @@ EXPORT_SYMBOL_GPL(page_cache_ra_unbounded); · 92 unmodified lines
361	* behaviour which would occur if page allocations are causing VM writeback.	361	* behaviour which would occur if page allocations are causing VM writeback.
362	* We really don't want to intermingle reads and writes like that.	362	* We really don't want to intermingle reads and writes like that.
363	*/	363	*/
364	void do_page_cache_ra(struct readahead_control *ractl,	364	static void do_page_cache_ra(struct readahead_control *ractl,
365	unsigned long nr_to_read, unsigned long lookahead_size)	365	unsigned long nr_to_read, unsigned long lookahead_size)
366	{	366	{
367	struct inode *inode = ractl->mapping->host;	367	struct inode *inode = ractl->mapping->host;
@@ -545,11 +545,103 @@ static int try_context_readahead(struct address_space *mapping, · 177 unmodified lines
545	return 1;	545	return 1;
546	}	546	}
547		547
		548	/*
		549	* There are some parts of the kernel which assume that PMD entries
		550	* are exactly HPAGE_PMD_ORDER. Those should be fixed, but until then,
		551	* limit the maximum allocation order to PMD size. I'm not aware of any
		552	* assumptions about maximum order if THP are disabled, but 8 seems like
		553	* a good order (that's 1MB if you're using 4kB pages)
		554	*/
		555	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		556	#define MAX_PAGECACHE_ORDER HPAGE_PMD_ORDER
		557	#else
		558	#define MAX_PAGECACHE_ORDER 8
		559	#endif
		560
		561	static inline int ra_alloc_folio(struct readahead_control *ractl, pgoff_t index,
		562	pgoff_t mark, unsigned int order, gfp_t gfp)
		563	{
		564	int err;
		565	struct folio *folio = filemap_alloc_folio(gfp, order);
		566
		567	if (!folio)
		568	return -ENOMEM;
		569	if (mark - index < (1UL << order))
		570	folio_set_readahead(folio);
		571	err = filemap_add_folio(ractl->mapping, folio, index, gfp);
		572	if (err)
		573	folio_put(folio);
		574	else
		575	ractl->_nr_pages += 1UL << order;
		576	return err;
		577	}
		578
		579	void page_cache_ra_order(struct readahead_control *ractl,
		580	struct file_ra_state *ra, unsigned int new_order)
		581	{
		582	struct address_space *mapping = ractl->mapping;
		583	pgoff_t index = readahead_index(ractl);
		584	pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;
		585	pgoff_t mark = index + ra->size - ra->async_size;
		586	int err = 0;
		587	gfp_t gfp = readahead_gfp_mask(mapping);
		588
		589	if (!mapping_large_folio_support(mapping) \|\| ra->size < 4)
		590	goto fallback;
		591
		592	limit = min(limit, index + ra->size - 1);
		593
		594	if (new_order < MAX_PAGECACHE_ORDER) {
		595	new_order += 2;
		596	if (new_order > MAX_PAGECACHE_ORDER)
		597	new_order = MAX_PAGECACHE_ORDER;
		598	while ((1 << new_order) > ra->size)
		599	new_order--;
		600	}
		601
		602	while (index <= limit) {
		603	unsigned int order = new_order;
		604
		605	/* Align with smaller pages if needed */
		606	if (index & ((1UL << order) - 1)) {
		607	order = __ffs(index);
		608	if (order == 1)
		609	order = 0;
		610	}
		611	/* Don't allocate pages past EOF */
		612	while (index + (1UL << order) - 1 > limit) {
		613	if (--order == 1)
		614	order = 0;
		615	}
		616	err = ra_alloc_folio(ractl, index, mark, order, gfp);
		617	if (err)
		618	break;
		619	index += 1UL << order;
		620	}
		621
		622	if (index > limit) {
		623	ra->size += index - limit - 1;
		624	ra->async_size += index - limit - 1;
		625	}
		626
		627	read_pages(ractl, NULL, false);
		628
		629	/*
		630	* If there were already pages in the page cache, then we may have
		631	* left some gaps. Let the regular readahead code take care of this
		632	* situation.
		633	*/
		634	if (!err)
		635	return;
		636	fallback:
		637	do_page_cache_ra(ractl, ra->size, ra->async_size);
		638	}
		639
548	/*	640	/*
549	* A minimal readahead algorithm for trivial sequential/random reads.	641	* A minimal readahead algorithm for trivial sequential/random reads.
550	*/	642	*/
551	static void ondemand_readahead(struct readahead_control *ractl,	643	static void ondemand_readahead(struct readahead_control *ractl,
552	bool hit_readahead_marker, unsigned long req_size)	644	struct folio *folio, unsigned long req_size)
553	{	645	{
554	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);	646	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);
555	struct file_ra_state *ra = ractl->ra;	647	struct file_ra_state *ra = ractl->ra;
@@ -584,12 +676,12 @@ static void ondemand_readahead(struct readahead_control *ractl, · 28 unmodified lines
584	}	676	}
585		677
586	/*	678	/*
587	* Hit a marked page without valid readahead state.	679	* Hit a marked folio without valid readahead state.
588	* E.g. interleaved reads.	680	* E.g. interleaved reads.
589	* Query the pagecache for async_size, which normally equals to	681	* Query the pagecache for async_size, which normally equals to
590	* readahead size. Ramp it up and use it as the new readahead size.	682	* readahead size. Ramp it up and use it as the new readahead size.
591	*/	683	*/
592	if (hit_readahead_marker) {	684	if (folio) {
593	pgoff_t start;	685	pgoff_t start;
594		686
595	rcu_read_lock();	687	rcu_read_lock();
@@ -662,7 +754,7 @@ readit: · 66 unmodified lines
662	}	754	}
663		755
664	ractl->_index = ra->start;	756	ractl->_index = ra->start;
665	do_page_cache_ra(ractl, ra->size, ra->async_size);	757	page_cache_ra_order(ractl, ra, folio ? folio_order(folio) : 0);
666	}	758	}
667		759
668	void page_cache_sync_ra(struct readahead_control *ractl,	760	void page_cache_sync_ra(struct readahead_control *ractl,
@@ -690,7 +782,7 @@ void page_cache_sync_ra(struct readahead_control *ractl, · 21 unmodified lines
690	}	782	}
691		783
692	/* do read-ahead */	784	/* do read-ahead */
693	ondemand_readahead(ractl, false, req_count);	785	ondemand_readahead(ractl, NULL, req_count);
694	}	786	}
695	EXPORT_SYMBOL_GPL(page_cache_sync_ra);	787	EXPORT_SYMBOL_GPL(page_cache_sync_ra);
696		788
@@ -713,7 +805,7 @@ void page_cache_async_ra(struct readahead_control *ractl, · 16 unmodified lines
713	return;	805	return;
714		806
715	/* do read-ahead */	807	/* do read-ahead */
716	ondemand_readahead(ractl, true, req_count);	808	ondemand_readahead(ractl, folio, req_count);
717	}	809	}
718	EXPORT_SYMBOL_GPL(page_cache_async_ra);	810	EXPORT_SYMBOL_GPL(page_cache_async_ra);
719		811

mm/rmap.c+239−339

@@ -107,15 +107,15 @@ static inline void anon_vma_free(struct anon_vma *anon_vma) · 106 unmodified lines
107	VM_BUG_ON(atomic_read(&anon_vma->refcount));	107	VM_BUG_ON(atomic_read(&anon_vma->refcount));
108		108
109	/*	109	/*
110	* Synchronize against page_lock_anon_vma_read() such that	110	* Synchronize against folio_lock_anon_vma_read() such that
111	* we can safely hold the lock without the anon_vma getting	111	* we can safely hold the lock without the anon_vma getting
112	* freed.	112	* freed.
113	*	113	*
114	* Relies on the full mb implied by the atomic_dec_and_test() from	114	* Relies on the full mb implied by the atomic_dec_and_test() from
115	* put_anon_vma() against the acquire barrier implied by	115	* put_anon_vma() against the acquire barrier implied by
116	* down_read_trylock() from page_lock_anon_vma_read(). This orders:	116	* down_read_trylock() from folio_lock_anon_vma_read(). This orders:
117	*	117	*
118	* page_lock_anon_vma_read() VS put_anon_vma()	118	* folio_lock_anon_vma_read() VS put_anon_vma()
119	* down_read_trylock() atomic_dec_and_test()	119	* down_read_trylock() atomic_dec_and_test()
120	* LOCK MB	120	* LOCK MB
121	* atomic_read() rwsem_is_locked()	121	* atomic_read() rwsem_is_locked()
@@ -168,7 +168,7 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, · 46 unmodified lines
168	* allocate a new one.	168	* allocate a new one.
169	*	169	*
170	* Anon-vma allocations are very subtle, because we may have	170	* Anon-vma allocations are very subtle, because we may have
171	* optimistically looked up an anon_vma in page_lock_anon_vma_read()	171	* optimistically looked up an anon_vma in folio_lock_anon_vma_read()
172	* and that may actually touch the rwsem even in the newly	172	* and that may actually touch the rwsem even in the newly
173	* allocated vma (it depends on RCU to make sure that the	173	* allocated vma (it depends on RCU to make sure that the
174	* anon_vma isn't actually destroyed).	174	* anon_vma isn't actually destroyed).
@@ -526,28 +526,28 @@ out: · 351 unmodified lines
526	* atomic op -- the trylock. If we fail the trylock, we fall back to getting a	526	* atomic op -- the trylock. If we fail the trylock, we fall back to getting a
527	* reference like with page_get_anon_vma() and then block on the mutex.	527	* reference like with page_get_anon_vma() and then block on the mutex.
528	*/	528	*/
529	struct anon_vma page_lock_anon_vma_read(struct page page)	529	struct anon_vma folio_lock_anon_vma_read(struct folio folio)
530	{	530	{
531	struct anon_vma *anon_vma = NULL;	531	struct anon_vma *anon_vma = NULL;
532	struct anon_vma *root_anon_vma;	532	struct anon_vma *root_anon_vma;
533	unsigned long anon_mapping;	533	unsigned long anon_mapping;
534		534
535	rcu_read_lock();	535	rcu_read_lock();
536	anon_mapping = (unsigned long)READ_ONCE(page->mapping);	536	anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
537	if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)	537	if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
538	goto out;	538	goto out;
539	if (!page_mapped(page))	539	if (!folio_mapped(folio))
540	goto out;	540	goto out;
541		541
542	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);	542	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
543	root_anon_vma = READ_ONCE(anon_vma->root);	543	root_anon_vma = READ_ONCE(anon_vma->root);
544	if (down_read_trylock(&root_anon_vma->rwsem)) {	544	if (down_read_trylock(&root_anon_vma->rwsem)) {
545	/*	545	/*
546	* If the page is still mapped, then this anon_vma is still	546	* If the folio is still mapped, then this anon_vma is still
547	* its anon_vma, and holding the mutex ensures that it will	547	* its anon_vma, and holding the mutex ensures that it will
548	* not go away, see anon_vma_free().	548	* not go away, see anon_vma_free().
549	*/	549	*/
550	if (!page_mapped(page)) {	550	if (!folio_mapped(folio)) {
551	up_read(&root_anon_vma->rwsem);	551	up_read(&root_anon_vma->rwsem);
552	anon_vma = NULL;	552	anon_vma = NULL;
553	}	553	}
@@ -560,7 +560,7 @@ struct anon_vma page_lock_anon_vma_read(struct page page) · 6 unmodified lines
560	goto out;	560	goto out;
561	}	561	}
562		562
563	if (!page_mapped(page)) {	563	if (!folio_mapped(folio)) {
564	rcu_read_unlock();	564	rcu_read_unlock();
565	put_anon_vma(anon_vma);	565	put_anon_vma(anon_vma);
566	return NULL;	566	return NULL;
@@ -737,8 +737,9 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) · 170 unmodified lines
737	*/	737	*/
738	unsigned long page_address_in_vma(struct page page, struct vm_area_struct vma)	738	unsigned long page_address_in_vma(struct page page, struct vm_area_struct vma)
739	{	739	{
740	if (PageAnon(page)) {	740	struct folio *folio = page_folio(page);
741	struct anon_vma *page__anon_vma = page_anon_vma(page);	741	if (folio_test_anon(folio)) {
		742	struct anon_vma *page__anon_vma = folio_anon_vma(folio);
742	/*	743	/*
743	* Note: swapoff's unuse_vma() is more efficient with this	744	* Note: swapoff's unuse_vma() is more efficient with this
744	* check, and needs it to match anon_vma when KSM is active.	745	* check, and needs it to match anon_vma when KSM is active.
@@ -748,7 +749,7 @@ unsigned long page_address_in_vma(struct page page, struct vm_area_struct vma) · 3 unmodified lines
748	return -EFAULT;	749	return -EFAULT;
749	} else if (!vma->vm_file) {	750	} else if (!vma->vm_file) {
750	return -EFAULT;	751	return -EFAULT;
751	} else if (vma->vm_file->f_mapping != compound_head(page)->mapping) {	752	} else if (vma->vm_file->f_mapping != folio->mapping) {
752	return -EFAULT;	753	return -EFAULT;
753	}	754	}
754		755
@@ -789,30 +790,29 @@ out: · 34 unmodified lines
789	return pmd;	790	return pmd;
790	}	791	}
791		792
792	struct page_referenced_arg {	793	struct folio_referenced_arg {
793	int mapcount;	794	int mapcount;
794	int referenced;	795	int referenced;
795	unsigned long vm_flags;	796	unsigned long vm_flags;
796	struct mem_cgroup *memcg;	797	struct mem_cgroup *memcg;
797	};	798	};
798	/*	799	/*
799	* arg: page_referenced_arg will be passed	800	* arg: folio_referenced_arg will be passed
800	*/	801	*/
801	static bool page_referenced_one(struct page page, struct vm_area_struct vma,	802	static bool folio_referenced_one(struct folio *folio,
802	unsigned long address, void *arg)	803	struct vm_area_struct vma, unsigned long address, void arg)
803	{	804	{
804	struct page_referenced_arg *pra = arg;	805	struct folio_referenced_arg *pra = arg;
805	struct page_vma_mapped_walk pvmw = {	806	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
806	.page = page,
807	.vma = vma,
808	.address = address,
809	};
810	int referenced = 0;	807	int referenced = 0;
811		808
812	while (page_vma_mapped_walk(&pvmw)) {	809	while (page_vma_mapped_walk(&pvmw)) {
813	address = pvmw.address;	810	address = pvmw.address;
814		811
815	if (vma->vm_flags & VM_LOCKED) {	812	if ((vma->vm_flags & VM_LOCKED) &&
		813	(!folio_test_large(folio) \|\| !pvmw.pte)) {
		814	/* Restore the mlock which got missed */
		815	mlock_vma_folio(folio, vma, !pvmw.pte);
816	page_vma_mapped_walk_done(&pvmw);	816	page_vma_mapped_walk_done(&pvmw);
817	pra->vm_flags \|= VM_LOCKED;	817	pra->vm_flags \|= VM_LOCKED;
818	return false; /* To break the loop */	818	return false; /* To break the loop */
@@ -824,10 +824,10 @@ static bool page_referenced_one(struct page page, struct vm_area_struct vma, · 5 unmodified lines
824	/*	824	/*
825	* Don't treat a reference through	825	* Don't treat a reference through
826	* a sequentially read mapping as such.	826	* a sequentially read mapping as such.
827	* If the page has been used in another mapping,	827	* If the folio has been used in another mapping,
828	* we will catch it; if this other mapping is	828	* we will catch it; if this other mapping is
829	* already gone, the unmap path will have set	829	* already gone, the unmap path will have set
830	* PG_referenced or activated the page.	830	* the referenced flag or activated the folio.
831	*/	831	*/
832	if (likely(!(vma->vm_flags & VM_SEQ_READ)))	832	if (likely(!(vma->vm_flags & VM_SEQ_READ)))
833	referenced++;	833	referenced++;
@@ -837,7 +837,7 @@ static bool page_referenced_one(struct page page, struct vm_area_struct vma, · 3 unmodified lines
837	pvmw.pmd))	837	pvmw.pmd))
838	referenced++;	838	referenced++;
839	} else {	839	} else {
840	/* unexpected pmd-mapped page? */	840	/* unexpected pmd-mapped folio? */
841	WARN_ON_ONCE(1);	841	WARN_ON_ONCE(1);
842	}	842	}
843		843
@@ -845,13 +845,13 @@ static bool page_referenced_one(struct page page, struct vm_area_struct vma, · 1 unmodified line
845	}	845	}
846		846
847	if (referenced)	847	if (referenced)
848	clear_page_idle(page);	848	folio_clear_idle(folio);
849	if (test_and_clear_page_young(page))	849	if (folio_test_clear_young(folio))
850	referenced++;	850	referenced++;
851		851
852	if (referenced) {	852	if (referenced) {
853	pra->referenced++;	853	pra->referenced++;
854	pra->vm_flags \|= vma->vm_flags;	854	pra->vm_flags \|= vma->vm_flags & ~VM_LOCKED;
855	}	855	}
856		856
857	if (!pra->mapcount)	857	if (!pra->mapcount)
@@ -860,9 +860,9 @@ static bool page_referenced_one(struct page page, struct vm_area_struct vma, · 2 unmodified lines
860	return true;	860	return true;
861	}	861	}
862		862
863	static bool invalid_page_referenced_vma(struct vm_area_struct vma, void arg)	863	static bool invalid_folio_referenced_vma(struct vm_area_struct vma, void arg)
864	{	864	{
865	struct page_referenced_arg *pra = arg;	865	struct folio_referenced_arg *pra = arg;
866	struct mem_cgroup *memcg = pra->memcg;	866	struct mem_cgroup *memcg = pra->memcg;
867		867
868	if (!mm_match_cgroup(vma->vm_mm, memcg))	868	if (!mm_match_cgroup(vma->vm_mm, memcg))
@@ -872,40 +872,39 @@ static bool invalid_page_referenced_vma(struct vm_area_struct vma, void arg) · 3 unmodified lines
872	}	872	}
873		873
874	/**	874	/**
875	* page_referenced - test if the page was referenced	875	* folio_referenced() - Test if the folio was referenced.
876	* @page: the page to test	876	* @folio: The folio to test.
877	* @is_locked: caller holds lock on the page	877	* @is_locked: Caller holds lock on the folio.
878	* @memcg: target memory cgroup	878	* @memcg: target memory cgroup
879	* @vm_flags: collect encountered vma->vm_flags who actually referenced the page	879	* @vm_flags: A combination of all the vma->vm_flags which referenced the folio.
880	*	880	*
881	* Quick test_and_clear_referenced for all mappings to a page,	881	* Quick test_and_clear_referenced for all mappings of a folio,
882	* returns the number of ptes which referenced the page.	882	*
		883	* Return: The number of mappings which referenced the folio.
883	*/	884	*/
884	int page_referenced(struct page *page,	885	int folio_referenced(struct folio *folio, int is_locked,
885	int is_locked,	886	struct mem_cgroup memcg, unsigned long vm_flags)
886	struct mem_cgroup *memcg,
887	unsigned long *vm_flags)
888	{	887	{
889	int we_locked = 0;	888	int we_locked = 0;
890	struct page_referenced_arg pra = {	889	struct folio_referenced_arg pra = {
891	.mapcount = total_mapcount(page),	890	.mapcount = folio_mapcount(folio),
892	.memcg = memcg,	891	.memcg = memcg,
893	};	892	};
894	struct rmap_walk_control rwc = {	893	struct rmap_walk_control rwc = {
895	.rmap_one = page_referenced_one,	894	.rmap_one = folio_referenced_one,
896	.arg = (void *)&pra,	895	.arg = (void *)&pra,
897	.anon_lock = page_lock_anon_vma_read,	896	.anon_lock = folio_lock_anon_vma_read,
898	};	897	};
899		898
900	*vm_flags = 0;	899	*vm_flags = 0;
901	if (!pra.mapcount)	900	if (!pra.mapcount)
902	return 0;	901	return 0;
903		902
904	if (!page_rmapping(page))	903	if (!folio_raw_mapping(folio))
905	return 0;	904	return 0;
906		905
907	if (!is_locked && (!PageAnon(page) \|\| PageKsm(page))) {	906	if (!is_locked && (!folio_test_anon(folio) \|\| folio_test_ksm(folio))) {
908	we_locked = trylock_page(page);	907	we_locked = folio_trylock(folio);
909	if (!we_locked)	908	if (!we_locked)
910	return 1;	909	return 1;
911	}	910	}
@@ -916,37 +915,32 @@ int page_referenced(struct page *page, · 4 unmodified lines
916	* cgroups	915	* cgroups
917	*/	916	*/
918	if (memcg) {	917	if (memcg) {
919	rwc.invalid_vma = invalid_page_referenced_vma;	918	rwc.invalid_vma = invalid_folio_referenced_vma;
920	}	919	}
921		920
922	rmap_walk(page, &rwc);	921	rmap_walk(folio, &rwc);
923	*vm_flags = pra.vm_flags;	922	*vm_flags = pra.vm_flags;
924		923
925	if (we_locked)	924	if (we_locked)
926	unlock_page(page);	925	folio_unlock(folio);
927		926
928	return pra.referenced;	927	return pra.referenced;
929	}	928	}
930		929
931	static bool page_mkclean_one(struct page page, struct vm_area_struct vma,	930	static bool page_mkclean_one(struct folio folio, struct vm_area_struct vma,
932	unsigned long address, void *arg)	931	unsigned long address, void *arg)
933	{	932	{
934	struct page_vma_mapped_walk pvmw = {	933	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
935	.page = page,
936	.vma = vma,
937	.address = address,
938	.flags = PVMW_SYNC,
939	};
940	struct mmu_notifier_range range;	934	struct mmu_notifier_range range;
941	int *cleaned = arg;	935	int *cleaned = arg;
942		936
943	/*	937	/*
944	* We have to assume the worse case ie pmd for invalidation. Note that	938	* We have to assume the worse case ie pmd for invalidation. Note that
945	* the page can not be free from this function.	939	* the folio can not be freed from this function.
946	*/	940	*/
947	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,	941	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
948	0, vma, vma->vm_mm, address,	942	0, vma, vma->vm_mm, address,
949	vma_address_end(page, vma));	943	vma_address_end(&pvmw));
950	mmu_notifier_invalidate_range_start(&range);	944	mmu_notifier_invalidate_range_start(&range);
951		945
952	while (page_vma_mapped_walk(&pvmw)) {	946	while (page_vma_mapped_walk(&pvmw)) {
@@ -974,14 +968,14 @@ static bool page_mkclean_one(struct page page, struct vm_area_struct vma, · 21 unmodified lines
974	if (!pmd_dirty(pmd) && !pmd_write(pmd))	968	if (!pmd_dirty(pmd) && !pmd_write(pmd))
975	continue;	969	continue;
976		970
977	flush_cache_page(vma, address, page_to_pfn(page));	971	flush_cache_page(vma, address, folio_pfn(folio));
978	entry = pmdp_invalidate(vma, address, pmd);	972	entry = pmdp_invalidate(vma, address, pmd);
979	entry = pmd_wrprotect(entry);	973	entry = pmd_wrprotect(entry);
980	entry = pmd_mkclean(entry);	974	entry = pmd_mkclean(entry);
981	set_pmd_at(vma->vm_mm, address, pmd, entry);	975	set_pmd_at(vma->vm_mm, address, pmd, entry);
982	ret = 1;	976	ret = 1;
983	#else	977	#else
984	/* unexpected pmd-mapped page? */	978	/* unexpected pmd-mapped folio? */
985	WARN_ON_ONCE(1);	979	WARN_ON_ONCE(1);
986	#endif	980	#endif
987	}	981	}
@@ -1029,7 +1023,7 @@ int folio_mkclean(struct folio *folio) · 41 unmodified lines
1029	if (!mapping)	1023	if (!mapping)
1030	return 0;	1024	return 0;
1031		1025
1032	rmap_walk(&folio->page, &rwc);	1026	rmap_walk(folio, &rwc);
1033		1027
1034	return cleaned;	1028	return cleaned;
1035	}	1029	}
@@ -1057,8 +1051,8 @@ void page_move_anon_rmap(struct page page, struct vm_area_struct vma) · 21 unmodified lines
1057	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;	1051	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1058	/*	1052	/*
1059	* Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written	1053	* Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
1060	* simultaneously, so a concurrent reader (eg page_referenced()'s	1054	* simultaneously, so a concurrent reader (eg folio_referenced()'s
1061	* PageAnon()) will not see one without the other.	1055	* folio_test_anon()) will not see one without the other.
1062	*/	1056	*/
1063	WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);	1057	WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);
1064	}	1058	}
@@ -1108,6 +1102,7 @@ static void __page_set_anon_rmap(struct page *page, · 43 unmodified lines
1108	static void __page_check_anon_rmap(struct page *page,	1102	static void __page_check_anon_rmap(struct page *page,
1109	struct vm_area_struct *vma, unsigned long address)	1103	struct vm_area_struct *vma, unsigned long address)
1110	{	1104	{
		1105	struct folio *folio = page_folio(page);
1111	/*	1106	/*
1112	* The page's anon-rmap details (mapping and index) are guaranteed to	1107	* The page's anon-rmap details (mapping and index) are guaranteed to
1113	* be set up correctly at this point.	1108	* be set up correctly at this point.
@@ -1119,7 +1114,8 @@ static void __page_check_anon_rmap(struct page *page, · 5 unmodified lines
1119	* are initially only visible via the pagetables, and the pte is locked	1114	* are initially only visible via the pagetables, and the pte is locked
1120	* over the call to page_add_new_anon_rmap.	1115	* over the call to page_add_new_anon_rmap.
1121	*/	1116	*/
1122	VM_BUG_ON_PAGE(page_anon_vma(page)->root != vma->anon_vma->root, page);	1117	VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
		1118	folio);
1123	VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),	1119	VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),
1124	page);	1120	page);
1125	}	1121	}
@@ -1181,17 +1177,17 @@ void do_page_add_anon_rmap(struct page *page, · 55 unmodified lines
1181	__mod_lruvec_page_state(page, NR_ANON_MAPPED, nr);	1177	__mod_lruvec_page_state(page, NR_ANON_MAPPED, nr);
1182	}	1178	}
1183		1179
1184	if (unlikely(PageKsm(page))) {	1180	if (unlikely(PageKsm(page)))
1185	unlock_page_memcg(page);	1181	unlock_page_memcg(page);
1186	return;
1187	}
1188		1182
1189	/* address might be in next vma when migration races vma_adjust */	1183	/* address might be in next vma when migration races vma_adjust */
1190	if (first)	1184	else if (first)
1191	__page_set_anon_rmap(page, vma, address,	1185	__page_set_anon_rmap(page, vma, address,
1192	flags & RMAP_EXCLUSIVE);	1186	flags & RMAP_EXCLUSIVE);
1193	else	1187	else
1194	__page_check_anon_rmap(page, vma, address);	1188	__page_check_anon_rmap(page, vma, address);
		1189
		1190	mlock_vma_page(page, vma, compound);
1195	}	1191	}
1196		1192
1197	/**	1193	/**
@@ -1216,8 +1212,7 @@ void page_add_new_anon_rmap(struct page *page, · 18 unmodified lines
1216	VM_BUG_ON_PAGE(!PageTransHuge(page), page);	1212	VM_BUG_ON_PAGE(!PageTransHuge(page), page);
1217	/* increment count (starts at -1) */	1213	/* increment count (starts at -1) */
1218	atomic_set(compound_mapcount_ptr(page), 0);	1214	atomic_set(compound_mapcount_ptr(page), 0);
1219	if (hpage_pincount_available(page))	1215	atomic_set(compound_pincount_ptr(page), 0);
1220	atomic_set(compound_pincount_ptr(page), 0);
1221		1216
1222	__mod_lruvec_page_state(page, NR_ANON_THPS, nr);	1217	__mod_lruvec_page_state(page, NR_ANON_THPS, nr);
1223	} else {	1218	} else {
@@ -1232,12 +1227,14 @@ void page_add_new_anon_rmap(struct page *page, · 8 unmodified lines
1232		1227
1233	/**	1228	/**
1234	* page_add_file_rmap - add pte mapping to a file page	1229	* page_add_file_rmap - add pte mapping to a file page
1235	* @page: the page to add the mapping to	1230	* @page: the page to add the mapping to
1236	* @compound: charge the page as compound or small page	1231	* @vma: the vm area in which the mapping is added
		1232	* @compound: charge the page as compound or small page
1237	*	1233	*
1238	* The caller needs to hold the pte lock.	1234	* The caller needs to hold the pte lock.
1239	*/	1235	*/
1240	void page_add_file_rmap(struct page *page, bool compound)	1236	void page_add_file_rmap(struct page *page,
		1237	struct vm_area_struct *vma, bool compound)
1241	{	1238	{
1242	int i, nr = 1;	1239	int i, nr = 1;
1243		1240
@@ -1271,13 +1268,8 @@ void page_add_file_rmap(struct page *page, bool compound) · 27 unmodified lines
1271	nr_pages);	1268	nr_pages);
1272	} else {	1269	} else {
1273	if (PageTransCompound(page) && page_mapping(page)) {	1270	if (PageTransCompound(page) && page_mapping(page)) {
1274	struct page *head = compound_head(page);
1275
1276	VM_WARN_ON_ONCE(!PageLocked(page));	1271	VM_WARN_ON_ONCE(!PageLocked(page));
1277		1272	SetPageDoubleMap(compound_head(page));
1278	SetPageDoubleMap(head);
1279	if (PageMlocked(page))
1280	clear_page_mlock(head);
1281	}	1273	}
1282	if (!atomic_inc_and_test(&page->_mapcount))	1274	if (!atomic_inc_and_test(&page->_mapcount))
1283	goto out;	1275	goto out;
@@ -1285,6 +1277,8 @@ void page_add_file_rmap(struct page *page, bool compound) · 1 unmodified line
1285	__mod_lruvec_page_state(page, NR_FILE_MAPPED, nr);	1277	__mod_lruvec_page_state(page, NR_FILE_MAPPED, nr);
1286	out:	1278	out:
1287	unlock_page_memcg(page);	1279	unlock_page_memcg(page);
		1280
		1281	mlock_vma_page(page, vma, compound);
1288	}	1282	}
1289		1283
1290	static void page_remove_file_rmap(struct page *page, bool compound)	1284	static void page_remove_file_rmap(struct page *page, bool compound)
@@ -1327,9 +1321,6 @@ static void page_remove_file_rmap(struct page *page, bool compound) · 36 unmodified lines
1327	* pte lock(a spinlock) is held, which implies preemption disabled.	1321	* pte lock(a spinlock) is held, which implies preemption disabled.
1328	*/	1322	*/
1329	__mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr);	1323	__mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr);
1330
1331	if (unlikely(PageMlocked(page)))
1332	clear_page_mlock(page);
1333	}	1324	}
1334		1325
1335	static void page_remove_anon_compound_rmap(struct page *page)	1326	static void page_remove_anon_compound_rmap(struct page *page)
@@ -1369,9 +1360,6 @@ static void page_remove_anon_compound_rmap(struct page *page) · 33 unmodified lines
1369	nr = thp_nr_pages(page);	1360	nr = thp_nr_pages(page);
1370	}	1361	}
1371		1362
1372	if (unlikely(PageMlocked(page)))
1373	clear_page_mlock(page);
1374
1375	if (nr)	1363	if (nr)
1376	__mod_lruvec_page_state(page, NR_ANON_MAPPED, -nr);	1364	__mod_lruvec_page_state(page, NR_ANON_MAPPED, -nr);
1377	}	1365	}
@@ -1379,11 +1367,13 @@ static void page_remove_anon_compound_rmap(struct page *page) · 1 unmodified line
1379	/**	1367	/**
1380	* page_remove_rmap - take down pte mapping from a page	1368	* page_remove_rmap - take down pte mapping from a page
1381	* @page: page to remove mapping from	1369	* @page: page to remove mapping from
		1370	* @vma: the vm area from which the mapping is removed
1382	* @compound: uncharge the page as compound or small page	1371	* @compound: uncharge the page as compound or small page
1383	*	1372	*
1384	* The caller needs to hold the pte lock.	1373	* The caller needs to hold the pte lock.
1385	*/	1374	*/
1386	void page_remove_rmap(struct page *page, bool compound)	1375	void page_remove_rmap(struct page *page,
		1376	struct vm_area_struct *vma, bool compound)
1387	{	1377	{
1388	lock_page_memcg(page);	1378	lock_page_memcg(page);
1389		1379
@@ -1408,9 +1398,6 @@ void page_remove_rmap(struct page *page, bool compound) · 18 unmodified lines
1408	*/	1398	*/
1409	__dec_lruvec_page_state(page, NR_ANON_MAPPED);	1399	__dec_lruvec_page_state(page, NR_ANON_MAPPED);
1410		1400
1411	if (unlikely(PageMlocked(page)))
1412	clear_page_mlock(page);
1413
1414	if (PageTransCompound(page))	1401	if (PageTransCompound(page))
1415	deferred_split_huge_page(compound_head(page));	1402	deferred_split_huge_page(compound_head(page));
1416		1403
@@ -1425,20 +1412,18 @@ void page_remove_rmap(struct page *page, bool compound) · 8 unmodified lines
1425	*/	1412	*/
1426	out:	1413	out:
1427	unlock_page_memcg(page);	1414	unlock_page_memcg(page);
		1415
		1416	munlock_vma_page(page, vma, compound);
1428	}	1417	}
1429		1418
1430	/*	1419	/*
1431	* @arg: enum ttu_flags will be passed to this argument	1420	* @arg: enum ttu_flags will be passed to this argument
1432	*/	1421	*/
1433	static bool try_to_unmap_one(struct page page, struct vm_area_struct vma,	1422	static bool try_to_unmap_one(struct folio folio, struct vm_area_struct vma,
1434	unsigned long address, void *arg)	1423	unsigned long address, void *arg)
1435	{	1424	{
1436	struct mm_struct *mm = vma->vm_mm;	1425	struct mm_struct *mm = vma->vm_mm;
1437	struct page_vma_mapped_walk pvmw = {	1426	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1438	.page = page,
1439	.vma = vma,
1440	.address = address,
1441	};
1442	pte_t pteval;	1427	pte_t pteval;
1443	struct page *subpage;	1428	struct page *subpage;
1444	bool ret = true;	1429	bool ret = true;
@@ -1455,21 +1440,20 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 10 unmodified lines
1455	pvmw.flags = PVMW_SYNC;	1440	pvmw.flags = PVMW_SYNC;
1456		1441
1457	if (flags & TTU_SPLIT_HUGE_PMD)	1442	if (flags & TTU_SPLIT_HUGE_PMD)
1458	split_huge_pmd_address(vma, address, false, page);	1443	split_huge_pmd_address(vma, address, false, folio);
1459		1444
1460	/*	1445	/*
1461	* For THP, we have to assume the worse case ie pmd for invalidation.	1446	* For THP, we have to assume the worse case ie pmd for invalidation.
1462	* For hugetlb, it could be much worse if we need to do pud	1447	* For hugetlb, it could be much worse if we need to do pud
1463	* invalidation in the case of pmd sharing.	1448	* invalidation in the case of pmd sharing.
1464	*	1449	*
1465	* Note that the page can not be free in this function as call of	1450	* Note that the folio can not be freed in this function as call of
1466	* try_to_unmap() must hold a reference on the page.	1451	* try_to_unmap() must hold a reference on the folio.
1467	*/	1452	*/
1468	range.end = PageKsm(page) ?	1453	range.end = vma_address_end(&pvmw);
1469	address + PAGE_SIZE : vma_address_end(page, vma);
1470	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,	1454	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
1471	address, range.end);	1455	address, range.end);
1472	if (PageHuge(page)) {	1456	if (folio_test_hugetlb(folio)) {
1473	/*	1457	/*
1474	* If sharing is possible, start and end will be adjusted	1458	* If sharing is possible, start and end will be adjusted
1475	* accordingly.	1459	* accordingly.
@@ -1480,32 +1464,26 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 4 unmodified lines
1480	mmu_notifier_invalidate_range_start(&range);	1464	mmu_notifier_invalidate_range_start(&range);
1481		1465
1482	while (page_vma_mapped_walk(&pvmw)) {	1466	while (page_vma_mapped_walk(&pvmw)) {
		1467	/* Unexpected PMD-mapped THP? */
		1468	VM_BUG_ON_FOLIO(!pvmw.pte, folio);
		1469
1483	/*	1470	/*
1484	* If the page is mlock()d, we cannot swap it out.	1471	* If the folio is in an mlock()d vma, we must not swap it out.
1485	*/	1472	*/
1486	if (!(flags & TTU_IGNORE_MLOCK) &&	1473	if (!(flags & TTU_IGNORE_MLOCK) &&
1487	(vma->vm_flags & VM_LOCKED)) {	1474	(vma->vm_flags & VM_LOCKED)) {
1488	/*	1475	/* Restore the mlock which got missed */
1489	* PTE-mapped THP are never marked as mlocked: so do	1476	mlock_vma_folio(folio, vma, false);
1490	* not set it on a DoubleMap THP, nor on an Anon THP
1491	* (which may still be PTE-mapped after DoubleMap was
1492	* cleared). But stop unmapping even in those cases.
1493	*/
1494	if (!PageTransCompound(page) \|\| (PageHead(page) &&
1495	!PageDoubleMap(page) && !PageAnon(page)))
1496	mlock_vma_page(page);
1497	page_vma_mapped_walk_done(&pvmw);	1477	page_vma_mapped_walk_done(&pvmw);
1498	ret = false;	1478	ret = false;
1499	break;	1479	break;
1500	}	1480	}
1501		1481
1502	/* Unexpected PMD-mapped THP? */	1482	subpage = folio_page(folio,
1503	VM_BUG_ON_PAGE(!pvmw.pte, page);	1483	pte_pfn(*pvmw.pte) - folio_pfn(folio));
1504
1505	subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
1506	address = pvmw.address;	1484	address = pvmw.address;
1507		1485
1508	if (PageHuge(page) && !PageAnon(page)) {	1486	if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) {
1509	/*	1487	/*
1510	* To call huge_pmd_unshare, i_mmap_rwsem must be	1488	* To call huge_pmd_unshare, i_mmap_rwsem must be
1511	* held in write mode. Caller needs to explicitly	1489	* held in write mode. Caller needs to explicitly
@@ -1544,7 +1522,7 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 32 unmodified lines
1544	if (should_defer_flush(mm, flags)) {	1522	if (should_defer_flush(mm, flags)) {
1545	/*	1523	/*
1546	* We clear the PTE but do not flush so potentially	1524	* We clear the PTE but do not flush so potentially
1547	* a remote CPU could still be writing to the page.	1525	* a remote CPU could still be writing to the folio.
1548	* If the entry was previously clean then the	1526	* If the entry was previously clean then the
1549	* architecture must guarantee that a clear->dirty	1527	* architecture must guarantee that a clear->dirty
1550	* transition on a cached TLB entry is written through	1528	* transition on a cached TLB entry is written through
@@ -1557,22 +1535,22 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 6 unmodified lines
1557	pteval = ptep_clear_flush(vma, address, pvmw.pte);	1535	pteval = ptep_clear_flush(vma, address, pvmw.pte);
1558	}	1536	}
1559		1537
1560	/* Move the dirty bit to the page. Now the pte is gone. */	1538	/* Set the dirty flag on the folio now the pte is gone. */
1561	if (pte_dirty(pteval))	1539	if (pte_dirty(pteval))
1562	set_page_dirty(page);	1540	folio_mark_dirty(folio);
1563		1541
1564	/* Update high watermark before we lower rss */	1542	/* Update high watermark before we lower rss */
1565	update_hiwater_rss(mm);	1543	update_hiwater_rss(mm);
1566		1544
1567	if (PageHWPoison(subpage) && !(flags & TTU_IGNORE_HWPOISON)) {	1545	if (PageHWPoison(subpage) && !(flags & TTU_IGNORE_HWPOISON)) {
1568	pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));	1546	pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
1569	if (PageHuge(page)) {	1547	if (folio_test_hugetlb(folio)) {
1570	hugetlb_count_sub(compound_nr(page), mm);	1548	hugetlb_count_sub(folio_nr_pages(folio), mm);
1571	set_huge_swap_pte_at(mm, address,	1549	set_huge_swap_pte_at(mm, address,
1572	pvmw.pte, pteval,	1550	pvmw.pte, pteval,
1573	vma_mmu_pagesize(vma));	1551	vma_mmu_pagesize(vma));
1574	} else {	1552	} else {
1575	dec_mm_counter(mm, mm_counter(page));	1553	dec_mm_counter(mm, mm_counter(&folio->page));
1576	set_pte_at(mm, address, pvmw.pte, pteval);	1554	set_pte_at(mm, address, pvmw.pte, pteval);
1577	}	1555	}
1578		1556
@@ -1587,18 +1565,19 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 8 unmodified lines
1587	* migration) will not expect userfaults on already	1565	* migration) will not expect userfaults on already
1588	* copied pages.	1566	* copied pages.
1589	*/	1567	*/
1590	dec_mm_counter(mm, mm_counter(page));	1568	dec_mm_counter(mm, mm_counter(&folio->page));
1591	/* We have to invalidate as we cleared the pte */	1569	/* We have to invalidate as we cleared the pte */
1592	mmu_notifier_invalidate_range(mm, address,	1570	mmu_notifier_invalidate_range(mm, address,
1593	address + PAGE_SIZE);	1571	address + PAGE_SIZE);
1594	} else if (PageAnon(page)) {	1572	} else if (folio_test_anon(folio)) {
1595	swp_entry_t entry = { .val = page_private(subpage) };	1573	swp_entry_t entry = { .val = page_private(subpage) };
1596	pte_t swp_pte;	1574	pte_t swp_pte;
1597	/*	1575	/*
1598	* Store the swap location in the pte.	1576	* Store the swap location in the pte.
1599	* See handle_pte_fault() ...	1577	* See handle_pte_fault() ...
1600	*/	1578	*/
1601	if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) {	1579	if (unlikely(folio_test_swapbacked(folio) !=
		1580	folio_test_swapcache(folio))) {
1602	WARN_ON_ONCE(1);	1581	WARN_ON_ONCE(1);
1603	ret = false;	1582	ret = false;
1604	/* We have to invalidate as we cleared the pte */	1583	/* We have to invalidate as we cleared the pte */
@@ -1609,8 +1588,8 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 4 unmodified lines
1609	}	1588	}
1610		1589
1611	/* MADV_FREE page check */	1590	/* MADV_FREE page check */
1612	if (!PageSwapBacked(page)) {	1591	if (!folio_test_swapbacked(folio)) {
1613	if (!PageDirty(page)) {	1592	if (!folio_test_dirty(folio)) {
1614	/* Invalidate as we cleared the pte */	1593	/* Invalidate as we cleared the pte */
1615	mmu_notifier_invalidate_range(mm,	1594	mmu_notifier_invalidate_range(mm,
1616	address, address + PAGE_SIZE);	1595	address, address + PAGE_SIZE);
@@ -1619,11 +1598,11 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 2 unmodified lines
1619	}	1598	}
1620		1599
1621	/*	1600	/*
1622	* If the page was redirtied, it cannot be	1601	* If the folio was redirtied, it cannot be
1623	* discarded. Remap the page to page table.	1602	* discarded. Remap the page to page table.
1624	*/	1603	*/
1625	set_pte_at(mm, address, pvmw.pte, pteval);	1604	set_pte_at(mm, address, pvmw.pte, pteval);
1626	SetPageSwapBacked(page);	1605	folio_set_swapbacked(folio);
1627	ret = false;	1606	ret = false;
1628	page_vma_mapped_walk_done(&pvmw);	1607	page_vma_mapped_walk_done(&pvmw);
1629	break;	1608	break;
@@ -1660,16 +1639,17 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 30 unmodified lines
1660	address + PAGE_SIZE);	1639	address + PAGE_SIZE);
1661	} else {	1640	} else {
1662	/*	1641	/*
1663	* This is a locked file-backed page, thus it cannot	1642	* This is a locked file-backed folio,
1664	* be removed from the page cache and replaced by a new	1643	* so it cannot be removed from the page
1665	* page before mmu_notifier_invalidate_range_end, so no	1644	* cache and replaced by a new folio before
1666	* concurrent thread might update its page table to	1645	* mmu_notifier_invalidate_range_end, so no
1667	* point at new page while a device still is using this	1646	* concurrent thread might update its page table
1668	* page.	1647	* to point at a new folio while a device is
		1648	* still using this folio.
1669	*	1649	*
1670	* See Documentation/vm/mmu_notifier.rst	1650	* See Documentation/vm/mmu_notifier.rst
1671	*/	1651	*/
1672	dec_mm_counter(mm, mm_counter_file(page));	1652	dec_mm_counter(mm, mm_counter_file(&folio->page));
1673	}	1653	}
1674	discard:	1654	discard:
1675	/*	1655	/*
@@ -1679,8 +1659,10 @@ discard: · 3 unmodified lines
1679	*	1659	*
1680	* See Documentation/vm/mmu_notifier.rst	1660	* See Documentation/vm/mmu_notifier.rst
1681	*/	1661	*/
1682	page_remove_rmap(subpage, PageHuge(page));	1662	page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
1683	put_page(page);	1663	if (vma->vm_flags & VM_LOCKED)
		1664	mlock_page_drain(smp_processor_id());
		1665	folio_put(folio);
1684	}	1666	}
1685		1667
1686	mmu_notifier_invalidate_range_end(&range);	1668	mmu_notifier_invalidate_range_end(&range);
@@ -1693,35 +1675,35 @@ static bool invalid_migration_vma(struct vm_area_struct vma, void arg) · 6 unmodified lines
1693	return vma_is_temporary_stack(vma);	1675	return vma_is_temporary_stack(vma);
1694	}	1676	}
1695		1677
1696	static int page_not_mapped(struct page *page)	1678	static int page_not_mapped(struct folio *folio)
1697	{	1679	{
1698	return !page_mapped(page);	1680	return !folio_mapped(folio);
1699	}	1681	}
1700		1682
1701	/**	1683	/**
1702	* try_to_unmap - try to remove all page table mappings to a page	1684	* try_to_unmap - Try to remove all page table mappings to a folio.
1703	* @page: the page to get unmapped	1685	* @folio: The folio to unmap.
1704	* @flags: action and flags	1686	* @flags: action and flags
1705	*	1687	*
1706	* Tries to remove all the page table entries which are mapping this	1688	* Tries to remove all the page table entries which are mapping this
1707	* page, used in the pageout path. Caller must hold the page lock.	1689	* folio. It is the caller's responsibility to check if the folio is
		1690	* still mapped if needed (use TTU_SYNC to prevent accounting races).
1708	*	1691	*
1709	* It is the caller's responsibility to check if the page is still	1692	* Context: Caller must hold the folio lock.
1710	* mapped when needed (use TTU_SYNC to prevent accounting races).
1711	*/	1693	*/
1712	void try_to_unmap(struct page *page, enum ttu_flags flags)	1694	void try_to_unmap(struct folio *folio, enum ttu_flags flags)
1713	{	1695	{
1714	struct rmap_walk_control rwc = {	1696	struct rmap_walk_control rwc = {
1715	.rmap_one = try_to_unmap_one,	1697	.rmap_one = try_to_unmap_one,
1716	.arg = (void *)flags,	1698	.arg = (void *)flags,
1717	.done = page_not_mapped,	1699	.done = page_not_mapped,
1718	.anon_lock = page_lock_anon_vma_read,	1700	.anon_lock = folio_lock_anon_vma_read,
1719	};	1701	};
1720		1702
1721	if (flags & TTU_RMAP_LOCKED)	1703	if (flags & TTU_RMAP_LOCKED)
1722	rmap_walk_locked(page, &rwc);	1704	rmap_walk_locked(folio, &rwc);
1723	else	1705	else
1724	rmap_walk(page, &rwc);	1706	rmap_walk(folio, &rwc);
1725	}	1707	}
1726		1708
1727	/*	1709	/*
@@ -1730,15 +1712,11 @@ void try_to_unmap(struct page *page, enum ttu_flags flags) · 2 unmodified lines
1730	* If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs	1712	* If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
1731	* containing migration entries.	1713	* containing migration entries.
1732	*/	1714	*/
1733	static bool try_to_migrate_one(struct page page, struct vm_area_struct vma,	1715	static bool try_to_migrate_one(struct folio folio, struct vm_area_struct vma,
1734	unsigned long address, void *arg)	1716	unsigned long address, void *arg)
1735	{	1717	{
1736	struct mm_struct *mm = vma->vm_mm;	1718	struct mm_struct *mm = vma->vm_mm;
1737	struct page_vma_mapped_walk pvmw = {	1719	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1738	.page = page,
1739	.vma = vma,
1740	.address = address,
1741	};
1742	pte_t pteval;	1720	pte_t pteval;
1743	struct page *subpage;	1721	struct page *subpage;
1744	bool ret = true;	1722	bool ret = true;
@@ -1759,7 +1737,7 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 14 unmodified lines
1759	* TTU_SPLIT_HUGE_PMD and it wants to freeze.	1737	* TTU_SPLIT_HUGE_PMD and it wants to freeze.
1760	*/	1738	*/
1761	if (flags & TTU_SPLIT_HUGE_PMD)	1739	if (flags & TTU_SPLIT_HUGE_PMD)
1762	split_huge_pmd_address(vma, address, true, page);	1740	split_huge_pmd_address(vma, address, true, folio);
1763		1741
1764	/*	1742	/*
1765	* For THP, we have to assume the worse case ie pmd for invalidation.	1743	* For THP, we have to assume the worse case ie pmd for invalidation.
@@ -1769,11 +1747,10 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 3 unmodified lines
1769	* Note that the page can not be free in this function as call of	1747	* Note that the page can not be free in this function as call of
1770	* try_to_unmap() must hold a reference on the page.	1748	* try_to_unmap() must hold a reference on the page.
1771	*/	1749	*/
1772	range.end = PageKsm(page) ?	1750	range.end = vma_address_end(&pvmw);
1773	address + PAGE_SIZE : vma_address_end(page, vma);
1774	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,	1751	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
1775	address, range.end);	1752	address, range.end);
1776	if (PageHuge(page)) {	1753	if (folio_test_hugetlb(folio)) {
1777	/*	1754	/*
1778	* If sharing is possible, start and end will be adjusted	1755	* If sharing is possible, start and end will be adjusted
1779	* accordingly.	1756	* accordingly.
@@ -1787,21 +1764,24 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 7 unmodified lines
1787	#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION	1764	#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1788	/* PMD-mapped THP migration entry */	1765	/* PMD-mapped THP migration entry */
1789	if (!pvmw.pte) {	1766	if (!pvmw.pte) {
1790	VM_BUG_ON_PAGE(PageHuge(page) \|\|	1767	subpage = folio_page(folio,
1791	!PageTransCompound(page), page);	1768	pmd_pfn(*pvmw.pmd) - folio_pfn(folio));
		1769	VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) \|\|
		1770	!folio_test_pmd_mappable(folio), folio);
1792		1771
1793	set_pmd_migration_entry(&pvmw, page);	1772	set_pmd_migration_entry(&pvmw, subpage);
1794	continue;	1773	continue;
1795	}	1774	}
1796	#endif	1775	#endif
1797		1776
1798	/* Unexpected PMD-mapped THP? */	1777	/* Unexpected PMD-mapped THP? */
1799	VM_BUG_ON_PAGE(!pvmw.pte, page);	1778	VM_BUG_ON_FOLIO(!pvmw.pte, folio);
1800		1779
1801	subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);	1780	subpage = folio_page(folio,
		1781	pte_pfn(*pvmw.pte) - folio_pfn(folio));
1802	address = pvmw.address;	1782	address = pvmw.address;
1803		1783
1804	if (PageHuge(page) && !PageAnon(page)) {	1784	if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) {
1805	/*	1785	/*
1806	* To call huge_pmd_unshare, i_mmap_rwsem must be	1786	* To call huge_pmd_unshare, i_mmap_rwsem must be
1807	* held in write mode. Caller needs to explicitly	1787	* held in write mode. Caller needs to explicitly
@@ -1839,15 +1819,15 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 31 unmodified lines
1839	flush_cache_page(vma, address, pte_pfn(*pvmw.pte));	1819	flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
1840	pteval = ptep_clear_flush(vma, address, pvmw.pte);	1820	pteval = ptep_clear_flush(vma, address, pvmw.pte);
1841		1821
1842	/* Move the dirty bit to the page. Now the pte is gone. */	1822	/* Set the dirty flag on the folio now the pte is gone. */
1843	if (pte_dirty(pteval))	1823	if (pte_dirty(pteval))
1844	set_page_dirty(page);	1824	folio_mark_dirty(folio);
1845		1825
1846	/* Update high watermark before we lower rss */	1826	/* Update high watermark before we lower rss */
1847	update_hiwater_rss(mm);	1827	update_hiwater_rss(mm);
1848		1828
1849	if (is_zone_device_page(page)) {	1829	if (folio_is_zone_device(folio)) {
1850	unsigned long pfn = page_to_pfn(page);	1830	unsigned long pfn = folio_pfn(folio);
1851	swp_entry_t entry;	1831	swp_entry_t entry;
1852	pte_t swp_pte;	1832	pte_t swp_pte;
1853		1833
@@ -1883,16 +1863,16 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 29 unmodified lines
1883	* changed when hugepage migrations to device private	1863	* changed when hugepage migrations to device private
1884	* memory are supported.	1864	* memory are supported.
1885	*/	1865	*/
1886	subpage = page;	1866	subpage = &folio->page;
1887	} else if (PageHWPoison(subpage)) {	1867	} else if (PageHWPoison(subpage)) {
1888	pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));	1868	pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
1889	if (PageHuge(page)) {	1869	if (folio_test_hugetlb(folio)) {
1890	hugetlb_count_sub(compound_nr(page), mm);	1870	hugetlb_count_sub(folio_nr_pages(folio), mm);
1891	set_huge_swap_pte_at(mm, address,	1871	set_huge_swap_pte_at(mm, address,
1892	pvmw.pte, pteval,	1872	pvmw.pte, pteval,
1893	vma_mmu_pagesize(vma));	1873	vma_mmu_pagesize(vma));
1894	} else {	1874	} else {
1895	dec_mm_counter(mm, mm_counter(page));	1875	dec_mm_counter(mm, mm_counter(&folio->page));
1896	set_pte_at(mm, address, pvmw.pte, pteval);	1876	set_pte_at(mm, address, pvmw.pte, pteval);
1897	}	1877	}
1898		1878
@@ -1907,7 +1887,7 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 8 unmodified lines
1907	* migration) will not expect userfaults on already	1887	* migration) will not expect userfaults on already
1908	* copied pages.	1888	* copied pages.
1909	*/	1889	*/
1910	dec_mm_counter(mm, mm_counter(page));	1890	dec_mm_counter(mm, mm_counter(&folio->page));
1911	/* We have to invalidate as we cleared the pte */	1891	/* We have to invalidate as we cleared the pte */
1912	mmu_notifier_invalidate_range(mm, address,	1892	mmu_notifier_invalidate_range(mm, address,
1913	address + PAGE_SIZE);	1893	address + PAGE_SIZE);
@@ -1953,8 +1933,10 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 39 unmodified lines
1953	*	1933	*
1954	* See Documentation/vm/mmu_notifier.rst	1934	* See Documentation/vm/mmu_notifier.rst
1955	*/	1935	*/
1956	page_remove_rmap(subpage, PageHuge(page));	1936	page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
1957	put_page(page);	1937	if (vma->vm_flags & VM_LOCKED)
		1938	mlock_page_drain(smp_processor_id());
		1939	folio_put(folio);
1958	}	1940	}
1959		1941
1960	mmu_notifier_invalidate_range_end(&range);	1942	mmu_notifier_invalidate_range_end(&range);
@@ -1964,19 +1946,19 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 3 unmodified lines
1964		1946
1965	/**	1947	/**
1966	* try_to_migrate - try to replace all page table mappings with swap entries	1948	* try_to_migrate - try to replace all page table mappings with swap entries
1967	* @page: the page to replace page table entries for	1949	* @folio: the folio to replace page table entries for
1968	* @flags: action and flags	1950	* @flags: action and flags
1969	*	1951	*
1970	* Tries to remove all the page table entries which are mapping this page and	1952	* Tries to remove all the page table entries which are mapping this folio and
1971	* replace them with special swap entries. Caller must hold the page lock.	1953	* replace them with special swap entries. Caller must hold the folio lock.
1972	*/	1954	*/
1973	void try_to_migrate(struct page *page, enum ttu_flags flags)	1955	void try_to_migrate(struct folio *folio, enum ttu_flags flags)
1974	{	1956	{
1975	struct rmap_walk_control rwc = {	1957	struct rmap_walk_control rwc = {
1976	.rmap_one = try_to_migrate_one,	1958	.rmap_one = try_to_migrate_one,
1977	.arg = (void *)flags,	1959	.arg = (void *)flags,
1978	.done = page_not_mapped,	1960	.done = page_not_mapped,
1979	.anon_lock = page_lock_anon_vma_read,	1961	.anon_lock = folio_lock_anon_vma_read,
1980	};	1962	};
1981		1963
1982	/*	1964	/*
@@ -1987,7 +1969,7 @@ void try_to_migrate(struct page *page, enum ttu_flags flags) · 4 unmodified lines
1987	TTU_SYNC)))	1969	TTU_SYNC)))
1988	return;	1970	return;
1989		1971
1990	if (is_zone_device_page(page) && !is_device_private_page(page))	1972	if (folio_is_zone_device(folio) && !folio_is_device_private(folio))
1991	return;	1973	return;
1992		1974
1993	/*	1975	/*
@@ -1998,83 +1980,13 @@ void try_to_migrate(struct page *page, enum ttu_flags flags) · 4 unmodified lines
1998	* locking requirements of exec(), migration skips	1980	* locking requirements of exec(), migration skips
1999	* temporary VMAs until after exec() completes.	1981	* temporary VMAs until after exec() completes.
2000	*/	1982	*/
2001	if (!PageKsm(page) && PageAnon(page))	1983	if (!folio_test_ksm(folio) && folio_test_anon(folio))
2002	rwc.invalid_vma = invalid_migration_vma;	1984	rwc.invalid_vma = invalid_migration_vma;
2003		1985
2004	if (flags & TTU_RMAP_LOCKED)	1986	if (flags & TTU_RMAP_LOCKED)
2005	rmap_walk_locked(page, &rwc);	1987	rmap_walk_locked(folio, &rwc);
2006	else	1988	else
2007	rmap_walk(page, &rwc);	1989	rmap_walk(folio, &rwc);
2008	}
2009
2010	/*
2011	* Walks the vma's mapping a page and mlocks the page if any locked vma's are
2012	* found. Once one is found the page is locked and the scan can be terminated.
2013	*/
2014	static bool page_mlock_one(struct page page, struct vm_area_struct vma,
2015	unsigned long address, void *unused)
2016	{
2017	struct page_vma_mapped_walk pvmw = {
2018	.page = page,
2019	.vma = vma,
2020	.address = address,
2021	};
2022
2023	/* An un-locked vma doesn't have any pages to lock, continue the scan */
2024	if (!(vma->vm_flags & VM_LOCKED))
2025	return true;
2026
2027	while (page_vma_mapped_walk(&pvmw)) {
2028	/*
2029	* Need to recheck under the ptl to serialise with
2030	* __munlock_pagevec_fill() after VM_LOCKED is cleared in
2031	* munlock_vma_pages_range().
2032	*/
2033	if (vma->vm_flags & VM_LOCKED) {
2034	/*
2035	* PTE-mapped THP are never marked as mlocked; but
2036	* this function is never called on a DoubleMap THP,
2037	* nor on an Anon THP (which may still be PTE-mapped
2038	* after DoubleMap was cleared).
2039	*/
2040	mlock_vma_page(page);
2041	/*
2042	* No need to scan further once the page is marked
2043	* as mlocked.
2044	*/
2045	page_vma_mapped_walk_done(&pvmw);
2046	return false;
2047	}
2048	}
2049
2050	return true;
2051	}
2052
2053	/**
2054	* page_mlock - try to mlock a page
2055	* @page: the page to be mlocked
2056	*
2057	* Called from munlock code. Checks all of the VMAs mapping the page and mlocks
2058	* the page if any are found. The page will be returned with PG_mlocked cleared
2059	* if it is not mapped by any locked vmas.
2060	*/
2061	void page_mlock(struct page *page)
2062	{
2063	struct rmap_walk_control rwc = {
2064	.rmap_one = page_mlock_one,
2065	.done = page_not_mapped,
2066	.anon_lock = page_lock_anon_vma_read,
2067
2068	};
2069
2070	VM_BUG_ON_PAGE(!PageLocked(page) \|\| PageLRU(page), page);
2071	VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page);
2072
2073	/* Anon THP are only marked as mlocked when singly mapped */
2074	if (PageTransCompound(page) && PageAnon(page))
2075	return;
2076
2077	rmap_walk(page, &rwc);
2078	}	1990	}
2079		1991
2080	#ifdef CONFIG_DEVICE_PRIVATE	1992	#ifdef CONFIG_DEVICE_PRIVATE
@@ -2085,15 +1997,11 @@ struct make_exclusive_args { · 4 unmodified lines
2085	bool valid;	1997	bool valid;
2086	};	1998	};
2087		1999
2088	static bool page_make_device_exclusive_one(struct page *page,	2000	static bool page_make_device_exclusive_one(struct folio *folio,
2089	struct vm_area_struct vma, unsigned long address, void priv)	2001	struct vm_area_struct vma, unsigned long address, void priv)
2090	{	2002	{
2091	struct mm_struct *mm = vma->vm_mm;	2003	struct mm_struct *mm = vma->vm_mm;
2092	struct page_vma_mapped_walk pvmw = {	2004	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
2093	.page = page,
2094	.vma = vma,
2095	.address = address,
2096	};
2097	struct make_exclusive_args *args = priv;	2005	struct make_exclusive_args *args = priv;
2098	pte_t pteval;	2006	pte_t pteval;
2099	struct page *subpage;	2007	struct page *subpage;
@@ -2104,12 +2012,13 @@ static bool page_make_device_exclusive_one(struct page *page, · 4 unmodified lines
2104		2012
2105	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma,	2013	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma,
2106	vma->vm_mm, address, min(vma->vm_end,	2014	vma->vm_mm, address, min(vma->vm_end,
2107	address + page_size(page)), args->owner);	2015	address + folio_size(folio)),
		2016	args->owner);
2108	mmu_notifier_invalidate_range_start(&range);	2017	mmu_notifier_invalidate_range_start(&range);
2109		2018
2110	while (page_vma_mapped_walk(&pvmw)) {	2019	while (page_vma_mapped_walk(&pvmw)) {
2111	/* Unexpected PMD-mapped THP? */	2020	/* Unexpected PMD-mapped THP? */
2112	VM_BUG_ON_PAGE(!pvmw.pte, page);	2021	VM_BUG_ON_FOLIO(!pvmw.pte, folio);
2113		2022
2114	if (!pte_present(*pvmw.pte)) {	2023	if (!pte_present(*pvmw.pte)) {
2115	ret = false;	2024	ret = false;
@@ -2117,16 +2026,17 @@ static bool page_make_device_exclusive_one(struct page *page, · 1 unmodified line
2117	break;	2026	break;
2118	}	2027	}
2119		2028
2120	subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);	2029	subpage = folio_page(folio,
		2030	pte_pfn(*pvmw.pte) - folio_pfn(folio));
2121	address = pvmw.address;	2031	address = pvmw.address;
2122		2032
2123	/* Nuke the page table entry. */	2033	/* Nuke the page table entry. */
2124	flush_cache_page(vma, address, pte_pfn(*pvmw.pte));	2034	flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
2125	pteval = ptep_clear_flush(vma, address, pvmw.pte);	2035	pteval = ptep_clear_flush(vma, address, pvmw.pte);
2126		2036
2127	/* Move the dirty bit to the page. Now the pte is gone. */	2037	/* Set the dirty flag on the folio now the pte is gone. */
2128	if (pte_dirty(pteval))	2038	if (pte_dirty(pteval))
2129	set_page_dirty(page);	2039	folio_mark_dirty(folio);
2130		2040
2131	/*	2041	/*
2132	* Check that our target page is still mapped at the expected	2042	* Check that our target page is still mapped at the expected
@@ -2159,7 +2069,7 @@ static bool page_make_device_exclusive_one(struct page *page, · 26 unmodified lines
2159	* There is a reference on the page for the swap entry which has	2069	* There is a reference on the page for the swap entry which has
2160	* been removed, so shouldn't take another.	2070	* been removed, so shouldn't take another.
2161	*/	2071	*/
2162	page_remove_rmap(subpage, false);	2072	page_remove_rmap(subpage, vma, false);
2163	}	2073	}
2164		2074
2165	mmu_notifier_invalidate_range_end(&range);	2075	mmu_notifier_invalidate_range_end(&range);
@@ -2168,21 +2078,22 @@ static bool page_make_device_exclusive_one(struct page *page, · 2 unmodified lines
2168	}	2078	}
2169		2079
2170	/**	2080	/**
2171	* page_make_device_exclusive - mark the page exclusively owned by a device	2081	* folio_make_device_exclusive - Mark the folio exclusively owned by a device.
2172	* @page: the page to replace page table entries for	2082	* @folio: The folio to replace page table entries for.
2173	* @mm: the mm_struct where the page is expected to be mapped	2083	* @mm: The mm_struct where the folio is expected to be mapped.
2174	* @address: address where the page is expected to be mapped	2084	* @address: Address where the folio is expected to be mapped.
2175	* @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks	2085	* @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks
2176	*	2086	*
2177	* Tries to remove all the page table entries which are mapping this page and	2087	* Tries to remove all the page table entries which are mapping this
2178	* replace them with special device exclusive swap entries to grant a device	2088	* folio and replace them with special device exclusive swap entries to
2179	* exclusive access to the page. Caller must hold the page lock.	2089	* grant a device exclusive access to the folio.
2180	*	2090	*
2181	* Returns false if the page is still mapped, or if it could not be unmapped	2091	* Context: Caller must hold the folio lock.
		2092	* Return: false if the page is still mapped, or if it could not be unmapped
2182	* from the expected address. Otherwise returns true (success).	2093	* from the expected address. Otherwise returns true (success).
2183	*/	2094	*/
2184	static bool page_make_device_exclusive(struct page page, struct mm_struct mm,	2095	static bool folio_make_device_exclusive(struct folio *folio,
2185	unsigned long address, void *owner)	2096	struct mm_struct mm, unsigned long address, void owner)
2186	{	2097	{
2187	struct make_exclusive_args args = {	2098	struct make_exclusive_args args = {
2188	.mm = mm,	2099	.mm = mm,
@@ -2193,21 +2104,20 @@ static bool page_make_device_exclusive(struct page page, struct mm_struct mm, · 4 unmodified lines
2193	struct rmap_walk_control rwc = {	2104	struct rmap_walk_control rwc = {
2194	.rmap_one = page_make_device_exclusive_one,	2105	.rmap_one = page_make_device_exclusive_one,
2195	.done = page_not_mapped,	2106	.done = page_not_mapped,
2196	.anon_lock = page_lock_anon_vma_read,	2107	.anon_lock = folio_lock_anon_vma_read,
2197	.arg = &args,	2108	.arg = &args,
2198	};	2109	};
2199		2110
2200	/*	2111	/*
2201	* Restrict to anonymous pages for now to avoid potential writeback	2112	* Restrict to anonymous folios for now to avoid potential writeback
2202	* issues. Also tail pages shouldn't be passed to rmap_walk so skip	2113	* issues.
2203	* those.
2204	*/	2114	*/
2205	if (!PageAnon(page) \|\| PageTail(page))	2115	if (!folio_test_anon(folio))
2206	return false;	2116	return false;
2207		2117
2208	rmap_walk(page, &rwc);	2118	rmap_walk(folio, &rwc);
2209		2119
2210	return args.valid && !page_mapcount(page);	2120	return args.valid && !folio_mapcount(folio);
2211	}	2121	}
2212		2122
2213	/**	2123	/**
@@ -2245,15 +2155,16 @@ int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, · 31 unmodified lines
2245	return npages;	2155	return npages;
2246		2156
2247	for (i = 0; i < npages; i++, start += PAGE_SIZE) {	2157	for (i = 0; i < npages; i++, start += PAGE_SIZE) {
2248	if (!trylock_page(pages[i])) {	2158	struct folio *folio = page_folio(pages[i]);
2249	put_page(pages[i]);	2159	if (PageTail(pages[i]) \|\| !folio_trylock(folio)) {
		2160	folio_put(folio);
2250	pages[i] = NULL;	2161	pages[i] = NULL;
2251	continue;	2162	continue;
2252	}	2163	}
2253		2164
2254	if (!page_make_device_exclusive(pages[i], mm, start, owner)) {	2165	if (!folio_make_device_exclusive(folio, mm, start, owner)) {
2255	unlock_page(pages[i]);	2166	folio_unlock(folio);
2256	put_page(pages[i]);	2167	folio_put(folio);
2257	pages[i] = NULL;	2168	pages[i] = NULL;
2258	}	2169	}
2259	}	2170	}
@@ -2272,21 +2183,21 @@ void __put_anon_vma(struct anon_vma *anon_vma) · 12 unmodified lines
2272	anon_vma_free(root);	2183	anon_vma_free(root);
2273	}	2184	}
2274		2185
2275	static struct anon_vma rmap_walk_anon_lock(struct page page,	2186	static struct anon_vma rmap_walk_anon_lock(struct folio folio,
2276	struct rmap_walk_control *rwc)	2187	const struct rmap_walk_control *rwc)
2277	{	2188	{
2278	struct anon_vma *anon_vma;	2189	struct anon_vma *anon_vma;
2279		2190
2280	if (rwc->anon_lock)	2191	if (rwc->anon_lock)
2281	return rwc->anon_lock(page);	2192	return rwc->anon_lock(folio);
2282		2193
2283	/*	2194	/*
2284	* Note: remove_migration_ptes() cannot use page_lock_anon_vma_read()	2195	* Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
2285	* because that depends on page_mapped(); but not all its usages	2196	* because that depends on page_mapped(); but not all its usages
2286	* are holding mmap_lock. Users without mmap_lock are required to	2197	* are holding mmap_lock. Users without mmap_lock are required to
2287	* take a reference count to prevent the anon_vma disappearing	2198	* take a reference count to prevent the anon_vma disappearing
2288	*/	2199	*/
2289	anon_vma = page_anon_vma(page);	2200	anon_vma = folio_anon_vma(folio);
2290	if (!anon_vma)	2201	if (!anon_vma)
2291	return NULL;	2202	return NULL;
2292		2203
@@ -2302,35 +2213,30 @@ static struct anon_vma rmap_walk_anon_lock(struct page page, · 9 unmodified lines
2302	*	2213	*
2303	* Find all the mappings of a page using the mapping pointer and the vma chains	2214	* Find all the mappings of a page using the mapping pointer and the vma chains
2304	* contained in the anon_vma struct it points to.	2215	* contained in the anon_vma struct it points to.
2305	*
2306	* When called from page_mlock(), the mmap_lock of the mm containing the vma
2307	* where the page was found will be held for write. So, we won't recheck
2308	* vm_flags for that VMA. That should be OK, because that vma shouldn't be
2309	* LOCKED.
2310	*/	2216	*/
2311	static void rmap_walk_anon(struct page page, struct rmap_walk_control rwc,	2217	static void rmap_walk_anon(struct folio *folio,
2312	bool locked)	2218	const struct rmap_walk_control *rwc, bool locked)
2313	{	2219	{
2314	struct anon_vma *anon_vma;	2220	struct anon_vma *anon_vma;
2315	pgoff_t pgoff_start, pgoff_end;	2221	pgoff_t pgoff_start, pgoff_end;
2316	struct anon_vma_chain *avc;	2222	struct anon_vma_chain *avc;
2317		2223
2318	if (locked) {	2224	if (locked) {
2319	anon_vma = page_anon_vma(page);	2225	anon_vma = folio_anon_vma(folio);
2320	/* anon_vma disappear under us? */	2226	/* anon_vma disappear under us? */
2321	VM_BUG_ON_PAGE(!anon_vma, page);	2227	VM_BUG_ON_FOLIO(!anon_vma, folio);
2322	} else {	2228	} else {
2323	anon_vma = rmap_walk_anon_lock(page, rwc);	2229	anon_vma = rmap_walk_anon_lock(folio, rwc);
2324	}	2230	}
2325	if (!anon_vma)	2231	if (!anon_vma)
2326	return;	2232	return;
2327		2233
2328	pgoff_start = page_to_pgoff(page);	2234	pgoff_start = folio_pgoff(folio);
2329	pgoff_end = pgoff_start + thp_nr_pages(page) - 1;	2235	pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
2330	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,	2236	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
2331	pgoff_start, pgoff_end) {	2237	pgoff_start, pgoff_end) {
2332	struct vm_area_struct *vma = avc->vma;	2238	struct vm_area_struct *vma = avc->vma;
2333	unsigned long address = vma_address(page, vma);	2239	unsigned long address = vma_address(&folio->page, vma);
2334		2240
2335	VM_BUG_ON_VMA(address == -EFAULT, vma);	2241	VM_BUG_ON_VMA(address == -EFAULT, vma);
2336	cond_resched();	2242	cond_resched();
@@ -2338,9 +2244,9 @@ static void rmap_walk_anon(struct page page, struct rmap_walk_control rwc, · 1 unmodified line
2338	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))	2244	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2339	continue;	2245	continue;
2340		2246
2341	if (!rwc->rmap_one(page, vma, address, rwc->arg))	2247	if (!rwc->rmap_one(folio, vma, address, rwc->arg))
2342	break;	2248	break;
2343	if (rwc->done && rwc->done(page))	2249	if (rwc->done && rwc->done(folio))
2344	break;	2250	break;
2345	}	2251	}
2346		2252
@@ -2355,16 +2261,11 @@ static void rmap_walk_anon(struct page page, struct rmap_walk_control rwc, · 8 unmodified lines
2355	*	2261	*
2356	* Find all the mappings of a page using the mapping pointer and the vma chains	2262	* Find all the mappings of a page using the mapping pointer and the vma chains
2357	* contained in the address_space struct it points to.	2263	* contained in the address_space struct it points to.
2358	*
2359	* When called from page_mlock(), the mmap_lock of the mm containing the vma
2360	* where the page was found will be held for write. So, we won't recheck
2361	* vm_flags for that VMA. That should be OK, because that vma shouldn't be
2362	* LOCKED.
2363	*/	2264	*/
2364	static void rmap_walk_file(struct page page, struct rmap_walk_control rwc,	2265	static void rmap_walk_file(struct folio *folio,
2365	bool locked)	2266	const struct rmap_walk_control *rwc, bool locked)
2366	{	2267	{
2367	struct address_space *mapping = page_mapping(page);	2268	struct address_space *mapping = folio_mapping(folio);
2368	pgoff_t pgoff_start, pgoff_end;	2269	pgoff_t pgoff_start, pgoff_end;
2369	struct vm_area_struct *vma;	2270	struct vm_area_struct *vma;
2370		2271
@@ -2374,18 +2275,18 @@ static void rmap_walk_file(struct page page, struct rmap_walk_control rwc, · 3 unmodified lines
2374	* structure at mapping cannot be freed and reused yet,	2275	* structure at mapping cannot be freed and reused yet,
2375	* so we can safely take mapping->i_mmap_rwsem.	2276	* so we can safely take mapping->i_mmap_rwsem.
2376	*/	2277	*/
2377	VM_BUG_ON_PAGE(!PageLocked(page), page);	2278	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2378		2279
2379	if (!mapping)	2280	if (!mapping)
2380	return;	2281	return;
2381		2282
2382	pgoff_start = page_to_pgoff(page);	2283	pgoff_start = folio_pgoff(folio);
2383	pgoff_end = pgoff_start + thp_nr_pages(page) - 1;	2284	pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
2384	if (!locked)	2285	if (!locked)
2385	i_mmap_lock_read(mapping);	2286	i_mmap_lock_read(mapping);
2386	vma_interval_tree_foreach(vma, &mapping->i_mmap,	2287	vma_interval_tree_foreach(vma, &mapping->i_mmap,
2387	pgoff_start, pgoff_end) {	2288	pgoff_start, pgoff_end) {
2388	unsigned long address = vma_address(page, vma);	2289	unsigned long address = vma_address(&folio->page, vma);
2389		2290
2390	VM_BUG_ON_VMA(address == -EFAULT, vma);	2291	VM_BUG_ON_VMA(address == -EFAULT, vma);
2391	cond_resched();	2292	cond_resched();
@@ -2393,9 +2294,9 @@ static void rmap_walk_file(struct page page, struct rmap_walk_control rwc, · 1 unmodified line
2393	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))	2294	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2394	continue;	2295	continue;
2395		2296
2396	if (!rwc->rmap_one(page, vma, address, rwc->arg))	2297	if (!rwc->rmap_one(folio, vma, address, rwc->arg))
2397	goto done;	2298	goto done;
2398	if (rwc->done && rwc->done(page))	2299	if (rwc->done && rwc->done(folio))
2399	goto done;	2300	goto done;
2400	}	2301	}
2401		2302
@@ -2404,25 +2305,25 @@ done: · 2 unmodified lines
2404	i_mmap_unlock_read(mapping);	2305	i_mmap_unlock_read(mapping);
2405	}	2306	}
2406		2307
2407	void rmap_walk(struct page page, struct rmap_walk_control rwc)	2308	void rmap_walk(struct folio folio, const struct rmap_walk_control rwc)
2408	{	2309	{
2409	if (unlikely(PageKsm(page)))	2310	if (unlikely(folio_test_ksm(folio)))
2410	rmap_walk_ksm(page, rwc);	2311	rmap_walk_ksm(folio, rwc);
2411	else if (PageAnon(page))	2312	else if (folio_test_anon(folio))
2412	rmap_walk_anon(page, rwc, false);	2313	rmap_walk_anon(folio, rwc, false);
2413	else	2314	else
2414	rmap_walk_file(page, rwc, false);	2315	rmap_walk_file(folio, rwc, false);
2415	}	2316	}
2416		2317
2417	/* Like rmap_walk, but caller holds relevant rmap lock */	2318	/* Like rmap_walk, but caller holds relevant rmap lock */
2418	void rmap_walk_locked(struct page page, struct rmap_walk_control rwc)	2319	void rmap_walk_locked(struct folio folio, const struct rmap_walk_control rwc)
2419	{	2320	{
2420	/* no ksm support for now */	2321	/* no ksm support for now */
2421	VM_BUG_ON_PAGE(PageKsm(page), page);	2322	VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
2422	if (PageAnon(page))	2323	if (folio_test_anon(folio))
2423	rmap_walk_anon(page, rwc, true);	2324	rmap_walk_anon(folio, rwc, true);
2424	else	2325	else
2425	rmap_walk_file(page, rwc, true);	2326	rmap_walk_file(folio, rwc, true);
2426	}	2327	}
2427		2328
2428	#ifdef CONFIG_HUGETLB_PAGE	2329	#ifdef CONFIG_HUGETLB_PAGE
@@ -2450,8 +2351,7 @@ void hugepage_add_new_anon_rmap(struct page *page, · 21 unmodified lines
2450	{	2351	{
2451	BUG_ON(address < vma->vm_start \|\| address >= vma->vm_end);	2352	BUG_ON(address < vma->vm_start \|\| address >= vma->vm_end);
2452	atomic_set(compound_mapcount_ptr(page), 0);	2353	atomic_set(compound_mapcount_ptr(page), 0);
2453	if (hpage_pincount_available(page))	2354	atomic_set(compound_pincount_ptr(page), 0);
2454	atomic_set(compound_pincount_ptr(page), 0);
2455		2355
2456	__page_set_anon_rmap(page, vma, address, 1);	2356	__page_set_anon_rmap(page, vma, address, 1);
2457	}	2357	}

mm/swap.c+68−105

@@ -74,8 +74,8 @@ static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = { · 73 unmodified lines
74	};	74	};
75		75
76	/*	76	/*
77	* This path almost never happens for VM activity - pages are normally	77	* This path almost never happens for VM activity - pages are normally freed
78	* freed via pagevecs. But it gets used by networking.	78	* via pagevecs. But it gets used by networking - and for compound pages.
79	*/	79	*/
80	static void __page_cache_release(struct page *page)	80	static void __page_cache_release(struct page *page)
81	{	81	{
@@ -89,6 +89,14 @@ static void __page_cache_release(struct page *page) · 7 unmodified lines
89	__clear_page_lru_flags(page);	89	__clear_page_lru_flags(page);
90	unlock_page_lruvec_irqrestore(lruvec, flags);	90	unlock_page_lruvec_irqrestore(lruvec, flags);
91	}	91	}
		92	/* See comment on PageMlocked in release_pages() */
		93	if (unlikely(PageMlocked(page))) {
		94	int nr_pages = thp_nr_pages(page);
		95
		96	__ClearPageMlocked(page);
		97	mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
		98	count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages);
		99	}
92	__ClearPageWaiters(page);	100	__ClearPageWaiters(page);
93	}	101	}
94		102
@@ -114,17 +122,9 @@ static void __put_compound_page(struct page *page) · 19 unmodified lines
114		122
115	void __put_page(struct page *page)	123	void __put_page(struct page *page)
116	{	124	{
117	if (is_zone_device_page(page)) {	125	if (unlikely(is_zone_device_page(page)))
118	put_dev_pagemap(page->pgmap);	126	free_zone_device_page(page);
119		127	else if (unlikely(PageCompound(page)))
120	/*
121	* The page belongs to the device that created pgmap. Do
122	* not return it to page allocator.
123	*/
124	return;
125	}
126
127	if (unlikely(PageCompound(page)))
128	__put_compound_page(page);	128	__put_compound_page(page);
129	else	129	else
130	__put_single_page(page);	130	__put_single_page(page);
@@ -482,22 +482,12 @@ EXPORT_SYMBOL(folio_add_lru); · 351 unmodified lines
482	void lru_cache_add_inactive_or_unevictable(struct page *page,	482	void lru_cache_add_inactive_or_unevictable(struct page *page,
483	struct vm_area_struct *vma)	483	struct vm_area_struct *vma)
484	{	484	{
485	bool unevictable;
486
487	VM_BUG_ON_PAGE(PageLRU(page), page);	485	VM_BUG_ON_PAGE(PageLRU(page), page);
488		486
489	unevictable = (vma->vm_flags & (VM_LOCKED \| VM_SPECIAL)) == VM_LOCKED;	487	if (unlikely((vma->vm_flags & (VM_LOCKED \| VM_SPECIAL)) == VM_LOCKED))
490	if (unlikely(unevictable) && !TestSetPageMlocked(page)) {	488	mlock_new_page(page);
491	int nr_pages = thp_nr_pages(page);	489	else
492	/*	490	lru_cache_add(page);
493	* We use the irq-unsafe __mod_zone_page_state because this
494	* counter is not modified from interrupt context, and the pte
495	* lock is held(spinlock), which implies preemption disabled.
496	*/
497	__mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages);
498	count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
499	}
500	lru_cache_add(page);
501	}	491	}
502		492
503	/*	493	/*
@@ -636,35 +626,37 @@ void lru_add_drain_cpu(int cpu) · 132 unmodified lines
636	pagevec_lru_move_fn(pvec, lru_lazyfree_fn);	626	pagevec_lru_move_fn(pvec, lru_lazyfree_fn);
637		627
638	activate_page_drain(cpu);	628	activate_page_drain(cpu);
		629	mlock_page_drain(cpu);
639	}	630	}
640		631
641	/**	632	/**
642	* deactivate_file_page - forcefully deactivate a file page	633	* deactivate_file_folio() - Forcefully deactivate a file folio.
643	* @page: page to deactivate	634	* @folio: Folio to deactivate.
644	*	635	*
645	* This function hints the VM that @page is a good reclaim candidate,	636	* This function hints to the VM that @folio is a good reclaim candidate,
646	* for example if its invalidation fails due to the page being dirty	637	* for example if its invalidation fails due to the folio being dirty
647	* or under writeback.	638	* or under writeback.
		639	*
		640	* Context: Caller holds a reference on the page.
648	*/	641	*/
649	void deactivate_file_page(struct page *page)	642	void deactivate_file_folio(struct folio *folio)
650	{	643	{
		644	struct pagevec *pvec;
		645
651	/*	646	/*
652	* In a workload with many unevictable page such as mprotect,	647	* In a workload with many unevictable pages such as mprotect,
653	* unevictable page deactivation for accelerating reclaim is pointless.	648	* unevictable folio deactivation for accelerating reclaim is pointless.
654	*/	649	*/
655	if (PageUnevictable(page))	650	if (folio_test_unevictable(folio))
656	return;	651	return;
657		652
658	if (likely(get_page_unless_zero(page))) {	653	folio_get(folio);
659	struct pagevec *pvec;	654	local_lock(&lru_pvecs.lock);
660		655	pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file);
661	local_lock(&lru_pvecs.lock);
662	pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file);
663		656
664	if (pagevec_add_and_need_flush(pvec, page))	657	if (pagevec_add_and_need_flush(pvec, &folio->page))
665	pagevec_lru_move_fn(pvec, lru_deactivate_file_fn);	658	pagevec_lru_move_fn(pvec, lru_deactivate_file_fn);
666	local_unlock(&lru_pvecs.lock);	659	local_unlock(&lru_pvecs.lock);
667	}
668	}	660	}
669		661
670	/*	662	/*
@@ -837,6 +829,7 @@ inline void __lru_add_drain_all(bool force_all_cpus) · 166 unmodified lines
837	pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) \|\|	829	pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) \|\|
838	pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) \|\|	830	pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) \|\|
839	need_activate_page_drain(cpu) \|\|	831	need_activate_page_drain(cpu) \|\|
		832	need_mlock_page_drain(cpu) \|\|
840	has_bh_in_lru(cpu, NULL)) {	833	has_bh_in_lru(cpu, NULL)) {
841	INIT_WORK(work, lru_add_drain_per_cpu);	834	INIT_WORK(work, lru_add_drain_per_cpu);
842	queue_work_on(cpu, mm_percpu_wq, work);	835	queue_work_on(cpu, mm_percpu_wq, work);
@@ -935,18 +928,10 @@ void release_pages(struct page **pages, int nr) · 92 unmodified lines
935	unlock_page_lruvec_irqrestore(lruvec, flags);	928	unlock_page_lruvec_irqrestore(lruvec, flags);
936	lruvec = NULL;	929	lruvec = NULL;
937	}	930	}
938	/*	931	if (put_devmap_managed_page(page))
939	* ZONE_DEVICE pages that return 'false' from
940	* page_is_devmap_managed() do not require special
941	* processing, and instead, expect a call to
942	* put_page_testzero().
943	*/
944	if (page_is_devmap_managed(page)) {
945	put_devmap_managed_page(page);
946	continue;	932	continue;
947	}
948	if (put_page_testzero(page))	933	if (put_page_testzero(page))
949	put_dev_pagemap(page->pgmap);	934	free_zone_device_page(page);
950	continue;	935	continue;
951	}	936	}
952		937
@@ -974,6 +959,18 @@ void release_pages(struct page **pages, int nr) · 21 unmodified lines
974	__clear_page_lru_flags(page);	959	__clear_page_lru_flags(page);
975	}	960	}
976		961
		962	/*
		963	* In rare cases, when truncation or holepunching raced with
		964	* munlock after VM_LOCKED was cleared, Mlocked may still be
		965	* found set here. This does not indicate a problem, unless
		966	* "unevictable_pgs_cleared" appears worryingly large.
		967	*/
		968	if (unlikely(PageMlocked(page))) {
		969	__ClearPageMlocked(page);
		970	dec_zone_page_state(page, NR_MLOCK);
		971	count_vm_event(UNEVICTABLE_PGCLEARED);
		972	}
		973
977	__ClearPageWaiters(page);	974	__ClearPageWaiters(page);
978		975
979	list_add(&page->lru, &pages_to_free);	976	list_add(&page->lru, &pages_to_free);
@@ -1014,43 +1011,32 @@ static void __pagevec_lru_add_fn(struct folio folio, struct lruvec lruvec) · 34 unmodified lines
1014		1011
1015	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);	1012	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
1016		1013
		1014	folio_set_lru(folio);
1017	/*	1015	/*
1018	* A folio becomes evictable in two ways:	1016	* Is an smp_mb__after_atomic() still required here, before
1019	* 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()].	1017	* folio_evictable() tests PageMlocked, to rule out the possibility
1020	* 2) Before acquiring LRU lock to put the folio on the correct LRU	1018	* of stranding an evictable folio on an unevictable LRU? I think
1021	* and then	1019	* not, because __munlock_page() only clears PageMlocked while the LRU
1022	* a) do PageLRU check with lock [check_move_unevictable_pages]	1020	* lock is held.
1023	* b) do PageLRU check before lock [clear_page_mlock]
1024	*
1025	* (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need
1026	* following strict ordering:
1027	*
1028	* #0: __pagevec_lru_add_fn #1: clear_page_mlock
1029	*	1021	*
1030	* folio_set_lru() folio_test_clear_mlocked()	1022	* (That is not true of __page_cache_release(), and not necessarily
1031	* smp_mb() // explicit ordering // above provides strict	1023	* true of release_pages(): but those only clear PageMlocked after
1032	* // ordering	1024	* put_page_testzero() has excluded any other users of the page.)
1033	* folio_test_mlocked() folio_test_lru()
1034	*
1035	*
1036	* if '#1' does not observe setting of PG_lru by '#0' and
1037	* fails isolation, the explicit barrier will make sure that
1038	* folio_evictable check will put the folio on the correct
1039	* LRU. Without smp_mb(), folio_set_lru() can be reordered
1040	* after folio_test_mlocked() check and can make '#1' fail the
1041	* isolation of the folio whose mlocked bit is cleared (#0 is
1042	* also looking at the same folio) and the evictable folio will
1043	* be stranded on an unevictable LRU.
1044	*/	1025	*/
1045	folio_set_lru(folio);
1046	smp_mb__after_atomic();
1047
1048	if (folio_evictable(folio)) {	1026	if (folio_evictable(folio)) {
1049	if (was_unevictable)	1027	if (was_unevictable)
1050	__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);	1028	__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
1051	} else {	1029	} else {
1052	folio_clear_active(folio);	1030	folio_clear_active(folio);
1053	folio_set_unevictable(folio);	1031	folio_set_unevictable(folio);
		1032	/*
		1033	* folio->mlock_count = !!folio_test_mlocked(folio)?
		1034	* But that leaves __mlock_page() in doubt whether another
		1035	* actor has already counted the mlock or not. Err on the
		1036	* safe side, underestimate, let page reclaim fix it, rather
		1037	* than leaving a page on the unevictable LRU indefinitely.
		1038	*/
		1039	folio->mlock_count = 0;
1054	if (!was_unevictable)	1040	if (!was_unevictable)
1055	__count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);	1041	__count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);
1056	}	1042	}
@@ -1158,26 +1144,3 @@ void __init swap_setup(void) · 101 unmodified lines
1158	* _really_ don't want to cluster much more	1144	* _really_ don't want to cluster much more
1159	*/	1145	*/
1160	}	1146	}
1161
1162	#ifdef CONFIG_DEV_PAGEMAP_OPS
1163	void put_devmap_managed_page(struct page *page)
1164	{
1165	int count;
1166
1167	if (WARN_ON_ONCE(!page_is_devmap_managed(page)))
1168	return;
1169
1170	count = page_ref_dec_return(page);
1171
1172	/*
1173	* devmap page refcounts are 1-based, rather than 0-based: if
1174	* refcount is 1, then the page is free and the refcount is
1175	* stable because nobody holds a reference on the page.
1176	*/
1177	if (count == 1)
1178	free_devmap_managed_page(page);
1179	else if (!count)
1180	__put_page(page);
1181	}
1182	EXPORT_SYMBOL(put_devmap_managed_page);
1183	#endif

mm/truncate.c+51−60

 	folio_clear_mappedtodisk(folio);
 }
-/*
- * This is for invalidate_mapping_pages().  That function can be called at
- * any time, and is not supposed to throw away dirty pages.  But pages can
- * be marked dirty at any time too, so use remove_mapping which safely
- * discards clean, unused pages.
- *
- * Returns non-zero if the page was successfully invalidated.
- */
-static int
-invalidate_complete_page(struct address_space *mapping, struct page *page)
-{
-	if (page->mapping != mapping)
-		return 0;
-	if (page_has_private(page) && !try_to_release_page(page, 0))
-		return 0;
-	return remove_mapping(mapping, page);
-}
 int truncate_inode_folio(struct address_space *mapping, struct folio *folio)
 {
 	if (folio->mapping != mapping)
 }
 EXPORT_SYMBOL(generic_error_remove_page);
-/*
+static long mapping_evict_folio(struct address_space *mapping,
+		struct folio *folio)
+{
+	if (folio_test_dirty(folio) || folio_test_writeback(folio))
+		return 0;
+	/* The refcount will be elevated if any page in the folio is mapped */
+	if (folio_ref_count(folio) >
+			folio_nr_pages(folio) + folio_has_private(folio) + 1)
+		return 0;
+	if (folio_has_private(folio) && !filemap_release_folio(folio, 0))
+		return 0;
+	return remove_mapping(mapping, folio);
+}
+/**
+ * invalidate_inode_page() - Remove an unused page from the pagecache.
+ * @page: The page to remove.
+ *
  * Safely invalidate one page from its pagecache mapping.
- * It only drops clean, unused pages. The page must be locked.
+ * It only drops clean, unused pages.
  *
- * Returns 1 if the page is successfully invalidated, otherwise 0.
+ * Context: Page must be locked.
+ * Return: The number of pages successfully removed.
  */
-int invalidate_inode_page(struct page *page)
+long invalidate_inode_page(struct page *page)
 {
-	struct address_space *mapping = page_mapping(page);
+	struct folio *folio = page_folio(page);
+	struct address_space *mapping = folio_mapping(folio);
+	/* The page may have been truncated before it was locked */
 	if (!mapping)
 		return 0;
-	if (PageDirty(page) || PageWriteback(page))
+	return mapping_evict_folio(mapping, folio);
-		return 0;
-	if (page_mapped(page))
-		return 0;
-	return invalidate_complete_page(mapping, page);
 }
 /**
 }
 EXPORT_SYMBOL(truncate_inode_pages_final);
-static unsigned long __invalidate_mapping_pages(struct address_space *mapping,
+/**
+ * invalidate_mapping_pagevec - Invalidate all the unlocked pages of one inode
+ * @mapping: the address_space which holds the pages to invalidate
+ * @start: the offset 'from' which to invalidate
+ * @end: the offset 'to' which to invalidate (inclusive)
+ * @nr_pagevec: invalidate failed page number for caller
+ *
+ * This helper is similar to invalidate_mapping_pages(), except that it accounts
+ * for pages that are likely on a pagevec and counts them in @nr_pagevec, which
+ * will be used by the caller.
+ */
+unsigned long invalidate_mapping_pagevec(struct address_space *mapping,
 		pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
 {
 	pgoff_t indices[PAGEVEC_SIZE];
 	folio_batch_init(&fbatch);
 	while (find_lock_entries(mapping, index, end, &fbatch, indices)) {
 		for (i = 0; i < folio_batch_count(&fbatch); i++) {
-			struct page *page = &fbatch.folios[i]->page;
+			struct folio *folio = fbatch.folios[i];
-			/* We rely upon deletion not changing page->index */
+			/* We rely upon deletion not changing folio->index */
 			index = indices[i];
-			if (xa_is_value(page)) {
+			if (xa_is_value(folio)) {
 				count += invalidate_exceptional_entry(mapping,
 								      index,
-								      page);
+								      folio);
 				continue;
 			}
-			index += thp_nr_pages(page) - 1;
+			index += folio_nr_pages(folio) - 1;
-			ret = invalidate_inode_page(page);
+			ret = mapping_evict_folio(mapping, folio);
-			unlock_page(page);
+			folio_unlock(folio);
 			/*
-			 * Invalidation is a hint that the page is no longer
+			 * Invalidation is a hint that the folio is no longer
 			 * of interest and try to speed up its reclaim.
 			 */
 			if (!ret) {
-				deactivate_file_page(page);
+				deactivate_file_folio(folio);
 				/* It is likely on the pagevec of a remote CPU */
 				if (nr_pagevec)
 					(*nr_pagevec)++;
 unsigned long invalidate_mapping_pages(struct address_space *mapping,
 		pgoff_t start, pgoff_t end)
 {
-	return __invalidate_mapping_pages(mapping, start, end, NULL);
+	return invalidate_mapping_pagevec(mapping, start, end, NULL);
 }
 EXPORT_SYMBOL(invalidate_mapping_pages);
-/**
- * invalidate_mapping_pagevec - Invalidate all the unlocked pages of one inode
- * @mapping: the address_space which holds the pages to invalidate
- * @start: the offset 'from' which to invalidate
- * @end: the offset 'to' which to invalidate (inclusive)
- * @nr_pagevec: invalidate failed page number for caller
- *
- * This helper is similar to invalidate_mapping_pages(), except that it accounts
- * for pages that are likely on a pagevec and counts them in @nr_pagevec, which
- * will be used by the caller.
- */
-void invalidate_mapping_pagevec(struct address_space *mapping,
-		pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
-{
-	__invalidate_mapping_pages(mapping, start, end, nr_pagevec);
-}
 /*
- * This is like invalidate_complete_page(), except it ignores the page's
+ * This is like invalidate_inode_page(), except it ignores the page's
  * refcount.  We do this because invalidate_inode_pages2() needs stronger
  * invalidation guarantees, and cannot afford to leave pages behind because
  * shrink_page_list() has a temp ref on them, or because they're transiently

mm/userfaultfd.c+8−6

 	if (!pte_none(*dst_pte))
 		goto out_unlock;
-	if (page_in_cache)
+	if (page_in_cache) {
-		page_add_file_rmap(page, false);
+		/* Usually, cache pages are already added to LRU */
-	else
+		if (newly_allocated)
+			lru_cache_add(page);
+		page_add_file_rmap(page, dst_vma, false);
+	} else {
 		page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
+		lru_cache_add_inactive_or_unevictable(page, dst_vma);
+	}
 	/*
 	 * Must happen after rmap, as mm_counter() checks mapping (via
 	 */
 	inc_mm_counter(dst_mm, mm_counter(page));
-	if (newly_allocated)
-		lru_cache_add_inactive_or_unevictable(page, dst_vma);
 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
 	/* No need to invalidate - it was non-present before */

mm/util.c+34−2

@@ -681,9 +681,8 @@ bool folio_mapped(struct folio *folio) · 680 unmodified lines
681	}	681	}
682	EXPORT_SYMBOL(folio_mapped);	682	EXPORT_SYMBOL(folio_mapped);
683		683
684	struct anon_vma page_anon_vma(struct page page)	684	struct anon_vma folio_anon_vma(struct folio folio)
685	{	685	{
686	struct folio *folio = page_folio(page);
687	unsigned long mapping = (unsigned long)folio->mapping;	686	unsigned long mapping = (unsigned long)folio->mapping;
688		687
689	if ((mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)	688	if ((mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
@@ -742,6 +741,39 @@ int __page_mapcount(struct page *page) · 52 unmodified lines
742	}	741	}
743	EXPORT_SYMBOL_GPL(__page_mapcount);	742	EXPORT_SYMBOL_GPL(__page_mapcount);
744		743
		744	/**
		745	* folio_mapcount() - Calculate the number of mappings of this folio.
		746	* @folio: The folio.
		747	*
		748	* A large folio tracks both how many times the entire folio is mapped,
		749	* and how many times each individual page in the folio is mapped.
		750	* This function calculates the total number of times the folio is
		751	* mapped.
		752	*
		753	* Return: The number of times this folio is mapped.
		754	*/
		755	int folio_mapcount(struct folio *folio)
		756	{
		757	int i, compound, nr, ret;
		758
		759	if (likely(!folio_test_large(folio)))
		760	return atomic_read(&folio->_mapcount) + 1;
		761
		762	compound = folio_entire_mapcount(folio);
		763	nr = folio_nr_pages(folio);
		764	if (folio_test_hugetlb(folio))
		765	return compound;
		766	ret = compound;
		767	for (i = 0; i < nr; i++)
		768	ret += atomic_read(&folio_page(folio, i)->_mapcount) + 1;
		769	/* File pages has compound_mapcount included in _mapcount */
		770	if (!folio_test_anon(folio))
		771	return ret - compound * nr;
		772	if (folio_test_double_map(folio))
		773	ret -= nr;
		774	return ret;
		775	}
		776
745	/**	777	/**
746	* folio_copy - Copy the contents of one folio to another.	778	* folio_copy - Copy the contents of one folio to another.
747	* @dst: Folio to copy to.	779	* @dst: Folio to copy to.

mm/vmscan.c+156−149

 		drop_slab_node(nid);
 }
-static inline int is_page_cache_freeable(struct page *page)
+static inline int is_page_cache_freeable(struct folio *folio)
 {
 	/*
 	 * A freeable page cache page is referenced only by the caller
 	 * that isolated the page, the page cache and optional buffer
 	 * heads at page->private.
 	 */
-	int page_cache_pins = thp_nr_pages(page);
+	return folio_ref_count(folio) - folio_test_private(folio) ==
-	return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
+		1 + folio_nr_pages(folio);
 }
 /*
- * We detected a synchronous write error writing a page out.  Probably
+ * We detected a synchronous write error writing a folio out.  Probably
  * -ENOSPC.  We need to propagate that into the address_space for a subsequent
  * fsync(), msync() or close().
  *
  * The tricky part is that after writepage we cannot touch the mapping: nothing
- * prevents it from being freed up.  But we have a ref on the page and once
+ * prevents it from being freed up.  But we have a ref on the folio and once
- * that page is locked, the mapping is pinned.
+ * that folio is locked, the mapping is pinned.
  *
- * We're allowed to run sleeping lock_page() here because we know the caller has
+ * We're allowed to run sleeping folio_lock() here because we know the caller has
  * __GFP_FS.
  */
 static void handle_write_error(struct address_space *mapping,
-				struct page *page, int error)
+				struct folio *folio, int error)
 {
-	lock_page(page);
+	folio_lock(folio);
-	if (page_mapping(page) == mapping)
+	if (folio_mapping(folio) == mapping)
 		mapping_set_error(mapping, error);
-	unlock_page(page);
+	folio_unlock(folio);
 }
 static bool skip_throttle_noprogress(pg_data_t *pgdat)
  * pageout is called by shrink_page_list() for each dirty page.
  * Calls ->writepage().
  */
-static pageout_t pageout(struct page *page, struct address_space *mapping)
+static pageout_t pageout(struct folio *folio, struct address_space *mapping)
 {
 	/*
-	 * If the page is dirty, only perform writeback if that write
+	 * If the folio is dirty, only perform writeback if that write
 	 * will be non-blocking.  To prevent this allocation from being
 	 * stalled by pagecache activity.  But note that there may be
 	 * stalls if we need to run get_block().  We could test
 	 * PagePrivate for that.
 	 *
 	 * If this process is currently in __generic_file_write_iter() against
-	 * this page's queue, we can perform writeback even if that
+	 * this folio's queue, we can perform writeback even if that
 	 * will block.
 	 *
-	 * If the page is swapcache, write it back even if that would
+	 * If the folio is swapcache, write it back even if that would
 	 * block, for some throttling. This happens by accident, because
 	 * swap_backing_dev_info is bust: it doesn't reflect the
 	 * congestion state of the swapdevs.  Easy to fix, if needed.
 	 */
-	if (!is_page_cache_freeable(page))
+	if (!is_page_cache_freeable(folio))
 		return PAGE_KEEP;
 	if (!mapping) {
 		/*
-		 * Some data journaling orphaned pages can have
+		 * Some data journaling orphaned folios can have
-		 * page->mapping == NULL while being dirty with clean buffers.
+		 * folio->mapping == NULL while being dirty with clean buffers.
 		 */
-		if (page_has_private(page)) {
+		if (folio_test_private(folio)) {
-			if (try_to_free_buffers(page)) {
+			if (try_to_free_buffers(&folio->page)) {
-				ClearPageDirty(page);
+				folio_clear_dirty(folio);
-				pr_info("%s: orphaned page\n", __func__);
+				pr_info("%s: orphaned folio\n", __func__);
 				return PAGE_CLEAN;
 			}
 		}
 	if (mapping->a_ops->writepage == NULL)
 		return PAGE_ACTIVATE;
-	if (clear_page_dirty_for_io(page)) {
+	if (folio_clear_dirty_for_io(folio)) {
 		int res;
 		struct writeback_control wbc = {
 			.sync_mode = WB_SYNC_NONE,
 			.for_reclaim = 1,
 		};
-		SetPageReclaim(page);
+		folio_set_reclaim(folio);
-		res = mapping->a_ops->writepage(page, &wbc);
+		res = mapping->a_ops->writepage(&folio->page, &wbc);
 		if (res < 0)
-			handle_write_error(mapping, page, res);
+			handle_write_error(mapping, folio, res);
 		if (res == AOP_WRITEPAGE_ACTIVATE) {
-			ClearPageReclaim(page);
+			folio_clear_reclaim(folio);
 			return PAGE_ACTIVATE;
 		}
-		if (!PageWriteback(page)) {
+		if (!folio_test_writeback(folio)) {
 			/* synchronous write or broken a_ops? */
-			ClearPageReclaim(page);
+			folio_clear_reclaim(folio);
 		}
-		trace_mm_vmscan_writepage(page);
+		trace_mm_vmscan_write_folio(folio);
-		inc_node_page_state(page, NR_VMSCAN_WRITE);
+		node_stat_add_folio(folio, NR_VMSCAN_WRITE);
 		return PAGE_SUCCESS;
 	}
  * Same as remove_mapping, but if the page is removed from the mapping, it
  * gets returned with a refcount of 0.
  */
-static int __remove_mapping(struct address_space *mapping, struct page *page,
+static int __remove_mapping(struct address_space *mapping, struct folio *folio,
 			    bool reclaimed, struct mem_cgroup *target_memcg)
 {
 	int refcount;
 	void *shadow = NULL;
-	BUG_ON(!PageLocked(page));
+	BUG_ON(!folio_test_locked(folio));
-	BUG_ON(mapping != page_mapping(page));
+	BUG_ON(mapping != folio_mapping(folio));
-	if (!PageSwapCache(page))
+	if (!folio_test_swapcache(folio))
 		spin_lock(&mapping->host->i_lock);
 	xa_lock_irq(&mapping->i_pages);
 	/*
 	 * Note that if SetPageDirty is always performed via set_page_dirty,
 	 * and thus under the i_pages lock, then this ordering is not required.
 	 */
-	refcount = 1 + compound_nr(page);
+	refcount = 1 + folio_nr_pages(folio);
-	if (!page_ref_freeze(page, refcount))
+	if (!folio_ref_freeze(folio, refcount))
 		goto cannot_free;
 	/* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
-	if (unlikely(PageDirty(page))) {
+	if (unlikely(folio_test_dirty(folio))) {
-		page_ref_unfreeze(page, refcount);
+		folio_ref_unfreeze(folio, refcount);
 		goto cannot_free;
 	}
-	if (PageSwapCache(page)) {
+	if (folio_test_swapcache(folio)) {
-		swp_entry_t swap = { .val = page_private(page) };
+		swp_entry_t swap = folio_swap_entry(folio);
-		mem_cgroup_swapout(page, swap);
+		mem_cgroup_swapout(folio, swap);
 		if (reclaimed && !mapping_exiting(mapping))
-			shadow = workingset_eviction(page, target_memcg);
+			shadow = workingset_eviction(folio, target_memcg);
-		__delete_from_swap_cache(page, swap, shadow);
+		__delete_from_swap_cache(&folio->page, swap, shadow);
 		xa_unlock_irq(&mapping->i_pages);
-		put_swap_page(page, swap);
+		put_swap_page(&folio->page, swap);
 	} else {
 		void (*freepage)(struct page *);
 		 * exceptional entries and shadow exceptional entries in the
 		 * same address_space.
 		 */
-		if (reclaimed && page_is_file_lru(page) &&
+		if (reclaimed && folio_is_file_lru(folio) &&
 		    !mapping_exiting(mapping) && !dax_mapping(mapping))
-			shadow = workingset_eviction(page, target_memcg);
+			shadow = workingset_eviction(folio, target_memcg);
-		__delete_from_page_cache(page, shadow);
+		__filemap_remove_folio(folio, shadow);
 		xa_unlock_irq(&mapping->i_pages);
 		if (mapping_shrinkable(mapping))
 			inode_add_lru(mapping->host);
 		spin_unlock(&mapping->host->i_lock);
 		if (freepage != NULL)
-			freepage(page);
+			freepage(&folio->page);
 	}
 	return 1;
 cannot_free:
 	xa_unlock_irq(&mapping->i_pages);
-	if (!PageSwapCache(page))
+	if (!folio_test_swapcache(folio))
 		spin_unlock(&mapping->host->i_lock);
 	return 0;
 }
-/*
+/**
- * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
+ * remove_mapping() - Attempt to remove a folio from its mapping.
- * someone else has a ref on the page, abort and return 0.  If it was
+ * @mapping: The address space.
- * successfully detached, return 1.  Assumes the caller has a single ref on
+ * @folio: The folio to remove.
- * this page.
+ *
+ * If the folio is dirty, under writeback or if someone else has a ref
+ * on it, removal will fail.
+ * Return: The number of pages removed from the mapping.  0 if the folio
+ * could not be removed.
+ * Context: The caller should have a single refcount on the folio and
+ * hold its lock.
  */
-int remove_mapping(struct address_space *mapping, struct page *page)
+long remove_mapping(struct address_space *mapping, struct folio *folio)
 {
-	if (__remove_mapping(mapping, page, false, NULL)) {
+	if (__remove_mapping(mapping, folio, false, NULL)) {
 		/*
-		 * Unfreezing the refcount with 1 rather than 2 effectively
+		 * Unfreezing the refcount with 1 effectively
 		 * drops the pagecache ref for us without requiring another
 		 * atomic operation.
 		 */
-		page_ref_unfreeze(page, 1);
+		folio_ref_unfreeze(folio, 1);
-		return 1;
+		return folio_nr_pages(folio);
 	}
 	return 0;
 }
 /**
- * putback_lru_page - put previously isolated page onto appropriate LRU list
+ * folio_putback_lru - Put previously isolated folio onto appropriate LRU list.
- * @page: page to be put back to appropriate lru list
+ * @folio: Folio to be returned to an LRU list.
  *
- * Add previously isolated @page to appropriate LRU list.
+ * Add previously isolated @folio to appropriate LRU list.
- * Page may still be unevictable for other reasons.
+ * The folio may still be unevictable for other reasons.
  *
- * lru_lock must not be held, interrupts must be enabled.
+ * Context: lru_lock must not be held, interrupts must be enabled.
  */
-void putback_lru_page(struct page *page)
+void folio_putback_lru(struct folio *folio)
 {
-	lru_cache_add(page);
+	folio_add_lru(folio);
-	put_page(page);		/* drop ref from isolate */
+	folio_put(folio);		/* drop ref from isolate */
 }
 enum page_references {
 	PAGEREF_ACTIVATE,
 };
-static enum page_references page_check_references(struct page *page,
+static enum page_references folio_check_references(struct folio *folio,
 						  struct scan_control *sc)
 {
-	int referenced_ptes, referenced_page;
+	int referenced_ptes, referenced_folio;
 	unsigned long vm_flags;
-	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
+	referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,
-					  &vm_flags);
+					   &vm_flags);
-	referenced_page = TestClearPageReferenced(page);
+	referenced_folio = folio_test_clear_referenced(folio);
 	/*
-	 * Mlock lost the isolation race with us.  Let try_to_unmap()
+	 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
-	 * move the page to the unevictable list.
+	 * Let the folio, now marked Mlocked, be moved to the unevictable list.
 	 */
 	if (vm_flags & VM_LOCKED)
-		return PAGEREF_RECLAIM;
+		return PAGEREF_ACTIVATE;
 	if (referenced_ptes) {
 		/*
-		 * All mapped pages start out with page table
+		 * All mapped folios start out with page table
 		 * references from the instantiating fault, so we need
-		 * to look twice if a mapped file/anon page is used more
+		 * to look twice if a mapped file/anon folio is used more
 		 * than once.
 		 *
 		 * Mark it and spare it for another trip around the
 		 * inactive list.  Another page table reference will
 		 * lead to its activation.
 		 *
-		 * Note: the mark is set for activated pages as well
+		 * Note: the mark is set for activated folios as well
-		 * so that recently deactivated but used pages are
+		 * so that recently deactivated but used folios are
 		 * quickly recovered.
 		 */
-		SetPageReferenced(page);
+		folio_set_referenced(folio);
-		if (referenced_page || referenced_ptes > 1)
+		if (referenced_folio || referenced_ptes > 1)
 			return PAGEREF_ACTIVATE;
 		/*
-		 * Activate file-backed executable pages after first usage.
+		 * Activate file-backed executable folios after first usage.
 		 */
-		if ((vm_flags & VM_EXEC) && !PageSwapBacked(page))
+		if ((vm_flags & VM_EXEC) && !folio_test_swapbacked(folio))
 			return PAGEREF_ACTIVATE;
 		return PAGEREF_KEEP;
 	}
-	/* Reclaim if clean, defer dirty pages to writeback */
+	/* Reclaim if clean, defer dirty folios to writeback */
-	if (referenced_page && !PageSwapBacked(page))
+	if (referenced_folio && !folio_test_swapbacked(folio))
 		return PAGEREF_RECLAIM_CLEAN;
 	return PAGEREF_RECLAIM;
 }
 /* Check if a page is dirty or under writeback */
-static void page_check_dirty_writeback(struct page *page,
+static void folio_check_dirty_writeback(struct folio *folio,
 				       bool *dirty, bool *writeback)
 {
 	struct address_space *mapping;
 	 * Anonymous pages are not handled by flushers and must be written
 	 * from reclaim context. Do not stall reclaim based on them
 	 */
-	if (!page_is_file_lru(page) ||
+	if (!folio_is_file_lru(folio) ||
-	    (PageAnon(page) && !PageSwapBacked(page))) {
+	    (folio_test_anon(folio) && !folio_test_swapbacked(folio))) {
 		*dirty = false;
 		*writeback = false;
 		return;
 	}
-	/* By default assume that the page flags are accurate */
+	/* By default assume that the folio flags are accurate */
-	*dirty = PageDirty(page);
+	*dirty = folio_test_dirty(folio);
-	*writeback = PageWriteback(page);
+	*writeback = folio_test_writeback(folio);
 	/* Verify dirty/writeback state if the filesystem supports it */
-	if (!page_has_private(page))
+	if (!folio_test_private(folio))
 		return;
-	mapping = page_mapping(page);
+	mapping = folio_mapping(folio);
 	if (mapping && mapping->a_ops->is_dirty_writeback)
-		mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
+		mapping->a_ops->is_dirty_writeback(&folio->page, dirty, writeback);
 }
 static struct page *alloc_demote_page(struct page *page, unsigned long node)
 	while (!list_empty(page_list)) {
 		struct address_space *mapping;
 		struct page *page;
+		struct folio *folio;
 		enum page_references references = PAGEREF_RECLAIM;
 		bool dirty, writeback, may_enter_fs;
 		unsigned int nr_pages;
 		cond_resched();
-		page = lru_to_page(page_list);
+		folio = lru_to_folio(page_list);
-		list_del(&page->lru);
+		list_del(&folio->lru);
+		page = &folio->page;
 		if (!trylock_page(page))
 			goto keep;
 		 * reclaim_congested. kswapd will stall and start writing
 		 * pages if the tail of the LRU is all dirty unqueued pages.
 		 */
-		page_check_dirty_writeback(page, &dirty, &writeback);
+		folio_check_dirty_writeback(folio, &dirty, &writeback);
 		if (dirty || writeback)
-			stat->nr_dirty++;
+			stat->nr_dirty += nr_pages;
 		if (dirty && !writeback)
-			stat->nr_unqueued_dirty++;
+			stat->nr_unqueued_dirty += nr_pages;
 		/*
 		 * Treat this page as congested if the underlying BDI is or if
 		 */
 		mapping = page_mapping(page);
 		if (writeback && PageReclaim(page))
-			stat->nr_congested++;
+			stat->nr_congested += nr_pages;
 		/*
 		 * If a page at the tail of the LRU is under writeback, there
 			if (current_is_kswapd() &&
 			    PageReclaim(page) &&
 			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
-				stat->nr_immediate++;
+				stat->nr_immediate += nr_pages;
 				goto activate_locked;
 			/* Case 2 above */
 				 * and it's also appropriate in global reclaim.
 				 */
 				SetPageReclaim(page);
-				stat->nr_writeback++;
+				stat->nr_writeback += nr_pages;
 				goto activate_locked;
 			/* Case 3 above */
 		}
 		if (!ignore_references)
-			references = page_check_references(page, sc);
+			references = folio_check_references(folio, sc);
 		switch (references) {
 		case PAGEREF_ACTIVATE:
 			if (!PageSwapCache(page)) {
 				if (!(sc->gfp_mask & __GFP_IO))
 					goto keep_locked;
-				if (page_maybe_dma_pinned(page))
+				if (folio_maybe_dma_pinned(folio))
 					goto keep_locked;
 				if (PageTransHuge(page)) {
 					/* cannot split THP, skip it */
-					if (!can_split_huge_page(page, NULL))
+					if (!can_split_folio(folio, NULL))
 						goto activate_locked;
 					/*
 					 * Split pages without a PMD map right
 					 * away. Chances are some or all of the
 					 * tail pages can be freed without IO.
 					 */
-					if (!compound_mapcount(page) &&
+					if (!folio_entire_mapcount(folio) &&
-					    split_huge_page_to_list(page,
+					    split_folio_to_list(folio,
-								    page_list))
+								page_list))
 						goto activate_locked;
 				}
 				if (!add_to_swap(page)) {
 					if (!PageTransHuge(page))
 						goto activate_locked_split;
 					/* Fallback to swap normal pages */
-					if (split_huge_page_to_list(page,
+					if (split_folio_to_list(folio,
-								    page_list))
+								page_list))
 						goto activate_locked;
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 					count_vm_event(THP_SWPOUT_FALLBACK);
 				/* Adding to swap updated mapping */
 				mapping = page_mapping(page);
 			}
-		} else if (unlikely(PageTransHuge(page))) {
+		} else if (PageSwapBacked(page) && PageTransHuge(page)) {
-			/* Split file THP */
+			/* Split shmem THP */
-			if (split_huge_page_to_list(page, page_list))
+			if (split_folio_to_list(folio, page_list))
 				goto keep_locked;
 		}
 			enum ttu_flags flags = TTU_BATCH_FLUSH;
 			bool was_swapbacked = PageSwapBacked(page);
-			if (unlikely(PageTransHuge(page)))
+			if (PageTransHuge(page) &&
+					thp_order(page) >= HPAGE_PMD_ORDER)
 				flags |= TTU_SPLIT_HUGE_PMD;
-			try_to_unmap(page, flags);
+			try_to_unmap(folio, flags);
 			if (page_mapped(page)) {
 				stat->nr_unmap_fail += nr_pages;
 				if (!was_swapbacked && PageSwapBacked(page))
 			 * starts and then write it out here.
 			 */
 			try_to_unmap_flush_dirty();
-			switch (pageout(page, mapping)) {
+			switch (pageout(folio, mapping)) {
 			case PAGE_KEEP:
 				goto keep_locked;
 			case PAGE_ACTIVATE:
 				goto activate_locked;
 			case PAGE_SUCCESS:
-				stat->nr_pageout += thp_nr_pages(page);
+				stat->nr_pageout += nr_pages;
 				if (PageWriteback(page))
 					goto keep;
 			 */
 			count_vm_event(PGLAZYFREED);
 			count_memcg_page_event(page, PGLAZYFREED);
-		} else if (!mapping || !__remove_mapping(mapping, page, true,
+		} else if (!mapping || !__remove_mapping(mapping, folio, true,
 							 sc->target_mem_cgroup))
 			goto keep_locked;
 }
 /**
- * isolate_lru_page - tries to isolate a page from its LRU list
+ * folio_isolate_lru() - Try to isolate a folio from its LRU list.
- * @page: page to isolate from its LRU list
+ * @folio: Folio to isolate from its LRU list.
  *
- * Isolates a @page from an LRU list, clears PageLRU and adjusts the
+ * Isolate a @folio from an LRU list and adjust the vmstat statistic
- * vmstat statistic corresponding to whatever LRU list the page was on.
+ * corresponding to whatever LRU list the folio was on.
  *
- * Returns 0 if the page was removed from an LRU list.
+ * The folio will have its LRU flag cleared.  If it was found on the
- * Returns -EBUSY if the page was not on an LRU list.
+ * active list, it will have the Active flag set.  If it was found on the
- *
+ * unevictable list, it will have the Unevictable flag set.  These flags
- * The returned page will have PageLRU() cleared.  If it was found on
- * the active list, it will have PageActive set.  If it was found on
- * the unevictable list, it will have the PageUnevictable bit set. That flag
  * may need to be cleared by the caller before letting the page go.
  *
- * The vmstat statistic corresponding to the list on which the page was
+ * Context:
- * found will be decremented.
- *
- * Restrictions:
  *
  * (1) Must be called with an elevated refcount on the page. This is a
- *     fundamental difference from isolate_lru_pages (which is called
+ *     fundamental difference from isolate_lru_pages() (which is called
  *     without a stable reference).
- * (2) the lru_lock must not be held.
+ * (2) The lru_lock must not be held.
- * (3) interrupts must be enabled.
+ * (3) Interrupts must be enabled.
+ *
+ * Return: 0 if the folio was removed from an LRU list.
+ * -EBUSY if the folio was not on an LRU list.
  */
-int isolate_lru_page(struct page *page)
+int folio_isolate_lru(struct folio *folio)
 {
-	struct folio *folio = page_folio(page);
 	int ret = -EBUSY;
-	VM_BUG_ON_PAGE(!page_count(page), page);
+	VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio);
-	WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
-	if (TestClearPageLRU(page)) {
+	if (folio_test_clear_lru(folio)) {
 		struct lruvec *lruvec;
-		get_page(page);
+		folio_get(folio);
 		lruvec = folio_lruvec_lock_irq(folio);
-		del_page_from_lru_list(page, lruvec);
+		lruvec_del_folio(lruvec, folio);
 		unlock_page_lruvec_irq(lruvec);
 		ret = 0;
 	}
  *
  * If the pages are mostly unmapped, the processing is fast and it is
  * appropriate to hold lru_lock across the whole operation.  But if
- * the pages are mapped, the processing is slow (page_referenced()), so
+ * the pages are mapped, the processing is slow (folio_referenced()), so
  * we should drop lru_lock around each page.  It's impossible to balance
  * this, so instead we remove the pages from the LRU while processing them.
  * It is safe to rely on PG_active against the non-LRU pages in here because
 	LIST_HEAD(l_hold);	/* The pages which were snipped off */
 	LIST_HEAD(l_active);
 	LIST_HEAD(l_inactive);
-	struct page *page;
 	unsigned nr_deactivate, nr_activate;
 	unsigned nr_rotated = 0;
 	int file = is_file_lru(lru);
 	spin_unlock_irq(&lruvec->lru_lock);
 	while (!list_empty(&l_hold)) {
+		struct folio *folio;
+		struct page *page;
 		cond_resched();
-		page = lru_to_page(&l_hold);
+		folio = lru_to_folio(&l_hold);
-		list_del(&page->lru);
+		list_del(&folio->lru);
+		page = &folio->page;
 		if (unlikely(!page_evictable(page))) {
 			putback_lru_page(page);
 			}
 		}
-		if (page_referenced(page, 0, sc->target_mem_cgroup,
+		if (folio_referenced(folio, 0, sc->target_mem_cgroup,
-				    &vm_flags)) {
+				     &vm_flags)) {
 			/*
 			 * Identify referenced, file-backed active pages and
 			 * give them one more trip around the active list. So

mm/workingset.c+13−12

@@ -245,31 +245,32 @@ void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages) · 244 unmodified lines
245	}	245	}
246		246
247	/**	247	/**
248	* workingset_eviction - note the eviction of a page from memory	248	* workingset_eviction - note the eviction of a folio from memory
249	* @target_memcg: the cgroup that is causing the reclaim	249	* @target_memcg: the cgroup that is causing the reclaim
250	* @page: the page being evicted	250	* @folio: the folio being evicted
251	*	251	*
252	* Return: a shadow entry to be stored in @page->mapping->i_pages in place	252	* Return: a shadow entry to be stored in @folio->mapping->i_pages in place
253	* of the evicted @page so that a later refault can be detected.	253	* of the evicted @folio so that a later refault can be detected.
254	*/	254	*/
255	void workingset_eviction(struct page page, struct mem_cgroup *target_memcg)	255	void workingset_eviction(struct folio folio, struct mem_cgroup *target_memcg)
256	{	256	{
257	struct pglist_data *pgdat = page_pgdat(page);	257	struct pglist_data *pgdat = folio_pgdat(folio);
258	unsigned long eviction;	258	unsigned long eviction;
259	struct lruvec *lruvec;	259	struct lruvec *lruvec;
260	int memcgid;	260	int memcgid;
261		261
262	/* Page is fully exclusive and pins page's memory cgroup pointer */	262	/* Folio is fully exclusive and pins folio's memory cgroup pointer */
263	VM_BUG_ON_PAGE(PageLRU(page), page);	263	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
264	VM_BUG_ON_PAGE(page_count(page), page);	264	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
265	VM_BUG_ON_PAGE(!PageLocked(page), page);	265	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
266		266
267	lruvec = mem_cgroup_lruvec(target_memcg, pgdat);	267	lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
268	/* XXX: target_memcg can be NULL, go through lruvec */	268	/* XXX: target_memcg can be NULL, go through lruvec */
269	memcgid = mem_cgroup_id(lruvec_memcg(lruvec));	269	memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
270	eviction = atomic_long_read(&lruvec->nonresident_age);	270	eviction = atomic_long_read(&lruvec->nonresident_age);
271	workingset_age_nonresident(lruvec, thp_nr_pages(page));	271	workingset_age_nonresident(lruvec, folio_nr_pages(folio));
272	return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));	272	return pack_shadow(memcgid, pgdat, eviction,
		273	folio_test_workingset(folio));
273	}	274	}
274		275
275	/**	276	/**

tools/testing/selftests/vm/transhuge-stress.c+24−11

 #define PAGEMAP_PFN(ent)	((ent) & ((1ull << 55) - 1))
 int pagemap_fd;
+int backing_fd = -1;
+int mmap_flags = MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE;
+#define PROT_RW (PROT_READ | PROT_WRITE)
 int64_t allocate_transhuge(void *ptr)
 {
 	uint64_t ent[2];
 	/* drop pmd */
-	if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
+	if (mmap(ptr, HPAGE_SIZE, PROT_RW, MAP_FIXED | mmap_flags,
-				MAP_FIXED | MAP_ANONYMOUS |
+		 backing_fd, 0) != ptr)
-				MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
 		errx(2, "mmap transhuge");
 	if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
 	size_t ram, len;
 	void *ptr, *p;
 	struct timespec a, b;
+	int i = 0;
+	char *name = NULL;
 	double s;
 	uint8_t *map;
 	size_t map_len;
 		ram = SIZE_MAX / 4;
 	else
 		ram *= sysconf(_SC_PAGESIZE);
+	len = ram;
+	while (++i < argc) {
+		if (!strcmp(argv[i], "-h"))
+			errx(1, "usage: %s [size in MiB]", argv[0]);
+		else if (!strcmp(argv[i], "-f"))
+			name = argv[++i];
+		else
+			len = atoll(argv[i]) << 20;
+	}
-	if (argc == 1)
+	if (name) {
-		len = ram;
+		backing_fd = open(name, O_RDWR);
-	else if (!strcmp(argv[1], "-h"))
+		if (backing_fd == -1)
-		errx(1, "usage: %s [size in MiB]", argv[0]);
+			errx(2, "open %s", name);
-	else
+		mmap_flags = MAP_SHARED;
-		len = atoll(argv[1]) << 20;
+	}
 	warnx("allocate %zd transhuge pages, using %zd MiB virtual memory"
 	      " and %zd MiB of ram", len >> HPAGE_SHIFT, len >> 20,
 		err(2, "open pagemap");
 	len -= len % HPAGE_SIZE;
-	ptr = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
+	ptr = mmap(NULL, len + HPAGE_SIZE, PROT_RW, mmap_flags, backing_fd, 0);
-			MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
 	if (ptr == MAP_FAILED)
 		err(2, "initial mmap");
 	ptr += HPAGE_SIZE - (uintptr_t)ptr % HPAGE_SIZE;