Merge tag 'folio-5.18c' of git://git.infradead.org/users/willy/pagecache

Pull folio updates from Matthew Wilcox: - Rewrite how munlock works to massively reduce the contention on i_mmap_rwsem (Hugh Dickins): https://lore.kernel.org/linux-mm/8e4356d-9622-a7f0-b2c-f116b5f2efea@google.com/ - Sort out the page refcount mess for ZONE_DEVICE pages (Christoph Hellwig): https://lore.kernel.org/linux-mm/20220210072828.2930359-1-hch@lst.de/ - Convert GUP to use folios and make pincount available for order-1 pages. (Matthew Wilcox) - Convert a few more truncation functions to use folios (Matthew Wilcox) - Convert page_vma_mapped_walk to use PFNs instead of pages (Matthew Wilcox) - Convert rmap_walk to use folios (Matthew Wilcox) - Convert most of shrink_page_list() to use a folio (Matthew Wilcox) - Add support for creating large folios in readahead (Matthew Wilcox) * tag 'folio-5.18c' of git://git.infradead.org/users/willy/pagecache: (114 commits) mm/damon: minor cleanup for damon_pa_young selftests/vm/transhuge-stress: Support file-backed PMD folios mm/filemap: Support VM_HUGEPAGE for file mappings mm/readahead: Switch to page_cache_ra_order mm/readahead: Align file mappings for non-DAX mm/readahead: Add large folio readahead mm: Support arbitrary THP sizes mm: Make large folios depend on THP mm: Fix READ_ONLY_THP warning mm/filemap: Allow large folios to be added to the page cache mm: Turn can_split_huge_page() into can_split_folio() mm/vmscan: Convert pageout() to take a folio mm/vmscan: Turn page_check_references() into folio_check_references() mm/vmscan: Account large folios correctly mm/vmscan: Optimise shrink_page_list for non-PMD-sized folios mm/vmscan: Free non-shmem folios without splitting them mm/rmap: Constify the rmap_walk_control argument mm/rmap: Convert rmap_walk() to take a folio mm: Turn page_anon_vma() into folio_anon_vma() mm/rmap: Turn page_lock_anon_vma_read() into folio_lock_anon_vma_read() ...

Linus Torvalds committed 4 years ago

Commit 9030fb02 parents 3bf03b9 2a3c4bc100 files · +2900 −3020Browse files

Showing the changes to include (13 of 100 files). Show all

Unified · Split

13 files

include/linux/fs.h +0 −120
include/linux/hmm.h +2 −7
include/linux/huge_mm.h +9 −50
include/linux/hugetlb.h +5 −0
include/linux/ksm.h +3 −3
include/linux/memremap.h +24 −3
include/linux/mm.h +152 −144
include/linux/mm_inline.h +8 −3
include/linux/mm_types.h +22 −4
include/linux/pagemap.h +133 −13
include/linux/rmap.h +47 −29
include/linux/swap.h +4 −5
include/trace/events/vmscan.h +5 −5

include/linux/fs.h+0−120

@@ -2753,54 +2753,6 @@ extern void init_special_inode(struct inode *, umode_t, dev_t); · 2752 unmodified lines
2753	2753	extern void make_bad_inode(struct inode *);
2754	2754	extern bool is_bad_inode(struct inode *);
2755	2755
2756		−unsigned long invalidate_mapping_pages(struct address_space *mapping,
2757		− pgoff_t start, pgoff_t end);
2758		−
2759		−void invalidate_mapping_pagevec(struct address_space *mapping,
2760		− pgoff_t start, pgoff_t end,
2761		− unsigned long *nr_pagevec);
2762		−
2763		−static inline void invalidate_remote_inode(struct inode *inode)
2764		−{
2765		− if (S_ISREG(inode->i_mode) \|\| S_ISDIR(inode->i_mode) \|\|
2766		− S_ISLNK(inode->i_mode))
2767		− invalidate_mapping_pages(inode->i_mapping, 0, -1);
2768		−}
2769		−extern int invalidate_inode_pages2(struct address_space *mapping);
2770		−extern int invalidate_inode_pages2_range(struct address_space *mapping,
2771		− pgoff_t start, pgoff_t end);
2772		−extern int write_inode_now(struct inode *, int);
2773		−extern int filemap_fdatawrite(struct address_space *);
2774		−extern int filemap_flush(struct address_space *);
2775		−extern int filemap_fdatawait_keep_errors(struct address_space *mapping);
2776		−extern int filemap_fdatawait_range(struct address_space *, loff_t lstart,
2777		− loff_t lend);
2778		−extern int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
2779		− loff_t start_byte, loff_t end_byte);
2780		−
2781		−static inline int filemap_fdatawait(struct address_space *mapping)
2782		−{
2783		− return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
2784		−}
2785		−
2786		−extern bool filemap_range_has_page(struct address_space *, loff_t lstart,
2787		− loff_t lend);
2788		−extern int filemap_write_and_wait_range(struct address_space *mapping,
2789		− loff_t lstart, loff_t lend);
2790		−extern int __filemap_fdatawrite_range(struct address_space *mapping,
2791		− loff_t start, loff_t end, int sync_mode);
2792		−extern int filemap_fdatawrite_range(struct address_space *mapping,
2793		− loff_t start, loff_t end);
2794		−extern int filemap_check_errors(struct address_space *mapping);
2795		−extern void __filemap_set_wb_err(struct address_space *mapping, int err);
2796		−int filemap_fdatawrite_wbc(struct address_space *mapping,
2797		− struct writeback_control *wbc);
2798		−
2799		−static inline int filemap_write_and_wait(struct address_space *mapping)
2800		−{
2801		− return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
2802		−}
2803		−
2804	2756	extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,
2805	2757	loff_t lend);
2806	2758	extern int __must_check file_check_and_advance_wb_err(struct file *file);
@@ -2812,67 +2764,6 @@ static inline int file_write_and_wait(struct file *file) · 5 unmodified lines
2812	2764	return file_write_and_wait_range(file, 0, LLONG_MAX);
2813	2765	}
2814	2766
2815		−/**
2816		− * filemap_set_wb_err - set a writeback error on an address_space
2817		− * @mapping: mapping in which to set writeback error
2818		− * @err: error to be set in mapping
2819		− *
2820		− * When writeback fails in some way, we must record that error so that
2821		− * userspace can be informed when fsync and the like are called. We endeavor
2822		− * to report errors on any file that was open at the time of the error. Some
2823		− * internal callers also need to know when writeback errors have occurred.
2824		− *
2825		− * When a writeback error occurs, most filesystems will want to call
2826		− * filemap_set_wb_err to record the error in the mapping so that it will be
2827		− * automatically reported whenever fsync is called on the file.
2828		− */
2829		−static inline void filemap_set_wb_err(struct address_space *mapping, int err)
2830		−{
2831		− /* Fastpath for common case of no error */
2832		− if (unlikely(err))
2833		− __filemap_set_wb_err(mapping, err);
2834		−}
2835		−
2836		−/**
2837		− * filemap_check_wb_err - has an error occurred since the mark was sampled?
2838		− * @mapping: mapping to check for writeback errors
2839		− * @since: previously-sampled errseq_t
2840		− *
2841		− * Grab the errseq_t value from the mapping, and see if it has changed "since"
2842		− * the given value was sampled.
2843		− *
2844		− * If it has then report the latest error set, otherwise return 0.
2845		− */
2846		−static inline int filemap_check_wb_err(struct address_space *mapping,
2847		− errseq_t since)
2848		−{
2849		− return errseq_check(&mapping->wb_err, since);
2850		−}
2851		−
2852		−/**
2853		− * filemap_sample_wb_err - sample the current errseq_t to test for later errors
2854		− * @mapping: mapping to be sampled
2855		− *
2856		− * Writeback errors are always reported relative to a particular sample point
2857		− * in the past. This function provides those sample points.
2858		− */
2859		−static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
2860		−{
2861		− return errseq_sample(&mapping->wb_err);
2862		−}
2863		−
2864		−/**
2865		− * file_sample_sb_err - sample the current errseq_t to test for later errors
2866		− * @file: file pointer to be sampled
2867		− *
2868		− * Grab the most current superblock-level errseq_t value for the given
2869		− * struct file.
2870		− */
2871		−static inline errseq_t file_sample_sb_err(struct file *file)
2872		−{
2873		− return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
2874		−}
2875		−
2876	2767	extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,
2877	2768	int datasync);
2878	2769	extern int vfs_fsync(struct file *file, int datasync);
@@ -3627,15 +3518,4 @@ extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len, · 748 unmodified lines
3627	3518	extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
3628	3519	int advice);
3629	3520
3630		−/*
3631		− * Flush file data before changing attributes. Caller must hold any locks
3632		− * required to prevent further writes to this file until we're done setting
3633		− * flags.
3634		− */
3635		−static inline int inode_drain_writes(struct inode *inode)
3636		−{
3637		− inode_dio_wait(inode);
3638		− return filemap_write_and_wait(inode->i_mapping);
3639		−}
3640		−
3641	3521	#endif /* _LINUX_FS_H */

include/linux/hmm.h+2−7

@@ -9,14 +9,9 @@ · 8 unmodified lines
9	9	#ifndef LINUX_HMM_H
10	10	#define LINUX_HMM_H
11	11
12		−#include <linux/kconfig.h>
13		−#include <linux/pgtable.h>
	12	+#include <linux/mm.h>
14	13
15		−#include <linux/device.h>
16		−#include <linux/migrate.h>
17		−#include <linux/memremap.h>
18		−#include <linux/completion.h>
19		−#include <linux/mmu_notifier.h>
	14	+struct mmu_interval_notifier;
20	15
21	16	/*
22	17	* On output:

include/linux/huge_mm.h+9−50

@@ -185,7 +185,7 @@ void prep_transhuge_page(struct page *page); · 184 unmodified lines
185	185	void free_transhuge_page(struct page *page);
186	186	bool is_transparent_hugepage(struct page *page);
187	187
188		−bool can_split_huge_page(struct page page, int pextra_pins);
	188	+bool can_split_folio(struct folio folio, int pextra_pins);
189	189	int split_huge_page_to_list(struct page page, struct list_head list);
190	190	static inline int split_huge_page(struct page *page)
191	191	{
@@ -194,7 +194,7 @@ static inline int split_huge_page(struct page *page) · 2 unmodified lines
194	194	void deferred_split_huge_page(struct page *page);
195	195
196	196	void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
197		− unsigned long address, bool freeze, struct page *page);
	197	+ unsigned long address, bool freeze, struct folio *folio);
198	198
199	199	#define split_huge_pmd(__vma, __pmd, __address) \
200	200	do { \
@@ -207,7 +207,7 @@ void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd, · 6 unmodified lines
207	207
208	208
209	209	void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
210		− bool freeze, struct page *page);
	210	+ bool freeze, struct folio *folio);
211	211
212	212	void __split_huge_pud(struct vm_area_struct vma, pud_t pud,
213	213	unsigned long address);
@@ -250,30 +250,6 @@ static inline spinlock_t pud_trans_huge_lock(pud_t pud, · 36 unmodified lines
250	250	return NULL;
251	251	}
252	252
253		−/**
254		− * thp_order - Order of a transparent huge page.
255		− * @page: Head page of a transparent huge page.
256		− */
257		−static inline unsigned int thp_order(struct page *page)
258		−{
259		− VM_BUG_ON_PGFLAGS(PageTail(page), page);
260		− if (PageHead(page))
261		− return HPAGE_PMD_ORDER;
262		− return 0;
263		−}
264		−
265		−/**
266		− * thp_nr_pages - The number of regular pages in this huge page.
267		− * @page: The head page of a huge page.
268		− */
269		−static inline int thp_nr_pages(struct page *page)
270		−{
271		− VM_BUG_ON_PGFLAGS(PageTail(page), page);
272		− if (PageHead(page))
273		− return HPAGE_PMD_NR;
274		− return 1;
275		−}
276		−
277	253	/**
278	254	* folio_test_pmd_mappable - Can we map this folio with a PMD?
279	255	* @folio: The folio to test
@@ -336,18 +312,6 @@ static inline struct list_head page_deferred_list(struct page page) · 56 unmodified lines
336	312	#define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
337	313	#define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
338	314
339		−static inline unsigned int thp_order(struct page *page)
340		−{
341		− VM_BUG_ON_PGFLAGS(PageTail(page), page);
342		− return 0;
343		−}
344		−
345		−static inline int thp_nr_pages(struct page *page)
346		−{
347		− VM_BUG_ON_PGFLAGS(PageTail(page), page);
348		− return 1;
349		−}
350		−
351	315	static inline bool folio_test_pmd_mappable(struct folio *folio)
352	316	{
353	317	return false;
@@ -387,7 +351,7 @@ static inline bool is_transparent_hugepage(struct page *page) · 33 unmodified lines
387	351	#define thp_get_unmapped_area NULL
388	352
389	353	static inline bool
390		−can_split_huge_page(struct page page, int pextra_pins)
	354	+can_split_folio(struct folio folio, int pextra_pins)
391	355	{
392	356	BUILD_BUG();
393	357	return false;
@@ -406,9 +370,9 @@ static inline void deferred_split_huge_page(struct page *page) {} · 12 unmodified lines
406	370	do { } while (0)
407	371
408	372	static inline void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
409		− unsigned long address, bool freeze, struct page *page) {}
	373	+ unsigned long address, bool freeze, struct folio *folio) {}
410	374	static inline void split_huge_pmd_address(struct vm_area_struct *vma,
411		− unsigned long address, bool freeze, struct page *page) {}
	375	+ unsigned long address, bool freeze, struct folio *folio) {}
412	376
413	377	#define split_huge_pud(__vma, __pmd, __address) \
414	378	do { } while (0)
@@ -483,15 +447,10 @@ static inline bool thp_migration_supported(void) · 68 unmodified lines
483	447	}
484	448	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
485	449
486		−/**
487		− * thp_size - Size of a transparent huge page.
488		− * @page: Head page of a transparent huge page.
489		− *
490		− * Return: Number of bytes in this page.
491		− */
492		−static inline unsigned long thp_size(struct page *page)
	450	+static inline int split_folio_to_list(struct folio *folio,
	451	+ struct list_head *list)
493	452	{
494		− return PAGE_SIZE << thp_order(page);
	453	+ return split_huge_page_to_list(&folio->page, list);
495	454	}
496	455
497	456	#endif /* _LINUX_HUGE_MM_H */

include/linux/hugetlb.h+5−0

@@ -970,6 +970,11 @@ static inline struct hstate page_hstate(struct page page) · 969 unmodified lines
970	970	return NULL;
971	971	}
972	972
	973	+static inline struct hstate *size_to_hstate(unsigned long size)
	974	+{
	975	+ return NULL;
	976	+}
	977	+
973	978	static inline unsigned long huge_page_size(struct hstate *h)
974	979	{
975	980	return PAGE_SIZE;

include/linux/ksm.h+3−3

@@ -51,7 +51,7 @@ static inline void ksm_exit(struct mm_struct *mm) · 50 unmodified lines
51	51	struct page ksm_might_need_to_copy(struct page page,
52	52	struct vm_area_struct *vma, unsigned long address);
53	53
54		−void rmap_walk_ksm(struct page page, struct rmap_walk_control rwc);
	54	+void rmap_walk_ksm(struct folio folio, const struct rmap_walk_control rwc);
55	55	void folio_migrate_ksm(struct folio newfolio, struct folio folio);
56	56
57	57	#else /* !CONFIG_KSM */
@@ -78,8 +78,8 @@ static inline struct page ksm_might_need_to_copy(struct page page, · 20 unmodified lines
78	78	return page;
79	79	}
80	80
81		−static inline void rmap_walk_ksm(struct page *page,
82		− struct rmap_walk_control *rwc)
	81	+static inline void rmap_walk_ksm(struct folio *folio,
	82	+ const struct rmap_walk_control *rwc)
83	83	{
84	84	}
85	85

include/linux/memremap.h+24−3

@@ -1,6 +1,8 @@
1	1	/* SPDX-License-Identifier: GPL-2.0 */
2	2	#ifndef _LINUX_MEMREMAP_H_
3	3	#define _LINUX_MEMREMAP_H_
	4	+
	5	+#include <linux/mm.h>
4	6	#include <linux/range.h>
5	7	#include <linux/ioport.h>
6	8	#include <linux/percpu-refcount.h>
@@ -66,9 +68,9 @@ enum memory_type { · 59 unmodified lines
66	68
67	69	struct dev_pagemap_ops {
68	70	/*
69		− * Called once the page refcount reaches 1. (ZONE_DEVICE pages never
70		− * reach 0 refcount unless there is a refcount bug. This allows the
71		− * device driver to implement its own memory management.)
	71	+ * Called once the page refcount reaches 0. The reference count will be
	72	+ * reset to one by the core code after the method is called to prepare
	73	+ * for handing out the page again.
72	74	*/
73	75	void (page_free)(struct page page);
74	76
@@ -129,6 +131,25 @@ static inline unsigned long pgmap_vmemmap_nr(struct dev_pagemap *pgmap) · 54 unmodified lines
129	131	return 1 << pgmap->vmemmap_shift;
130	132	}
131	133
	134	+static inline bool is_device_private_page(const struct page *page)
	135	+{
	136	+ return IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
	137	+ is_zone_device_page(page) &&
	138	+ page->pgmap->type == MEMORY_DEVICE_PRIVATE;
	139	+}
	140	+
	141	+static inline bool folio_is_device_private(const struct folio *folio)
	142	+{
	143	+ return is_device_private_page(&folio->page);
	144	+}
	145	+
	146	+static inline bool is_pci_p2pdma_page(const struct page *page)
	147	+{
	148	+ return IS_ENABLED(CONFIG_PCI_P2PDMA) &&
	149	+ is_zone_device_page(page) &&
	150	+ page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
	151	+}
	152	+
132	153	#ifdef CONFIG_ZONE_DEVICE
133	154	void memremap_pages(struct dev_pagemap pgmap, int nid);
134	155	void memunmap_pages(struct dev_pagemap *pgmap);

include/linux/mm.h+152−144

@@ -3,9 +3,6 @@ · 2 unmodified lines
3	3	#define _LINUX_MM_H
4	4
5	5	#include <linux/errno.h>
6		−
7		−#ifdef __KERNEL__
8		−
9	6	#include <linux/mmdebug.h>
10	7	#include <linux/gfp.h>
11	8	#include <linux/bug.h>
@@ -26,7 +23,6 @@ · 14 unmodified lines
26	23	#include <linux/err.h>
27	24	#include <linux/page-flags.h>
28	25	#include <linux/page_ref.h>
29		−#include <linux/memremap.h>
30	26	#include <linux/overflow.h>
31	27	#include <linux/sizes.h>
32	28	#include <linux/sched.h>
@@ -216,8 +212,10 @@ int overcommit_policy_handler(struct ctl_table , int, void , size_t *, · 183 unmodified lines
216	212
217	213	#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
218	214	#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
	215	+#define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio))
219	216	#else
220	217	#define nth_page(page,n) ((page) + (n))
	218	+#define folio_page_idx(folio, p) ((p) - &(folio)->page)
221	219	#endif
222	220
223	221	/* to align the pointer to the (next) page boundary */
@@ -227,6 +225,10 @@ int overcommit_policy_handler(struct ctl_table , int, void , size_t *, · 3 unmodified lines
227	225	#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
228	226
229	227	#define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
	228	+static inline struct folio lru_to_folio(struct list_head head)
	229	+{
	230	+ return list_entry((head)->prev, struct folio, lru);
	231	+}
230	232
231	233	void setup_initial_init_mm(void start_code, void end_code,
232	234	void end_data, void brk);
@@ -775,21 +777,26 @@ static inline int is_vmalloc_or_module_addr(const void *x) · 542 unmodified lines
775	777	}
776	778	#endif
777	779
778		−static inline int head_compound_mapcount(struct page *head)
	780	+/*
	781	+ * How many times the entire folio is mapped as a single unit (eg by a
	782	+ * PMD or PUD entry). This is probably not what you want, except for
	783	+ * debugging purposes; look at folio_mapcount() or page_mapcount()
	784	+ * instead.
	785	+ */
	786	+static inline int folio_entire_mapcount(struct folio *folio)
779	787	{
780		− return atomic_read(compound_mapcount_ptr(head)) + 1;
	788	+ VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
	789	+ return atomic_read(folio_mapcount_ptr(folio)) + 1;
781	790	}
782	791
783	792	/*
784	793	* Mapcount of compound page as a whole, does not include mapped sub-pages.
785	794	*
786		− * Must be called only for compound pages or any their tail sub-pages.
	795	+ * Must be called only for compound pages.
787	796	*/
788	797	static inline int compound_mapcount(struct page *page)
789	798	{
790		− VM_BUG_ON_PAGE(!PageCompound(page), page);
791		− page = compound_head(page);
792		− return head_compound_mapcount(page);
	799	+ return folio_entire_mapcount(page_folio(page));
793	800	}
794	801
795	802	/*
@@ -819,8 +826,14 @@ static inline int page_mapcount(struct page *page) · 23 unmodified lines
819	826	return atomic_read(&page->_mapcount) + 1;
820	827	}
821	828
	829	+int folio_mapcount(struct folio *folio);
	830	+
822	831	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
823		−int total_mapcount(struct page *page);
	832	+static inline int total_mapcount(struct page *page)
	833	+{
	834	+ return folio_mapcount(page_folio(page));
	835	+}
	836	+
824	837	int page_trans_huge_mapcount(struct page *page);
825	838	#else
826	839	static inline int total_mapcount(struct page *page)
@@ -890,33 +903,17 @@ static inline void destroy_compound_page(struct page *page) · 63 unmodified lines
890	903	compound_page_dtors[page[1].compound_dtor](page);
891	904	}
892	905
893		−static inline bool hpage_pincount_available(struct page *page)
894		−{
895		− /*
896		− * Can the page->hpage_pinned_refcount field be used? That field is in
897		− * the 3rd page of the compound page, so the smallest (2-page) compound
898		− * pages cannot support it.
899		− */
900		− page = compound_head(page);
901		− return PageCompound(page) && compound_order(page) > 1;
902		−}
903		−
904	906	static inline int head_compound_pincount(struct page *head)
905	907	{
906	908	return atomic_read(compound_pincount_ptr(head));
907	909	}
908	910
909		−static inline int compound_pincount(struct page *page)
910		−{
911		− VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
912		− page = compound_head(page);
913		− return head_compound_pincount(page);
914		−}
915		−
916	911	static inline void set_compound_order(struct page *page, unsigned int order)
917	912	{
918	913	page[1].compound_order = order;
	914	+#ifdef CONFIG_64BIT
919	915	page[1].compound_nr = 1U << order;
	916	+#endif
920	917	}
921	918
922	919	/* Returns the number of pages in this potentially compound page. */
@@ -924,7 +921,11 @@ static inline unsigned long compound_nr(struct page *page) · 1 unmodified line
924	921	{
925	922	if (!PageHead(page))
926	923	return 1;
	924	+#ifdef CONFIG_64BIT
927	925	return page[1].compound_nr;
	926	+#else
	927	+ return 1UL << compound_order(page);
	928	+#endif
928	929	}
929	930
930	931	/* Returns the number of bytes in this potentially compound page. */
@@ -939,6 +940,37 @@ static inline unsigned int page_shift(struct page *page) · 8 unmodified lines
939	940	return PAGE_SHIFT + compound_order(page);
940	941	}
941	942
	943	+/**
	944	+ * thp_order - Order of a transparent huge page.
	945	+ * @page: Head page of a transparent huge page.
	946	+ */
	947	+static inline unsigned int thp_order(struct page *page)
	948	+{
	949	+ VM_BUG_ON_PGFLAGS(PageTail(page), page);
	950	+ return compound_order(page);
	951	+}
	952	+
	953	+/**
	954	+ * thp_nr_pages - The number of regular pages in this huge page.
	955	+ * @page: The head page of a huge page.
	956	+ */
	957	+static inline int thp_nr_pages(struct page *page)
	958	+{
	959	+ VM_BUG_ON_PGFLAGS(PageTail(page), page);
	960	+ return compound_nr(page);
	961	+}
	962	+
	963	+/**
	964	+ * thp_size - Size of a transparent huge page.
	965	+ * @page: Head page of a transparent huge page.
	966	+ *
	967	+ * Return: Number of bytes in this page.
	968	+ */
	969	+static inline unsigned long thp_size(struct page *page)
	970	+{
	971	+ return PAGE_SIZE << thp_order(page);
	972	+}
	973	+
942	974	void free_compound_page(struct page *page);
943	975
944	976	#ifdef CONFIG_MMU
@@ -1090,59 +1122,35 @@ static inline bool is_zone_device_page(const struct page *page) · 145 unmodified lines
1090	1122	}
1091	1123	#endif
1092	1124
	1125	+static inline bool folio_is_zone_device(const struct folio *folio)
	1126	+{
	1127	+ return is_zone_device_page(&folio->page);
	1128	+}
	1129	+
1093	1130	static inline bool is_zone_movable_page(const struct page *page)
1094	1131	{
1095	1132	return page_zonenum(page) == ZONE_MOVABLE;
1096	1133	}
1097	1134
1098		−#ifdef CONFIG_DEV_PAGEMAP_OPS
1099		−void free_devmap_managed_page(struct page *page);
	1135	+#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX)
1100	1136	DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
1101	1137
1102		−static inline bool page_is_devmap_managed(struct page *page)
	1138	+bool __put_devmap_managed_page(struct page *page);
	1139	+static inline bool put_devmap_managed_page(struct page *page)
1103	1140	{
1104	1141	if (!static_branch_unlikely(&devmap_managed_key))
1105	1142	return false;
1106	1143	if (!is_zone_device_page(page))
1107	1144	return false;
1108		− switch (page->pgmap->type) {
1109		− case MEMORY_DEVICE_PRIVATE:
1110		− case MEMORY_DEVICE_FS_DAX:
1111		− return true;
1112		− default:
1113		− break;
1114		− }
1115		− return false;
	1145	+ return __put_devmap_managed_page(page);
1116	1146	}
1117	1147
1118		−void put_devmap_managed_page(struct page *page);
1119		−
1120		−#else /* CONFIG_DEV_PAGEMAP_OPS */
1121		−static inline bool page_is_devmap_managed(struct page *page)
	1148	+#else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
	1149	+static inline bool put_devmap_managed_page(struct page *page)
1122	1150	{
1123	1151	return false;
1124	1152	}
1125		−
1126		−static inline void put_devmap_managed_page(struct page *page)
1127		−{
1128		−}
1129		−#endif /* CONFIG_DEV_PAGEMAP_OPS */
1130		−
1131		−static inline bool is_device_private_page(const struct page *page)
1132		−{
1133		− return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
1134		− IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
1135		− is_zone_device_page(page) &&
1136		− page->pgmap->type == MEMORY_DEVICE_PRIVATE;
1137		−}
1138		−
1139		−static inline bool is_pci_p2pdma_page(const struct page *page)
1140		−{
1141		− return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
1142		− IS_ENABLED(CONFIG_PCI_P2PDMA) &&
1143		− is_zone_device_page(page) &&
1144		− page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
1145		−}
	1153	+#endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
1146	1154
1147	1155	/* 127: arbitrary random number, small enough to assemble well */
1148	1156	#define folio_ref_zero_or_close_to_overflow(folio) \
@@ -1168,9 +1176,6 @@ static inline void get_page(struct page *page) · 19 unmodified lines
1168	1176	}
1169	1177
1170	1178	bool __must_check try_grab_page(struct page *page, unsigned int flags);
1171		−struct page try_grab_compound_head(struct page page, int refs,
1172		− unsigned int flags);
1173		−
1174	1179
1175	1180	static inline __must_check bool try_get_page(struct page *page)
1176	1181	{
@@ -1225,16 +1230,11 @@ static inline void put_page(struct page *page) · 48 unmodified lines
1225	1230	struct folio *folio = page_folio(page);
1226	1231
1227	1232	/*
1228		− * For devmap managed pages we need to catch refcount transition from
1229		− * 2 to 1, when refcount reach one it means the page is free and we
1230		− * need to inform the device driver through callback. See
1231		− * include/linux/memremap.h and HMM for details.
	1233	+ * For some devmap managed pages we need to catch refcount transition
	1234	+ * from 2 to 1:
1232	1235	*/
1233		− if (page_is_devmap_managed(&folio->page)) {
1234		− put_devmap_managed_page(&folio->page);
	1236	+ if (put_devmap_managed_page(&folio->page))
1235	1237	return;
1236		− }
1237		−
1238	1238	folio_put(folio);
1239	1239	}
1240	1240
@@ -1264,10 +1264,9 @@ static inline void put_page(struct page *page) · 23 unmodified lines
1264	1264	* applications that don't have huge page reference counts, this won't be an
1265	1265	* issue.
1266	1266	*
1267		− * Locking: the lockless algorithm described in page_cache_get_speculative()
1268		− * and page_cache_gup_pin_speculative() provides safe operation for
1269		− * get_user_pages and page_mkclean and other calls that race to set up page
1270		− * table entries.
	1267	+ * Locking: the lockless algorithm described in folio_try_get_rcu()
	1268	+ * provides safe operation for get_user_pages(), page_mkclean() and
	1269	+ * other calls that race to set up page table entries.
1271	1270	*/
1272	1271	#define GUP_PIN_COUNTING_BIAS (1U << 10)
1273	1272
@@ -1278,70 +1277,11 @@ void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, · 4 unmodified lines
1278	1277	bool make_dirty);
1279	1278	void unpin_user_pages(struct page **pages, unsigned long npages);
1280	1279
1281		−/**
1282		− * page_maybe_dma_pinned - Report if a page is pinned for DMA.
1283		− * @page: The page.
1284		− *
1285		− * This function checks if a page has been pinned via a call to
1286		− * a function in the pin_user_pages() family.
1287		− *
1288		− * For non-huge pages, the return value is partially fuzzy: false is not fuzzy,
1289		− * because it means "definitely not pinned for DMA", but true means "probably
1290		− * pinned for DMA, but possibly a false positive due to having at least
1291		− * GUP_PIN_COUNTING_BIAS worth of normal page references".
1292		− *
1293		− * False positives are OK, because: a) it's unlikely for a page to get that many
1294		− * refcounts, and b) all the callers of this routine are expected to be able to
1295		− * deal gracefully with a false positive.
1296		− *
1297		− * For huge pages, the result will be exactly correct. That's because we have
1298		− * more tracking data available: the 3rd struct page in the compound page is
1299		− * used to track the pincount (instead using of the GUP_PIN_COUNTING_BIAS
1300		− * scheme).
1301		− *
1302		− * For more information, please see Documentation/core-api/pin_user_pages.rst.
1303		− *
1304		− * Return: True, if it is likely that the page has been "dma-pinned".
1305		− * False, if the page is definitely not dma-pinned.
1306		− */
1307		−static inline bool page_maybe_dma_pinned(struct page *page)
1308		−{
1309		− if (hpage_pincount_available(page))
1310		− return compound_pincount(page) > 0;
1311		−
1312		− /*
1313		− * page_ref_count() is signed. If that refcount overflows, then
1314		− * page_ref_count() returns a negative value, and callers will avoid
1315		− * further incrementing the refcount.
1316		− *
1317		− * Here, for that overflow case, use the signed bit to count a little
1318		− * bit higher via unsigned math, and thus still get an accurate result.
1319		− */
1320		− return ((unsigned int)page_ref_count(compound_head(page))) >=
1321		− GUP_PIN_COUNTING_BIAS;
1322		−}
1323		−
1324	1280	static inline bool is_cow_mapping(vm_flags_t flags)
1325	1281	{
1326	1282	return (flags & (VM_SHARED \| VM_MAYWRITE)) == VM_MAYWRITE;
1327	1283	}
1328	1284
1329		−/*
1330		− * This should most likely only be called during fork() to see whether we
1331		− * should break the cow immediately for a page on the src mm.
1332		− */
1333		−static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma,
1334		− struct page *page)
1335		−{
1336		− if (!is_cow_mapping(vma->vm_flags))
1337		− return false;
1338		−
1339		− if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
1340		− return false;
1341		−
1342		− return page_maybe_dma_pinned(page);
1343		−}
1344		−
1345	1285	#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
1346	1286	#define SECTION_IN_PAGE_FLAGS
1347	1287	#endif
@@ -1586,6 +1526,74 @@ static inline unsigned long folio_pfn(struct folio *folio) · 238 unmodified lines
1586	1526	return page_to_pfn(&folio->page);
1587	1527	}
1588	1528
	1529	+static inline atomic_t folio_pincount_ptr(struct folio folio)
	1530	+{
	1531	+ return &folio_page(folio, 1)->compound_pincount;
	1532	+}
	1533	+
	1534	+/**
	1535	+ * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA.
	1536	+ * @folio: The folio.
	1537	+ *
	1538	+ * This function checks if a folio has been pinned via a call to
	1539	+ * a function in the pin_user_pages() family.
	1540	+ *
	1541	+ * For small folios, the return value is partially fuzzy: false is not fuzzy,
	1542	+ * because it means "definitely not pinned for DMA", but true means "probably
	1543	+ * pinned for DMA, but possibly a false positive due to having at least
	1544	+ * GUP_PIN_COUNTING_BIAS worth of normal folio references".
	1545	+ *
	1546	+ * False positives are OK, because: a) it's unlikely for a folio to
	1547	+ * get that many refcounts, and b) all the callers of this routine are
	1548	+ * expected to be able to deal gracefully with a false positive.
	1549	+ *
	1550	+ * For large folios, the result will be exactly correct. That's because
	1551	+ * we have more tracking data available: the compound_pincount is used
	1552	+ * instead of the GUP_PIN_COUNTING_BIAS scheme.
	1553	+ *
	1554	+ * For more information, please see Documentation/core-api/pin_user_pages.rst.
	1555	+ *
	1556	+ * Return: True, if it is likely that the page has been "dma-pinned".
	1557	+ * False, if the page is definitely not dma-pinned.
	1558	+ */
	1559	+static inline bool folio_maybe_dma_pinned(struct folio *folio)
	1560	+{
	1561	+ if (folio_test_large(folio))
	1562	+ return atomic_read(folio_pincount_ptr(folio)) > 0;
	1563	+
	1564	+ /*
	1565	+ * folio_ref_count() is signed. If that refcount overflows, then
	1566	+ * folio_ref_count() returns a negative value, and callers will avoid
	1567	+ * further incrementing the refcount.
	1568	+ *
	1569	+ * Here, for that overflow case, use the sign bit to count a little
	1570	+ * bit higher via unsigned math, and thus still get an accurate result.
	1571	+ */
	1572	+ return ((unsigned int)folio_ref_count(folio)) >=
	1573	+ GUP_PIN_COUNTING_BIAS;
	1574	+}
	1575	+
	1576	+static inline bool page_maybe_dma_pinned(struct page *page)
	1577	+{
	1578	+ return folio_maybe_dma_pinned(page_folio(page));
	1579	+}
	1580	+
	1581	+/*
	1582	+ * This should most likely only be called during fork() to see whether we
	1583	+ * should break the cow immediately for a page on the src mm.
	1584	+ */
	1585	+static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma,
	1586	+ struct page *page)
	1587	+{
	1588	+ if (!is_cow_mapping(vma->vm_flags))
	1589	+ return false;
	1590	+
	1591	+ if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
	1592	+ return false;
	1593	+
	1594	+ return page_maybe_dma_pinned(page);
	1595	+}
	1596	+
1589	1597	/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */
1590	1598	#ifdef CONFIG_MIGRATION
1591	1599	static inline bool is_pinnable_page(struct page *page)
@@ -1600,6 +1608,11 @@ static inline bool is_pinnable_page(struct page *page) · 8 unmodified lines
1600	1608	}
1601	1609	#endif
1602	1610
	1611	+static inline bool folio_is_pinnable(struct folio *folio)
	1612	+{
	1613	+ return is_pinnable_page(&folio->page);
	1614	+}
	1615	+
1603	1616	static inline void set_page_zone(struct page *page, enum zone_type zone)
1604	1617	{
1605	1618	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
@@ -1749,7 +1762,6 @@ static inline void folio_address(const struct folio folio) · 143 unmodified lines
1749	1762	}
1750	1763
1751	1764	extern void page_rmapping(struct page page);
1752		−extern struct anon_vma page_anon_vma(struct page page);
1753	1765	extern pgoff_t __page_file_index(struct page *page);
1754	1766
1755	1767	/*
@@ -1855,7 +1867,6 @@ extern void truncate_setsize(struct inode *inode, loff_t newsize); · 99 unmodified lines
1855	1867	void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
1856	1868	void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1857	1869	int generic_error_remove_page(struct address_space mapping, struct page page);
1858		−int invalidate_inode_page(struct page *page);
1859	1870
1860	1871	#ifdef CONFIG_MMU
1861	1872	extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
@@ -2921,13 +2932,11 @@ struct page follow_page(struct vm_area_struct vma, unsigned long address, · 1059 unmodified lines
2921	2932	#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
2922	2933	#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
2923	2934	* and return without waiting upon it */
2924		−#define FOLL_POPULATE 0x40 /* fault in pages (with FOLL_MLOCK) */
2925	2935	#define FOLL_NOFAULT 0x80 /* do not fault in pages */
2926	2936	#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2927	2937	#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2928	2938	#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2929	2939	#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
2930		−#define FOLL_MLOCK 0x1000 /* lock present pages */
2931	2940	#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
2932	2941	#define FOLL_COW 0x4000 /* internal GUP flag */
2933	2942	#define FOLL_ANON 0x8000 /* don't do file mappings */
@@ -3381,5 +3390,4 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start, · 447 unmodified lines
3381	3390	}
3382	3391	#endif
3383	3392
3384		−#endif /* __KERNEL__ */
3385	3393	#endif /* _LINUX_MM_H */

include/linux/mm_inline.h+8−3

@@ -99,7 +99,8 @@ void lruvec_add_folio(struct lruvec lruvec, struct folio folio) · 98 unmodified lines
99	99
100	100	update_lru_size(lruvec, lru, folio_zonenum(folio),
101	101	folio_nr_pages(folio));
102		− list_add(&folio->lru, &lruvec->lists[lru]);
	102	+ if (lru != LRU_UNEVICTABLE)
	103	+ list_add(&folio->lru, &lruvec->lists[lru]);
103	104	}
104	105
105	106	static __always_inline void add_page_to_lru_list(struct page *page,
@@ -115,6 +116,7 @@ void lruvec_add_folio_tail(struct lruvec lruvec, struct folio folio) · 9 unmodified lines
115	116
116	117	update_lru_size(lruvec, lru, folio_zonenum(folio),
117	118	folio_nr_pages(folio));
	119	+ /* This is not expected to be used on LRU_UNEVICTABLE */
118	120	list_add_tail(&folio->lru, &lruvec->lists[lru]);
119	121	}
120	122
@@ -127,8 +129,11 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page, · 6 unmodified lines
127	129	static __always_inline
128	130	void lruvec_del_folio(struct lruvec lruvec, struct folio folio)
129	131	{
130		− list_del(&folio->lru);
131		− update_lru_size(lruvec, folio_lru_list(folio), folio_zonenum(folio),
	132	+ enum lru_list lru = folio_lru_list(folio);
	133	+
	134	+ if (lru != LRU_UNEVICTABLE)
	135	+ list_del(&folio->lru);
	136	+ update_lru_size(lruvec, lru, folio_zonenum(folio),
132	137	-folio_nr_pages(folio));
133	138	}
134	139

include/linux/mm_types.h+22−4

@@ -85,7 +85,16 @@ struct page { · 84 unmodified lines
85	85	* lruvec->lru_lock. Sometimes used as a generic list
86	86	* by the page owner.
87	87	*/
88		− struct list_head lru;
	88	+ union {
	89	+ struct list_head lru;
	90	+ /* Or, for the Unevictable "LRU list" slot */
	91	+ struct {
	92	+ /* Always even, to negate PageTail */
	93	+ void *__filler;
	94	+ /* Count page's or folio's mlocks */
	95	+ unsigned int mlock_count;
	96	+ };
	97	+ };
89	98	/* See page-flags.h for PAGE_MAPPING_FLAGS */
90	99	struct address_space *mapping;
91	100	pgoff_t index; /* Our offset within mapping. */
@@ -126,11 +135,14 @@ struct page { · 34 unmodified lines
126	135	unsigned char compound_dtor;
127	136	unsigned char compound_order;
128	137	atomic_t compound_mapcount;
	138	+ atomic_t compound_pincount;
	139	+#ifdef CONFIG_64BIT
129	140	unsigned int compound_nr; /* 1 << compound_order */
	141	+#endif
130	142	};
131	143	struct { /* Second tail page of compound page */
132	144	unsigned long _compound_pad_1; /* compound_head */
133		− atomic_t hpage_pinned_refcount;
	145	+ unsigned long _compound_pad_2;
134	146	/* For both global and memcg */
135	147	struct list_head deferred_list;
136	148	};
@@ -241,7 +253,13 @@ struct folio { · 104 unmodified lines
241	253	struct {
242	254	/* public: */
243	255	unsigned long flags;
244		− struct list_head lru;
	256	+ union {
	257	+ struct list_head lru;
	258	+ struct {
	259	+ void *__filler;
	260	+ unsigned int mlock_count;
	261	+ };
	262	+ };
245	263	struct address_space *mapping;
246	264	pgoff_t index;
247	265	void *private;
@@ -285,7 +303,7 @@ static inline atomic_t compound_mapcount_ptr(struct page page) · 37 unmodified lines
285	303
286	304	static inline atomic_t compound_pincount_ptr(struct page page)
287	305	{
288		− return &page[2].hpage_pinned_refcount;
	306	+ return &page[1].compound_pincount;
289	307	}
290	308
291	309	/*

include/linux/pagemap.h+133−13

@@ -18,6 +18,120 @@ · 17 unmodified lines
18	18
19	19	struct folio_batch;
20	20
	21	+unsigned long invalidate_mapping_pages(struct address_space *mapping,
	22	+ pgoff_t start, pgoff_t end);
	23	+
	24	+static inline void invalidate_remote_inode(struct inode *inode)
	25	+{
	26	+ if (S_ISREG(inode->i_mode) \|\| S_ISDIR(inode->i_mode) \|\|
	27	+ S_ISLNK(inode->i_mode))
	28	+ invalidate_mapping_pages(inode->i_mapping, 0, -1);
	29	+}
	30	+int invalidate_inode_pages2(struct address_space *mapping);
	31	+int invalidate_inode_pages2_range(struct address_space *mapping,
	32	+ pgoff_t start, pgoff_t end);
	33	+int write_inode_now(struct inode *, int sync);
	34	+int filemap_fdatawrite(struct address_space *);
	35	+int filemap_flush(struct address_space *);
	36	+int filemap_fdatawait_keep_errors(struct address_space *mapping);
	37	+int filemap_fdatawait_range(struct address_space *, loff_t lstart, loff_t lend);
	38	+int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
	39	+ loff_t start_byte, loff_t end_byte);
	40	+
	41	+static inline int filemap_fdatawait(struct address_space *mapping)
	42	+{
	43	+ return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
	44	+}
	45	+
	46	+bool filemap_range_has_page(struct address_space *, loff_t lstart, loff_t lend);
	47	+int filemap_write_and_wait_range(struct address_space *mapping,
	48	+ loff_t lstart, loff_t lend);
	49	+int __filemap_fdatawrite_range(struct address_space *mapping,
	50	+ loff_t start, loff_t end, int sync_mode);
	51	+int filemap_fdatawrite_range(struct address_space *mapping,
	52	+ loff_t start, loff_t end);
	53	+int filemap_check_errors(struct address_space *mapping);
	54	+void __filemap_set_wb_err(struct address_space *mapping, int err);
	55	+int filemap_fdatawrite_wbc(struct address_space *mapping,
	56	+ struct writeback_control *wbc);
	57	+
	58	+static inline int filemap_write_and_wait(struct address_space *mapping)
	59	+{
	60	+ return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
	61	+}
	62	+
	63	+/**
	64	+ * filemap_set_wb_err - set a writeback error on an address_space
	65	+ * @mapping: mapping in which to set writeback error
	66	+ * @err: error to be set in mapping
	67	+ *
	68	+ * When writeback fails in some way, we must record that error so that
	69	+ * userspace can be informed when fsync and the like are called. We endeavor
	70	+ * to report errors on any file that was open at the time of the error. Some
	71	+ * internal callers also need to know when writeback errors have occurred.
	72	+ *
	73	+ * When a writeback error occurs, most filesystems will want to call
	74	+ * filemap_set_wb_err to record the error in the mapping so that it will be
	75	+ * automatically reported whenever fsync is called on the file.
	76	+ */
	77	+static inline void filemap_set_wb_err(struct address_space *mapping, int err)
	78	+{
	79	+ /* Fastpath for common case of no error */
	80	+ if (unlikely(err))
	81	+ __filemap_set_wb_err(mapping, err);
	82	+}
	83	+
	84	+/**
	85	+ * filemap_check_wb_err - has an error occurred since the mark was sampled?
	86	+ * @mapping: mapping to check for writeback errors
	87	+ * @since: previously-sampled errseq_t
	88	+ *
	89	+ * Grab the errseq_t value from the mapping, and see if it has changed "since"
	90	+ * the given value was sampled.
	91	+ *
	92	+ * If it has then report the latest error set, otherwise return 0.
	93	+ */
	94	+static inline int filemap_check_wb_err(struct address_space *mapping,
	95	+ errseq_t since)
	96	+{
	97	+ return errseq_check(&mapping->wb_err, since);
	98	+}
	99	+
	100	+/**
	101	+ * filemap_sample_wb_err - sample the current errseq_t to test for later errors
	102	+ * @mapping: mapping to be sampled
	103	+ *
	104	+ * Writeback errors are always reported relative to a particular sample point
	105	+ * in the past. This function provides those sample points.
	106	+ */
	107	+static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
	108	+{
	109	+ return errseq_sample(&mapping->wb_err);
	110	+}
	111	+
	112	+/**
	113	+ * file_sample_sb_err - sample the current errseq_t to test for later errors
	114	+ * @file: file pointer to be sampled
	115	+ *
	116	+ * Grab the most current superblock-level errseq_t value for the given
	117	+ * struct file.
	118	+ */
	119	+static inline errseq_t file_sample_sb_err(struct file *file)
	120	+{
	121	+ return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
	122	+}
	123	+
	124	+/*
	125	+ * Flush file data before changing attributes. Caller must hold any locks
	126	+ * required to prevent further writes to this file until we're done setting
	127	+ * flags.
	128	+ */
	129	+static inline int inode_drain_writes(struct inode *inode)
	130	+{
	131	+ inode_dio_wait(inode);
	132	+ return filemap_write_and_wait(inode->i_mapping);
	133	+}
	134	+
21	135	static inline bool mapping_empty(struct address_space *mapping)
22	136	{
23	137	return xa_empty(&mapping->i_pages);
@@ -192,9 +306,14 @@ static inline void mapping_set_large_folios(struct address_space *mapping) · 168 unmodified lines
192	306	__set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
193	307	}
194	308
	309	+/*
	310	+ * Large folio support currently depends on THP. These dependencies are
	311	+ * being worked on but are not yet fixed.
	312	+ */
195	313	static inline bool mapping_large_folio_support(struct address_space *mapping)
196	314	{
197		− return test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
	315	+ return IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
	316	+ test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
198	317	}
199	318
200	319	static inline int filemap_nr_thps(struct address_space *mapping)
@@ -212,7 +331,7 @@ static inline void filemap_nr_thps_inc(struct address_space *mapping) · 11 unmodified lines
212	331	if (!mapping_large_folio_support(mapping))
213	332	atomic_inc(&mapping->nr_thps);
214	333	#else
215		− WARN_ON_ONCE(1);
	334	+ WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
216	335	#endif
217	336	}
218	337
@@ -222,7 +341,7 @@ static inline void filemap_nr_thps_dec(struct address_space *mapping) · 3 unmodified lines
222	341	if (!mapping_large_folio_support(mapping))
223	342	atomic_dec(&mapping->nr_thps);
224	343	#else
225		− WARN_ON_ONCE(1);
	344	+ WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
226	345	#endif
227	346	}
228	347
@@ -283,16 +402,6 @@ static inline struct inode folio_inode(struct folio folio) · 54 unmodified lines
283	402	return folio->mapping->host;
284	403	}
285	404
286		−static inline bool page_cache_add_speculative(struct page *page, int count)
287		−{
288		− return folio_ref_try_add_rcu((struct folio *)page, count);
289		−}
290		−
291		−static inline bool page_cache_get_speculative(struct page *page)
292		−{
293		− return page_cache_add_speculative(page, 1);
294		−}
295		−
296	405	/**
297	406	* folio_attach_private - Attach private data to a folio.
298	407	* @folio: Folio to attach data to.
@@ -706,6 +815,17 @@ static inline loff_t folio_file_pos(struct folio *folio) · 407 unmodified lines
706	815	return page_file_offset(&folio->page);
707	816	}
708	817
	818	+/*
	819	+ * Get the offset in PAGE_SIZE (even for hugetlb folios).
	820	+ * (TODO: hugetlb folios should have ->index in PAGE_SIZE)
	821	+ */
	822	+static inline pgoff_t folio_pgoff(struct folio *folio)
	823	+{
	824	+ if (unlikely(folio_test_hugetlb(folio)))
	825	+ return hugetlb_basepage_index(&folio->page);
	826	+ return folio->index;
	827	+}
	828	+
709	829	extern pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
710	830	unsigned long address);
711	831

include/linux/rmap.h+47−29

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include <linux/rwsem.h>
12	12	#include <linux/memcontrol.h>
13	13	#include <linux/highmem.h>
	14	+#include <linux/pagemap.h>
14	15
15	16	/*
16	17	* The anon_vma heads a list of private "related" vmas, to scan if
@@ -167,18 +168,19 @@ struct anon_vma page_get_anon_vma(struct page page); · 150 unmodified lines
167	168	*/
168	169	void page_move_anon_rmap(struct page , struct vm_area_struct );
169	170	void page_add_anon_rmap(struct page , struct vm_area_struct ,
170		− unsigned long, bool);
	171	+ unsigned long address, bool compound);
171	172	void do_page_add_anon_rmap(struct page , struct vm_area_struct ,
172		− unsigned long, int);
	173	+ unsigned long address, int flags);
173	174	void page_add_new_anon_rmap(struct page , struct vm_area_struct ,
174		− unsigned long, bool);
175		−void page_add_file_rmap(struct page *, bool);
176		−void page_remove_rmap(struct page *, bool);
177		−
	175	+ unsigned long address, bool compound);
	176	+void page_add_file_rmap(struct page , struct vm_area_struct ,
	177	+ bool compound);
	178	+void page_remove_rmap(struct page , struct vm_area_struct ,
	179	+ bool compound);
178	180	void hugepage_add_anon_rmap(struct page , struct vm_area_struct ,
179		− unsigned long);
	181	+ unsigned long address);
180	182	void hugepage_add_new_anon_rmap(struct page , struct vm_area_struct ,
181		− unsigned long);
	183	+ unsigned long address);
182	184
183	185	static inline void page_dup_rmap(struct page *page, bool compound)
184	186	{
@@ -188,11 +190,11 @@ static inline void page_dup_rmap(struct page *page, bool compound) · 3 unmodified lines
188	190	/*
189	191	* Called from mm/vmscan.c to handle paging out
190	192	*/
191		−int page_referenced(struct page *, int is_locked,
	193	+int folio_referenced(struct folio *, int is_locked,
192	194	struct mem_cgroup memcg, unsigned long vm_flags);
193	195
194		−void try_to_migrate(struct page *page, enum ttu_flags flags);
195		−void try_to_unmap(struct page *, enum ttu_flags flags);
	196	+void try_to_migrate(struct folio *folio, enum ttu_flags flags);
	197	+void try_to_unmap(struct folio *, enum ttu_flags flags);
196	198
197	199	int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
198	200	unsigned long end, struct page **pages,
@@ -200,11 +202,13 @@ int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, · 1 unmodified line
200	202
201	203	/* Avoid racy checks */
202	204	#define PVMW_SYNC (1 << 0)
203		−/* Look for migarion entries rather than present PTEs */
	205	+/* Look for migration entries rather than present PTEs */
204	206	#define PVMW_MIGRATION (1 << 1)
205	207
206	208	struct page_vma_mapped_walk {
207		− struct page *page;
	209	+ unsigned long pfn;
	210	+ unsigned long nr_pages;
	211	+ pgoff_t pgoff;
208	212	struct vm_area_struct *vma;
209	213	unsigned long address;
210	214	pmd_t *pmd;
@@ -213,10 +217,30 @@ struct page_vma_mapped_walk { · 2 unmodified lines
213	217	unsigned int flags;
214	218	};
215	219
	220	+#define DEFINE_PAGE_VMA_WALK(name, _page, _vma, _address, _flags) \
	221	+ struct page_vma_mapped_walk name = { \
	222	+ .pfn = page_to_pfn(_page), \
	223	+ .nr_pages = compound_nr(page), \
	224	+ .pgoff = page_to_pgoff(page), \
	225	+ .vma = _vma, \
	226	+ .address = _address, \
	227	+ .flags = _flags, \
	228	+ }
	229	+
	230	+#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \
	231	+ struct page_vma_mapped_walk name = { \
	232	+ .pfn = folio_pfn(_folio), \
	233	+ .nr_pages = folio_nr_pages(_folio), \
	234	+ .pgoff = folio_pgoff(_folio), \
	235	+ .vma = _vma, \
	236	+ .address = _address, \
	237	+ .flags = _flags, \
	238	+ }
	239	+
216	240	static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
217	241	{
218	242	/* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
219		− if (pvmw->pte && !PageHuge(pvmw->page))
	243	+ if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma))
220	244	pte_unmap(pvmw->pte);
221	245	if (pvmw->ptl)
222	246	spin_unlock(pvmw->ptl);
@@ -237,18 +261,12 @@ unsigned long page_address_in_vma(struct page , struct vm_area_struct ); · 14 unmodified lines
237	261	*/
238	262	int folio_mkclean(struct folio *);
239	263
240		−/*
241		− * called in munlock()/munmap() path to check for other vmas holding
242		− * the page mlocked.
243		− */
244		−void page_mlock(struct page *page);
245		−
246		−void remove_migration_ptes(struct page old, struct page new, bool locked);
	264	+void remove_migration_ptes(struct folio src, struct folio dst, bool locked);
247	265
248	266	/*
249	267	* Called by memory-failure.c to kill processes.
250	268	*/
251		−struct anon_vma page_lock_anon_vma_read(struct page page);
	269	+struct anon_vma folio_lock_anon_vma_read(struct folio folio);
252	270	void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
253	271	int page_mapped_in_vma(struct page page, struct vm_area_struct vma);
254	272
@@ -267,15 +285,15 @@ struct rmap_walk_control { · 12 unmodified lines
267	285	* Return false if page table scanning in rmap_walk should be stopped.
268	286	* Otherwise, return true.
269	287	*/
270		− bool (rmap_one)(struct page page, struct vm_area_struct *vma,
	288	+ bool (rmap_one)(struct folio folio, struct vm_area_struct *vma,
271	289	unsigned long addr, void *arg);
272		− int (done)(struct page page);
273		− struct anon_vma (anon_lock)(struct page *page);
	290	+ int (done)(struct folio folio);
	291	+ struct anon_vma (anon_lock)(struct folio *folio);
274	292	bool (invalid_vma)(struct vm_area_struct vma, void *arg);
275	293	};
276	294
277		−void rmap_walk(struct page page, struct rmap_walk_control rwc);
278		−void rmap_walk_locked(struct page page, struct rmap_walk_control rwc);
	295	+void rmap_walk(struct folio folio, const struct rmap_walk_control rwc);
	296	+void rmap_walk_locked(struct folio folio, const struct rmap_walk_control rwc);
279	297
280	298	#else /* !CONFIG_MMU */
281	299
@@ -283,7 +301,7 @@ void rmap_walk_locked(struct page page, struct rmap_walk_control rwc); · 1 unmodified line
283	301	#define anon_vma_prepare(vma) (0)
284	302	#define anon_vma_link(vma) do {} while (0)
285	303
286		−static inline int page_referenced(struct page *page, int is_locked,
	304	+static inline int folio_referenced(struct folio *folio, int is_locked,
287	305	struct mem_cgroup *memcg,
288	306	unsigned long *vm_flags)
289	307	{
@@ -291,7 +309,7 @@ static inline int page_referenced(struct page *page, int is_locked, · 1 unmodified line
291	309	return 0;
292	310	}
293	311
294		−static inline void try_to_unmap(struct page *page, enum ttu_flags flags)
	312	+static inline void try_to_unmap(struct folio *folio, enum ttu_flags flags)
295	313	{
296	314	}
297	315

include/linux/swap.h+4−5

@@ -328,7 +328,7 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio) · 327 unmodified lines
328	328
329	329	/* linux/mm/workingset.c */
330	330	void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
331		−void workingset_eviction(struct page page, struct mem_cgroup *target_memcg);
	331	+void workingset_eviction(struct folio folio, struct mem_cgroup *target_memcg);
332	332	void workingset_refault(struct folio folio, void shadow);
333	333	void workingset_activation(struct folio *folio);
334	334
@@ -375,7 +375,6 @@ extern void lru_add_drain(void); · 40 unmodified lines
375	375	extern void lru_add_drain_cpu(int cpu);
376	376	extern void lru_add_drain_cpu_zone(struct zone *zone);
377	377	extern void lru_add_drain_all(void);
378		−extern void deactivate_file_page(struct page *page);
379	378	extern void deactivate_page(struct page *page);
380	379	extern void mark_page_lazyfree(struct page *page);
381	380	extern void swap_setup(void);
@@ -397,7 +396,7 @@ extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, · 15 unmodified lines
397	396	unsigned long *nr_scanned);
398	397	extern unsigned long shrink_all_memory(unsigned long nr_pages);
399	398	extern int vm_swappiness;
400		−extern int remove_mapping(struct address_space mapping, struct page page);
	399	+long remove_mapping(struct address_space mapping, struct folio folio);
401	400
402	401	extern unsigned long reclaim_pages(struct list_head *page_list);
403	402	#ifdef CONFIG_NUMA
@@ -743,7 +742,7 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · 339 unmodified lines
743	742	#endif
744	743
745	744	#ifdef CONFIG_MEMCG_SWAP
746		−extern void mem_cgroup_swapout(struct page *page, swp_entry_t entry);
	745	+void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry);
747	746	extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
748	747	static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
749	748	{
@@ -763,7 +762,7 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_p · 13 unmodified lines
763	762	extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
764	763	extern bool mem_cgroup_swap_full(struct page *page);
765	764	#else
766		−static inline void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
	765	+static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
767	766	{
768	767	}
769	768

include/trace/events/vmscan.h+5−5

@@ -327,11 +327,11 @@ TRACE_EVENT(mm_vmscan_lru_isolate, · 326 unmodified lines
327	327	__print_symbolic(__entry->lru, LRU_NAMES))
328	328	);
329	329
330		−TRACE_EVENT(mm_vmscan_writepage,
	330	+TRACE_EVENT(mm_vmscan_write_folio,
331	331
332		− TP_PROTO(struct page *page),
	332	+ TP_PROTO(struct folio *folio),
333	333
334		− TP_ARGS(page),
	334	+ TP_ARGS(folio),
335	335
336	336	TP_STRUCT__entry(
337	337	__field(unsigned long, pfn)
@@ -339,9 +339,9 @@ TRACE_EVENT(mm_vmscan_writepage, · 1 unmodified line
339	339	),
340	340
341	341	TP_fast_assign(
342		− __entry->pfn = page_to_pfn(page);
	342	+ __entry->pfn = folio_pfn(folio);
343	343	__entry->reclaim_flags = trace_reclaim_flags(
344		− page_is_file_lru(page));
	344	+ folio_is_file_lru(folio));
345	345	),
346	346
347	347	TP_printk("page=%p pfn=0x%lx flags=%s",