History

Merge tag 'folio-5.18c' of git://git.infradead.org/users/willy/pagecache

Pull folio updates from Matthew Wilcox: - Rewrite how munlock works to massively reduce the contention on i_mmap_rwsem (Hugh Dickins): https://lore.kernel.org/linux-mm/8e4356d-9622-a7f0-b2c-f116b5f2efea@google.com/ - Sort out the page refcount mess for ZONE_DEVICE pages (Christoph Hellwig): https://lore.kernel.org/linux-mm/20220210072828.2930359-1-hch@lst.de/ - Convert GUP to use folios and make pincount available for order-1 pages. (Matthew Wilcox) - Convert a few more truncation functions to use folios (Matthew Wilcox) - Convert page_vma_mapped_walk to use PFNs instead of pages (Matthew Wilcox) - Convert rmap_walk to use folios (Matthew Wilcox) - Convert most of shrink_page_list() to use a folio (Matthew Wilcox) - Add support for creating large folios in readahead (Matthew Wilcox) * tag 'folio-5.18c' of git://git.infradead.org/users/willy/pagecache: (114 commits) mm/damon: minor cleanup for damon_pa_young selftests/vm/transhuge-stress: Support file-backed PMD folios mm/filemap: Support VM_HUGEPAGE for file mappings mm/readahead: Switch to page_cache_ra_order mm/readahead: Align file mappings for non-DAX mm/readahead: Add large folio readahead mm: Support arbitrary THP sizes mm: Make large folios depend on THP mm: Fix READ_ONLY_THP warning mm/filemap: Allow large folios to be added to the page cache mm: Turn can_split_huge_page() into can_split_folio() mm/vmscan: Convert pageout() to take a folio mm/vmscan: Turn page_check_references() into folio_check_references() mm/vmscan: Account large folios correctly mm/vmscan: Optimise shrink_page_list for non-PMD-sized folios mm/vmscan: Free non-shmem folios without splitting them mm/rmap: Constify the rmap_walk_control argument mm/rmap: Convert rmap_walk() to take a folio mm: Turn page_anon_vma() into folio_anon_vma() mm/rmap: Turn page_lock_anon_vma_read() into folio_lock_anon_vma_read() ...

Linus Torvalds committed 4 years ago

Commit 9030fb02 parents 3bf03b9 2a3c4bc100 files · +2900 −3020Browse files

Showing the changes to include (13 of 100 files). Show all

Unified · Split

13 files
  1. include/linux/fs.h +0 −120
  2. include/linux/hmm.h +2 −7
  3. include/linux/huge_mm.h +9 −50
  4. include/linux/hugetlb.h +5 −0
  5. include/linux/ksm.h +3 −3
  6. include/linux/memremap.h +24 −3
  7. include/linux/mm.h +152 −144
  8. include/linux/mm_inline.h +8 −3
  9. include/linux/mm_types.h +22 −4
  10. include/linux/pagemap.h +133 −13
  11. include/linux/rmap.h +47 −29
  12. include/linux/swap.h +4 −5
  13. include/trace/events/vmscan.h +5 −5
include/linux/fs.h+0−120
@@ -2753,54 +2753,6 @@ extern void init_special_inode(struct inode *, umode_t, dev_t); · 2752 unmodified lines
2753extern void make_bad_inode(struct inode *);2753extern void make_bad_inode(struct inode *);
2754extern bool is_bad_inode(struct inode *);2754extern bool is_bad_inode(struct inode *);
2755 2755
2756unsigned long invalidate_mapping_pages(struct address_space *mapping,
2757 pgoff_t start, pgoff_t end);
2758
2759void invalidate_mapping_pagevec(struct address_space *mapping,
2760 pgoff_t start, pgoff_t end,
2761 unsigned long *nr_pagevec);
2762
2763static inline void invalidate_remote_inode(struct inode *inode)
2764{
2765 if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
2766 S_ISLNK(inode->i_mode))
2767 invalidate_mapping_pages(inode->i_mapping, 0, -1);
2768}
2769extern int invalidate_inode_pages2(struct address_space *mapping);
2770extern int invalidate_inode_pages2_range(struct address_space *mapping,
2771 pgoff_t start, pgoff_t end);
2772extern int write_inode_now(struct inode *, int);
2773extern int filemap_fdatawrite(struct address_space *);
2774extern int filemap_flush(struct address_space *);
2775extern int filemap_fdatawait_keep_errors(struct address_space *mapping);
2776extern int filemap_fdatawait_range(struct address_space *, loff_t lstart,
2777 loff_t lend);
2778extern int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
2779 loff_t start_byte, loff_t end_byte);
2780
2781static inline int filemap_fdatawait(struct address_space *mapping)
2782{
2783 return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
2784}
2785
2786extern bool filemap_range_has_page(struct address_space *, loff_t lstart,
2787 loff_t lend);
2788extern int filemap_write_and_wait_range(struct address_space *mapping,
2789 loff_t lstart, loff_t lend);
2790extern int __filemap_fdatawrite_range(struct address_space *mapping,
2791 loff_t start, loff_t end, int sync_mode);
2792extern int filemap_fdatawrite_range(struct address_space *mapping,
2793 loff_t start, loff_t end);
2794extern int filemap_check_errors(struct address_space *mapping);
2795extern void __filemap_set_wb_err(struct address_space *mapping, int err);
2796int filemap_fdatawrite_wbc(struct address_space *mapping,
2797 struct writeback_control *wbc);
2798
2799static inline int filemap_write_and_wait(struct address_space *mapping)
2800{
2801 return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
2802}
2803
2804extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,2756extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,
2805 loff_t lend);2757 loff_t lend);
2806extern int __must_check file_check_and_advance_wb_err(struct file *file);2758extern int __must_check file_check_and_advance_wb_err(struct file *file);
@@ -2812,67 +2764,6 @@ static inline int file_write_and_wait(struct file *file) · 5 unmodified lines
2812 return file_write_and_wait_range(file, 0, LLONG_MAX);2764 return file_write_and_wait_range(file, 0, LLONG_MAX);
2813}2765}
2814 2766
2815/**
2816 * filemap_set_wb_err - set a writeback error on an address_space
2817 * @mapping: mapping in which to set writeback error
2818 * @err: error to be set in mapping
2819 *
2820 * When writeback fails in some way, we must record that error so that
2821 * userspace can be informed when fsync and the like are called. We endeavor
2822 * to report errors on any file that was open at the time of the error. Some
2823 * internal callers also need to know when writeback errors have occurred.
2824 *
2825 * When a writeback error occurs, most filesystems will want to call
2826 * filemap_set_wb_err to record the error in the mapping so that it will be
2827 * automatically reported whenever fsync is called on the file.
2828 */
2829static inline void filemap_set_wb_err(struct address_space *mapping, int err)
2830{
2831 /* Fastpath for common case of no error */
2832 if (unlikely(err))
2833 __filemap_set_wb_err(mapping, err);
2834}
2835
2836/**
2837 * filemap_check_wb_err - has an error occurred since the mark was sampled?
2838 * @mapping: mapping to check for writeback errors
2839 * @since: previously-sampled errseq_t
2840 *
2841 * Grab the errseq_t value from the mapping, and see if it has changed "since"
2842 * the given value was sampled.
2843 *
2844 * If it has then report the latest error set, otherwise return 0.
2845 */
2846static inline int filemap_check_wb_err(struct address_space *mapping,
2847 errseq_t since)
2848{
2849 return errseq_check(&mapping->wb_err, since);
2850}
2851
2852/**
2853 * filemap_sample_wb_err - sample the current errseq_t to test for later errors
2854 * @mapping: mapping to be sampled
2855 *
2856 * Writeback errors are always reported relative to a particular sample point
2857 * in the past. This function provides those sample points.
2858 */
2859static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
2860{
2861 return errseq_sample(&mapping->wb_err);
2862}
2863
2864/**
2865 * file_sample_sb_err - sample the current errseq_t to test for later errors
2866 * @file: file pointer to be sampled
2867 *
2868 * Grab the most current superblock-level errseq_t value for the given
2869 * struct file.
2870 */
2871static inline errseq_t file_sample_sb_err(struct file *file)
2872{
2873 return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
2874}
2875
2876extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,2767extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,
2877 int datasync);2768 int datasync);
2878extern int vfs_fsync(struct file *file, int datasync);2769extern int vfs_fsync(struct file *file, int datasync);
@@ -3627,15 +3518,4 @@ extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len, · 748 unmodified lines
3627extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,3518extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
3628 int advice);3519 int advice);
3629 3520
3630/*
3631 * Flush file data before changing attributes. Caller must hold any locks
3632 * required to prevent further writes to this file until we're done setting
3633 * flags.
3634 */
3635static inline int inode_drain_writes(struct inode *inode)
3636{
3637 inode_dio_wait(inode);
3638 return filemap_write_and_wait(inode->i_mapping);
3639}
3640
3641#endif /* _LINUX_FS_H */3521#endif /* _LINUX_FS_H */
include/linux/hmm.h+2−7
@@ -9,14 +9,9 @@ · 8 unmodified lines
9#ifndef LINUX_HMM_H9#ifndef LINUX_HMM_H
10#define LINUX_HMM_H10#define LINUX_HMM_H
11 11
12#include <linux/kconfig.h>12#include <linux/mm.h>
13#include <linux/pgtable.h>
14 13
15#include <linux/device.h>14struct mmu_interval_notifier;
16#include <linux/migrate.h>
17#include <linux/memremap.h>
18#include <linux/completion.h>
19#include <linux/mmu_notifier.h>
20 15
21/*16/*
22 * On output:17 * On output:
include/linux/huge_mm.h+9−50
@@ -185,7 +185,7 @@ void prep_transhuge_page(struct page *page); · 184 unmodified lines
185void free_transhuge_page(struct page *page);185void free_transhuge_page(struct page *page);
186bool is_transparent_hugepage(struct page *page);186bool is_transparent_hugepage(struct page *page);
187 187
188bool can_split_huge_page(struct page *page, int *pextra_pins);188bool can_split_folio(struct folio *folio, int *pextra_pins);
189int split_huge_page_to_list(struct page *page, struct list_head *list);189int split_huge_page_to_list(struct page *page, struct list_head *list);
190static inline int split_huge_page(struct page *page)190static inline int split_huge_page(struct page *page)
191{191{
@@ -194,7 +194,7 @@ static inline int split_huge_page(struct page *page) · 2 unmodified lines
194void deferred_split_huge_page(struct page *page);194void deferred_split_huge_page(struct page *page);
195 195
196void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,196void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
197 unsigned long address, bool freeze, struct page *page);197 unsigned long address, bool freeze, struct folio *folio);
198 198
199#define split_huge_pmd(__vma, __pmd, __address) \199#define split_huge_pmd(__vma, __pmd, __address) \
200 do { \200 do { \
@@ -207,7 +207,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, · 6 unmodified lines
207 207
208 208
209void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,209void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
210 bool freeze, struct page *page);210 bool freeze, struct folio *folio);
211 211
212void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,212void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
213 unsigned long address);213 unsigned long address);
@@ -250,30 +250,6 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, · 36 unmodified lines
250 return NULL;250 return NULL;
251}251}
252 252
253/**
254 * thp_order - Order of a transparent huge page.
255 * @page: Head page of a transparent huge page.
256 */
257static inline unsigned int thp_order(struct page *page)
258{
259 VM_BUG_ON_PGFLAGS(PageTail(page), page);
260 if (PageHead(page))
261 return HPAGE_PMD_ORDER;
262 return 0;
263}
264
265/**
266 * thp_nr_pages - The number of regular pages in this huge page.
267 * @page: The head page of a huge page.
268 */
269static inline int thp_nr_pages(struct page *page)
270{
271 VM_BUG_ON_PGFLAGS(PageTail(page), page);
272 if (PageHead(page))
273 return HPAGE_PMD_NR;
274 return 1;
275}
276
277/**253/**
278 * folio_test_pmd_mappable - Can we map this folio with a PMD?254 * folio_test_pmd_mappable - Can we map this folio with a PMD?
279 * @folio: The folio to test255 * @folio: The folio to test
@@ -336,18 +312,6 @@ static inline struct list_head *page_deferred_list(struct page *page) · 56 unmodified lines
336#define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })312#define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
337#define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })313#define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
338 314
339static inline unsigned int thp_order(struct page *page)
340{
341 VM_BUG_ON_PGFLAGS(PageTail(page), page);
342 return 0;
343}
344
345static inline int thp_nr_pages(struct page *page)
346{
347 VM_BUG_ON_PGFLAGS(PageTail(page), page);
348 return 1;
349}
350
351static inline bool folio_test_pmd_mappable(struct folio *folio)315static inline bool folio_test_pmd_mappable(struct folio *folio)
352{316{
353 return false;317 return false;
@@ -387,7 +351,7 @@ static inline bool is_transparent_hugepage(struct page *page) · 33 unmodified lines
387#define thp_get_unmapped_area NULL351#define thp_get_unmapped_area NULL
388 352
389static inline bool353static inline bool
390can_split_huge_page(struct page *page, int *pextra_pins)354can_split_folio(struct folio *folio, int *pextra_pins)
391{355{
392 BUILD_BUG();356 BUILD_BUG();
393 return false;357 return false;
@@ -406,9 +370,9 @@ static inline void deferred_split_huge_page(struct page *page) {} · 12 unmodified lines
406 do { } while (0)370 do { } while (0)
407 371
408static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,372static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
409 unsigned long address, bool freeze, struct page *page) {}373 unsigned long address, bool freeze, struct folio *folio) {}
410static inline void split_huge_pmd_address(struct vm_area_struct *vma,374static inline void split_huge_pmd_address(struct vm_area_struct *vma,
411 unsigned long address, bool freeze, struct page *page) {}375 unsigned long address, bool freeze, struct folio *folio) {}
412 376
413#define split_huge_pud(__vma, __pmd, __address) \377#define split_huge_pud(__vma, __pmd, __address) \
414 do { } while (0)378 do { } while (0)
@@ -483,15 +447,10 @@ static inline bool thp_migration_supported(void) · 68 unmodified lines
483}447}
484#endif /* CONFIG_TRANSPARENT_HUGEPAGE */448#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
485 449
486/**450static inline int split_folio_to_list(struct folio *folio,
487 * thp_size - Size of a transparent huge page.451 struct list_head *list)
488 * @page: Head page of a transparent huge page.
489 *
490 * Return: Number of bytes in this page.
491 */
492static inline unsigned long thp_size(struct page *page)
493{452{
494 return PAGE_SIZE << thp_order(page);453 return split_huge_page_to_list(&folio->page, list);
495}454}
496 455
497#endif /* _LINUX_HUGE_MM_H */456#endif /* _LINUX_HUGE_MM_H */
include/linux/hugetlb.h+5−0
@@ -970,6 +970,11 @@ static inline struct hstate *page_hstate(struct page *page) · 969 unmodified lines
970 return NULL;970 return NULL;
971}971}
972 972
973static inline struct hstate *size_to_hstate(unsigned long size)
974{
975 return NULL;
976}
977
973static inline unsigned long huge_page_size(struct hstate *h)978static inline unsigned long huge_page_size(struct hstate *h)
974{979{
975 return PAGE_SIZE;980 return PAGE_SIZE;
include/linux/ksm.h+3−3
@@ -51,7 +51,7 @@ static inline void ksm_exit(struct mm_struct *mm) · 50 unmodified lines
51struct page *ksm_might_need_to_copy(struct page *page,51struct page *ksm_might_need_to_copy(struct page *page,
52 struct vm_area_struct *vma, unsigned long address);52 struct vm_area_struct *vma, unsigned long address);
53 53
54void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);54void rmap_walk_ksm(struct folio *folio, const struct rmap_walk_control *rwc);
55void folio_migrate_ksm(struct folio *newfolio, struct folio *folio);55void folio_migrate_ksm(struct folio *newfolio, struct folio *folio);
56 56
57#else /* !CONFIG_KSM */57#else /* !CONFIG_KSM */
@@ -78,8 +78,8 @@ static inline struct page *ksm_might_need_to_copy(struct page *page, · 20 unmodified lines
78 return page;78 return page;
79}79}
80 80
81static inline void rmap_walk_ksm(struct page *page,81static inline void rmap_walk_ksm(struct folio *folio,
82 struct rmap_walk_control *rwc)82 const struct rmap_walk_control *rwc)
83{83{
84}84}
85 85
include/linux/memremap.h+24−3
@@ -1,6 +1,8 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_MEMREMAP_H_2#ifndef _LINUX_MEMREMAP_H_
3#define _LINUX_MEMREMAP_H_3#define _LINUX_MEMREMAP_H_
4
5#include <linux/mm.h>
4#include <linux/range.h>6#include <linux/range.h>
5#include <linux/ioport.h>7#include <linux/ioport.h>
6#include <linux/percpu-refcount.h>8#include <linux/percpu-refcount.h>
@@ -66,9 +68,9 @@ enum memory_type { · 59 unmodified lines
66 68
67struct dev_pagemap_ops {69struct dev_pagemap_ops {
68 /*70 /*
69 * Called once the page refcount reaches 1. (ZONE_DEVICE pages never71 * Called once the page refcount reaches 0. The reference count will be
70 * reach 0 refcount unless there is a refcount bug. This allows the72 * reset to one by the core code after the method is called to prepare
71 * device driver to implement its own memory management.)73 * for handing out the page again.
72 */74 */
73 void (*page_free)(struct page *page);75 void (*page_free)(struct page *page);
74 76
@@ -129,6 +131,25 @@ static inline unsigned long pgmap_vmemmap_nr(struct dev_pagemap *pgmap) · 54 unmodified lines
129 return 1 << pgmap->vmemmap_shift;131 return 1 << pgmap->vmemmap_shift;
130}132}
131 133
134static inline bool is_device_private_page(const struct page *page)
135{
136 return IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
137 is_zone_device_page(page) &&
138 page->pgmap->type == MEMORY_DEVICE_PRIVATE;
139}
140
141static inline bool folio_is_device_private(const struct folio *folio)
142{
143 return is_device_private_page(&folio->page);
144}
145
146static inline bool is_pci_p2pdma_page(const struct page *page)
147{
148 return IS_ENABLED(CONFIG_PCI_P2PDMA) &&
149 is_zone_device_page(page) &&
150 page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
151}
152
132#ifdef CONFIG_ZONE_DEVICE153#ifdef CONFIG_ZONE_DEVICE
133void *memremap_pages(struct dev_pagemap *pgmap, int nid);154void *memremap_pages(struct dev_pagemap *pgmap, int nid);
134void memunmap_pages(struct dev_pagemap *pgmap);155void memunmap_pages(struct dev_pagemap *pgmap);
include/linux/mm.h+152−144
@@ -3,9 +3,6 @@ · 2 unmodified lines
3#define _LINUX_MM_H3#define _LINUX_MM_H
4 4
5#include <linux/errno.h>5#include <linux/errno.h>
6
7#ifdef __KERNEL__
8
9#include <linux/mmdebug.h>6#include <linux/mmdebug.h>
10#include <linux/gfp.h>7#include <linux/gfp.h>
11#include <linux/bug.h>8#include <linux/bug.h>
@@ -26,7 +23,6 @@ · 14 unmodified lines
26#include <linux/err.h>23#include <linux/err.h>
27#include <linux/page-flags.h>24#include <linux/page-flags.h>
28#include <linux/page_ref.h>25#include <linux/page_ref.h>
29#include <linux/memremap.h>
30#include <linux/overflow.h>26#include <linux/overflow.h>
31#include <linux/sizes.h>27#include <linux/sizes.h>
32#include <linux/sched.h>28#include <linux/sched.h>
@@ -216,8 +212,10 @@ int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, · 183 unmodified lines
216 212
217#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)213#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
218#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))214#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
215#define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio))
219#else216#else
220#define nth_page(page,n) ((page) + (n))217#define nth_page(page,n) ((page) + (n))
218#define folio_page_idx(folio, p) ((p) - &(folio)->page)
221#endif219#endif
222 220
223/* to align the pointer to the (next) page boundary */221/* to align the pointer to the (next) page boundary */
@@ -227,6 +225,10 @@ int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, · 3 unmodified lines
227#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)225#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
228 226
229#define lru_to_page(head) (list_entry((head)->prev, struct page, lru))227#define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
228static inline struct folio *lru_to_folio(struct list_head *head)
229{
230 return list_entry((head)->prev, struct folio, lru);
231}
230 232
231void setup_initial_init_mm(void *start_code, void *end_code,233void setup_initial_init_mm(void *start_code, void *end_code,
232 void *end_data, void *brk);234 void *end_data, void *brk);
@@ -775,21 +777,26 @@ static inline int is_vmalloc_or_module_addr(const void *x) · 542 unmodified lines
775}777}
776#endif778#endif
777 779
778static inline int head_compound_mapcount(struct page *head)780/*
781 * How many times the entire folio is mapped as a single unit (eg by a
782 * PMD or PUD entry). This is probably not what you want, except for
783 * debugging purposes; look at folio_mapcount() or page_mapcount()
784 * instead.
785 */
786static inline int folio_entire_mapcount(struct folio *folio)
779{787{
780 return atomic_read(compound_mapcount_ptr(head)) + 1;788 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
789 return atomic_read(folio_mapcount_ptr(folio)) + 1;
781}790}
782 791
783/*792/*
784 * Mapcount of compound page as a whole, does not include mapped sub-pages.793 * Mapcount of compound page as a whole, does not include mapped sub-pages.
785 *794 *
786 * Must be called only for compound pages or any their tail sub-pages.795 * Must be called only for compound pages.
787 */796 */
788static inline int compound_mapcount(struct page *page)797static inline int compound_mapcount(struct page *page)
789{798{
790 VM_BUG_ON_PAGE(!PageCompound(page), page);799 return folio_entire_mapcount(page_folio(page));
791 page = compound_head(page);
792 return head_compound_mapcount(page);
793}800}
794 801
795/*802/*
@@ -819,8 +826,14 @@ static inline int page_mapcount(struct page *page) · 23 unmodified lines
819 return atomic_read(&page->_mapcount) + 1;826 return atomic_read(&page->_mapcount) + 1;
820}827}
821 828
829int folio_mapcount(struct folio *folio);
830
822#ifdef CONFIG_TRANSPARENT_HUGEPAGE831#ifdef CONFIG_TRANSPARENT_HUGEPAGE
823int total_mapcount(struct page *page);832static inline int total_mapcount(struct page *page)
833{
834 return folio_mapcount(page_folio(page));
835}
836
824int page_trans_huge_mapcount(struct page *page);837int page_trans_huge_mapcount(struct page *page);
825#else838#else
826static inline int total_mapcount(struct page *page)839static inline int total_mapcount(struct page *page)
@@ -890,33 +903,17 @@ static inline void destroy_compound_page(struct page *page) · 63 unmodified lines
890 compound_page_dtors[page[1].compound_dtor](page);903 compound_page_dtors[page[1].compound_dtor](page);
891}904}
892 905
893static inline bool hpage_pincount_available(struct page *page)
894{
895 /*
896 * Can the page->hpage_pinned_refcount field be used? That field is in
897 * the 3rd page of the compound page, so the smallest (2-page) compound
898 * pages cannot support it.
899 */
900 page = compound_head(page);
901 return PageCompound(page) && compound_order(page) > 1;
902}
903
904static inline int head_compound_pincount(struct page *head)906static inline int head_compound_pincount(struct page *head)
905{907{
906 return atomic_read(compound_pincount_ptr(head));908 return atomic_read(compound_pincount_ptr(head));
907}909}
908 910
909static inline int compound_pincount(struct page *page)
910{
911 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
912 page = compound_head(page);
913 return head_compound_pincount(page);
914}
915
916static inline void set_compound_order(struct page *page, unsigned int order)911static inline void set_compound_order(struct page *page, unsigned int order)
917{912{
918 page[1].compound_order = order;913 page[1].compound_order = order;
914#ifdef CONFIG_64BIT
919 page[1].compound_nr = 1U << order;915 page[1].compound_nr = 1U << order;
916#endif
920}917}
921 918
922/* Returns the number of pages in this potentially compound page. */919/* Returns the number of pages in this potentially compound page. */
@@ -924,7 +921,11 @@ static inline unsigned long compound_nr(struct page *page) · 1 unmodified line
924{921{
925 if (!PageHead(page))922 if (!PageHead(page))
926 return 1;923 return 1;
924#ifdef CONFIG_64BIT
927 return page[1].compound_nr;925 return page[1].compound_nr;
926#else
927 return 1UL << compound_order(page);
928#endif
928}929}
929 930
930/* Returns the number of bytes in this potentially compound page. */931/* Returns the number of bytes in this potentially compound page. */
@@ -939,6 +940,37 @@ static inline unsigned int page_shift(struct page *page) · 8 unmodified lines
939 return PAGE_SHIFT + compound_order(page);940 return PAGE_SHIFT + compound_order(page);
940}941}
941 942
943/**
944 * thp_order - Order of a transparent huge page.
945 * @page: Head page of a transparent huge page.
946 */
947static inline unsigned int thp_order(struct page *page)
948{
949 VM_BUG_ON_PGFLAGS(PageTail(page), page);
950 return compound_order(page);
951}
952
953/**
954 * thp_nr_pages - The number of regular pages in this huge page.
955 * @page: The head page of a huge page.
956 */
957static inline int thp_nr_pages(struct page *page)
958{
959 VM_BUG_ON_PGFLAGS(PageTail(page), page);
960 return compound_nr(page);
961}
962
963/**
964 * thp_size - Size of a transparent huge page.
965 * @page: Head page of a transparent huge page.
966 *
967 * Return: Number of bytes in this page.
968 */
969static inline unsigned long thp_size(struct page *page)
970{
971 return PAGE_SIZE << thp_order(page);
972}
973
942void free_compound_page(struct page *page);974void free_compound_page(struct page *page);
943 975
944#ifdef CONFIG_MMU976#ifdef CONFIG_MMU
@@ -1090,59 +1122,35 @@ static inline bool is_zone_device_page(const struct page *page) · 145 unmodified lines
1090}1122}
1091#endif1123#endif
1092 1124
1125static inline bool folio_is_zone_device(const struct folio *folio)
1126{
1127 return is_zone_device_page(&folio->page);
1128}
1129
1093static inline bool is_zone_movable_page(const struct page *page)1130static inline bool is_zone_movable_page(const struct page *page)
1094{1131{
1095 return page_zonenum(page) == ZONE_MOVABLE;1132 return page_zonenum(page) == ZONE_MOVABLE;
1096}1133}
1097 1134
1098#ifdef CONFIG_DEV_PAGEMAP_OPS1135#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX)
1099void free_devmap_managed_page(struct page *page);
1100DECLARE_STATIC_KEY_FALSE(devmap_managed_key);1136DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
1101 1137
1102static inline bool page_is_devmap_managed(struct page *page)1138bool __put_devmap_managed_page(struct page *page);
1139static inline bool put_devmap_managed_page(struct page *page)
1103{1140{
1104 if (!static_branch_unlikely(&devmap_managed_key))1141 if (!static_branch_unlikely(&devmap_managed_key))
1105 return false;1142 return false;
1106 if (!is_zone_device_page(page))1143 if (!is_zone_device_page(page))
1107 return false;1144 return false;
1108 switch (page->pgmap->type) {1145 return __put_devmap_managed_page(page);
1109 case MEMORY_DEVICE_PRIVATE:
1110 case MEMORY_DEVICE_FS_DAX:
1111 return true;
1112 default:
1113 break;
1114 }
1115 return false;
1116}1146}
1117 1147
1118void put_devmap_managed_page(struct page *page);1148#else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
1119 1149static inline bool put_devmap_managed_page(struct page *page)
1120#else /* CONFIG_DEV_PAGEMAP_OPS */
1121static inline bool page_is_devmap_managed(struct page *page)
1122{1150{
1123 return false;1151 return false;
1124}1152}
1125 1153#endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
1126static inline void put_devmap_managed_page(struct page *page)
1127{
1128}
1129#endif /* CONFIG_DEV_PAGEMAP_OPS */
1130
1131static inline bool is_device_private_page(const struct page *page)
1132{
1133 return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
1134 IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
1135 is_zone_device_page(page) &&
1136 page->pgmap->type == MEMORY_DEVICE_PRIVATE;
1137}
1138
1139static inline bool is_pci_p2pdma_page(const struct page *page)
1140{
1141 return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
1142 IS_ENABLED(CONFIG_PCI_P2PDMA) &&
1143 is_zone_device_page(page) &&
1144 page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
1145}
1146 1154
1147/* 127: arbitrary random number, small enough to assemble well */1155/* 127: arbitrary random number, small enough to assemble well */
1148#define folio_ref_zero_or_close_to_overflow(folio) \1156#define folio_ref_zero_or_close_to_overflow(folio) \
@@ -1168,9 +1176,6 @@ static inline void get_page(struct page *page) · 19 unmodified lines
1168}1176}
1169 1177
1170bool __must_check try_grab_page(struct page *page, unsigned int flags);1178bool __must_check try_grab_page(struct page *page, unsigned int flags);
1171struct page *try_grab_compound_head(struct page *page, int refs,
1172 unsigned int flags);
1173
1174 1179
1175static inline __must_check bool try_get_page(struct page *page)1180static inline __must_check bool try_get_page(struct page *page)
1176{1181{
@@ -1225,16 +1230,11 @@ static inline void put_page(struct page *page) · 48 unmodified lines
1225 struct folio *folio = page_folio(page);1230 struct folio *folio = page_folio(page);
1226 1231
1227 /*1232 /*
1228 * For devmap managed pages we need to catch refcount transition from1233 * For some devmap managed pages we need to catch refcount transition
1229 * 2 to 1, when refcount reach one it means the page is free and we1234 * from 2 to 1:
1230 * need to inform the device driver through callback. See
1231 * include/linux/memremap.h and HMM for details.
1232 */1235 */
1233 if (page_is_devmap_managed(&folio->page)) {1236 if (put_devmap_managed_page(&folio->page))
1234 put_devmap_managed_page(&folio->page);
1235 return;1237 return;
1236 }
1237
1238 folio_put(folio);1238 folio_put(folio);
1239}1239}
1240 1240
@@ -1264,10 +1264,9 @@ static inline void put_page(struct page *page) · 23 unmodified lines
1264 * applications that don't have huge page reference counts, this won't be an1264 * applications that don't have huge page reference counts, this won't be an
1265 * issue.1265 * issue.
1266 *1266 *
1267 * Locking: the lockless algorithm described in page_cache_get_speculative()1267 * Locking: the lockless algorithm described in folio_try_get_rcu()
1268 * and page_cache_gup_pin_speculative() provides safe operation for1268 * provides safe operation for get_user_pages(), page_mkclean() and
1269 * get_user_pages and page_mkclean and other calls that race to set up page1269 * other calls that race to set up page table entries.
1270 * table entries.
1271 */1270 */
1272#define GUP_PIN_COUNTING_BIAS (1U << 10)1271#define GUP_PIN_COUNTING_BIAS (1U << 10)
1273 1272
@@ -1278,70 +1277,11 @@ void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, · 4 unmodified lines
1278 bool make_dirty);1277 bool make_dirty);
1279void unpin_user_pages(struct page **pages, unsigned long npages);1278void unpin_user_pages(struct page **pages, unsigned long npages);
1280 1279
1281/**
1282 * page_maybe_dma_pinned - Report if a page is pinned for DMA.
1283 * @page: The page.
1284 *
1285 * This function checks if a page has been pinned via a call to
1286 * a function in the pin_user_pages() family.
1287 *
1288 * For non-huge pages, the return value is partially fuzzy: false is not fuzzy,
1289 * because it means "definitely not pinned for DMA", but true means "probably
1290 * pinned for DMA, but possibly a false positive due to having at least
1291 * GUP_PIN_COUNTING_BIAS worth of normal page references".
1292 *
1293 * False positives are OK, because: a) it's unlikely for a page to get that many
1294 * refcounts, and b) all the callers of this routine are expected to be able to
1295 * deal gracefully with a false positive.
1296 *
1297 * For huge pages, the result will be exactly correct. That's because we have
1298 * more tracking data available: the 3rd struct page in the compound page is
1299 * used to track the pincount (instead using of the GUP_PIN_COUNTING_BIAS
1300 * scheme).
1301 *
1302 * For more information, please see Documentation/core-api/pin_user_pages.rst.
1303 *
1304 * Return: True, if it is likely that the page has been "dma-pinned".
1305 * False, if the page is definitely not dma-pinned.
1306 */
1307static inline bool page_maybe_dma_pinned(struct page *page)
1308{
1309 if (hpage_pincount_available(page))
1310 return compound_pincount(page) > 0;
1311
1312 /*
1313 * page_ref_count() is signed. If that refcount overflows, then
1314 * page_ref_count() returns a negative value, and callers will avoid
1315 * further incrementing the refcount.
1316 *
1317 * Here, for that overflow case, use the signed bit to count a little
1318 * bit higher via unsigned math, and thus still get an accurate result.
1319 */
1320 return ((unsigned int)page_ref_count(compound_head(page))) >=
1321 GUP_PIN_COUNTING_BIAS;
1322}
1323
1324static inline bool is_cow_mapping(vm_flags_t flags)1280static inline bool is_cow_mapping(vm_flags_t flags)
1325{1281{
1326 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;1282 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
1327}1283}
1328 1284
1329/*
1330 * This should most likely only be called during fork() to see whether we
1331 * should break the cow immediately for a page on the src mm.
1332 */
1333static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma,
1334 struct page *page)
1335{
1336 if (!is_cow_mapping(vma->vm_flags))
1337 return false;
1338
1339 if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
1340 return false;
1341
1342 return page_maybe_dma_pinned(page);
1343}
1344
1345#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)1285#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
1346#define SECTION_IN_PAGE_FLAGS1286#define SECTION_IN_PAGE_FLAGS
1347#endif1287#endif
@@ -1586,6 +1526,74 @@ static inline unsigned long folio_pfn(struct folio *folio) · 238 unmodified lines
1586 return page_to_pfn(&folio->page);1526 return page_to_pfn(&folio->page);
1587}1527}
1588 1528
1529static inline atomic_t *folio_pincount_ptr(struct folio *folio)
1530{
1531 return &folio_page(folio, 1)->compound_pincount;
1532}
1533
1534/**
1535 * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA.
1536 * @folio: The folio.
1537 *
1538 * This function checks if a folio has been pinned via a call to
1539 * a function in the pin_user_pages() family.
1540 *
1541 * For small folios, the return value is partially fuzzy: false is not fuzzy,
1542 * because it means "definitely not pinned for DMA", but true means "probably
1543 * pinned for DMA, but possibly a false positive due to having at least
1544 * GUP_PIN_COUNTING_BIAS worth of normal folio references".
1545 *
1546 * False positives are OK, because: a) it's unlikely for a folio to
1547 * get that many refcounts, and b) all the callers of this routine are
1548 * expected to be able to deal gracefully with a false positive.
1549 *
1550 * For large folios, the result will be exactly correct. That's because
1551 * we have more tracking data available: the compound_pincount is used
1552 * instead of the GUP_PIN_COUNTING_BIAS scheme.
1553 *
1554 * For more information, please see Documentation/core-api/pin_user_pages.rst.
1555 *
1556 * Return: True, if it is likely that the page has been "dma-pinned".
1557 * False, if the page is definitely not dma-pinned.
1558 */
1559static inline bool folio_maybe_dma_pinned(struct folio *folio)
1560{
1561 if (folio_test_large(folio))
1562 return atomic_read(folio_pincount_ptr(folio)) > 0;
1563
1564 /*
1565 * folio_ref_count() is signed. If that refcount overflows, then
1566 * folio_ref_count() returns a negative value, and callers will avoid
1567 * further incrementing the refcount.
1568 *
1569 * Here, for that overflow case, use the sign bit to count a little
1570 * bit higher via unsigned math, and thus still get an accurate result.
1571 */
1572 return ((unsigned int)folio_ref_count(folio)) >=
1573 GUP_PIN_COUNTING_BIAS;
1574}
1575
1576static inline bool page_maybe_dma_pinned(struct page *page)
1577{
1578 return folio_maybe_dma_pinned(page_folio(page));
1579}
1580
1581/*
1582 * This should most likely only be called during fork() to see whether we
1583 * should break the cow immediately for a page on the src mm.
1584 */
1585static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma,
1586 struct page *page)
1587{
1588 if (!is_cow_mapping(vma->vm_flags))
1589 return false;
1590
1591 if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
1592 return false;
1593
1594 return page_maybe_dma_pinned(page);
1595}
1596
1589/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */1597/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */
1590#ifdef CONFIG_MIGRATION1598#ifdef CONFIG_MIGRATION
1591static inline bool is_pinnable_page(struct page *page)1599static inline bool is_pinnable_page(struct page *page)
@@ -1600,6 +1608,11 @@ static inline bool is_pinnable_page(struct page *page) · 8 unmodified lines
1600}1608}
1601#endif1609#endif
1602 1610
1611static inline bool folio_is_pinnable(struct folio *folio)
1612{
1613 return is_pinnable_page(&folio->page);
1614}
1615
1603static inline void set_page_zone(struct page *page, enum zone_type zone)1616static inline void set_page_zone(struct page *page, enum zone_type zone)
1604{1617{
1605 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);1618 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
@@ -1749,7 +1762,6 @@ static inline void *folio_address(const struct folio *folio) · 143 unmodified lines
1749}1762}
1750 1763
1751extern void *page_rmapping(struct page *page);1764extern void *page_rmapping(struct page *page);
1752extern struct anon_vma *page_anon_vma(struct page *page);
1753extern pgoff_t __page_file_index(struct page *page);1765extern pgoff_t __page_file_index(struct page *page);
1754 1766
1755/*1767/*
@@ -1855,7 +1867,6 @@ extern void truncate_setsize(struct inode *inode, loff_t newsize); · 99 unmodified lines
1855void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);1867void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
1856void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);1868void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1857int generic_error_remove_page(struct address_space *mapping, struct page *page);1869int generic_error_remove_page(struct address_space *mapping, struct page *page);
1858int invalidate_inode_page(struct page *page);
1859 1870
1860#ifdef CONFIG_MMU1871#ifdef CONFIG_MMU
1861extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,1872extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
@@ -2921,13 +2932,11 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, · 1059 unmodified lines
2921#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */2932#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
2922#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO2933#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
2923 * and return without waiting upon it */2934 * and return without waiting upon it */
2924#define FOLL_POPULATE 0x40 /* fault in pages (with FOLL_MLOCK) */
2925#define FOLL_NOFAULT 0x80 /* do not fault in pages */2935#define FOLL_NOFAULT 0x80 /* do not fault in pages */
2926#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */2936#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2927#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */2937#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2928#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */2938#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2929#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */2939#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
2930#define FOLL_MLOCK 0x1000 /* lock present pages */
2931#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */2940#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
2932#define FOLL_COW 0x4000 /* internal GUP flag */2941#define FOLL_COW 0x4000 /* internal GUP flag */
2933#define FOLL_ANON 0x8000 /* don't do file mappings */2942#define FOLL_ANON 0x8000 /* don't do file mappings */
@@ -3381,5 +3390,4 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start, · 447 unmodified lines
3381}3390}
3382#endif3391#endif
3383 3392
3384#endif /* __KERNEL__ */
3385#endif /* _LINUX_MM_H */3393#endif /* _LINUX_MM_H */
include/linux/mm_inline.h+8−3
@@ -99,7 +99,8 @@ void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) · 98 unmodified lines
99 99
100 update_lru_size(lruvec, lru, folio_zonenum(folio),100 update_lru_size(lruvec, lru, folio_zonenum(folio),
101 folio_nr_pages(folio));101 folio_nr_pages(folio));
102 list_add(&folio->lru, &lruvec->lists[lru]);102 if (lru != LRU_UNEVICTABLE)
103 list_add(&folio->lru, &lruvec->lists[lru]);
103}104}
104 105
105static __always_inline void add_page_to_lru_list(struct page *page,106static __always_inline void add_page_to_lru_list(struct page *page,
@@ -115,6 +116,7 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) · 9 unmodified lines
115 116
116 update_lru_size(lruvec, lru, folio_zonenum(folio),117 update_lru_size(lruvec, lru, folio_zonenum(folio),
117 folio_nr_pages(folio));118 folio_nr_pages(folio));
119 /* This is not expected to be used on LRU_UNEVICTABLE */
118 list_add_tail(&folio->lru, &lruvec->lists[lru]);120 list_add_tail(&folio->lru, &lruvec->lists[lru]);
119}121}
120 122
@@ -127,8 +129,11 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page, · 6 unmodified lines
127static __always_inline129static __always_inline
128void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)130void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
129{131{
130 list_del(&folio->lru);132 enum lru_list lru = folio_lru_list(folio);
131 update_lru_size(lruvec, folio_lru_list(folio), folio_zonenum(folio),133
134 if (lru != LRU_UNEVICTABLE)
135 list_del(&folio->lru);
136 update_lru_size(lruvec, lru, folio_zonenum(folio),
132 -folio_nr_pages(folio));137 -folio_nr_pages(folio));
133}138}
134 139
include/linux/mm_types.h+22−4
@@ -85,7 +85,16 @@ struct page { · 84 unmodified lines
85 * lruvec->lru_lock. Sometimes used as a generic list85 * lruvec->lru_lock. Sometimes used as a generic list
86 * by the page owner.86 * by the page owner.
87 */87 */
88 struct list_head lru;88 union {
89 struct list_head lru;
90 /* Or, for the Unevictable "LRU list" slot */
91 struct {
92 /* Always even, to negate PageTail */
93 void *__filler;
94 /* Count page's or folio's mlocks */
95 unsigned int mlock_count;
96 };
97 };
89 /* See page-flags.h for PAGE_MAPPING_FLAGS */98 /* See page-flags.h for PAGE_MAPPING_FLAGS */
90 struct address_space *mapping;99 struct address_space *mapping;
91 pgoff_t index; /* Our offset within mapping. */100 pgoff_t index; /* Our offset within mapping. */
@@ -126,11 +135,14 @@ struct page { · 34 unmodified lines
126 unsigned char compound_dtor;135 unsigned char compound_dtor;
127 unsigned char compound_order;136 unsigned char compound_order;
128 atomic_t compound_mapcount;137 atomic_t compound_mapcount;
138 atomic_t compound_pincount;
139#ifdef CONFIG_64BIT
129 unsigned int compound_nr; /* 1 << compound_order */140 unsigned int compound_nr; /* 1 << compound_order */
141#endif
130 };142 };
131 struct { /* Second tail page of compound page */143 struct { /* Second tail page of compound page */
132 unsigned long _compound_pad_1; /* compound_head */144 unsigned long _compound_pad_1; /* compound_head */
133 atomic_t hpage_pinned_refcount;145 unsigned long _compound_pad_2;
134 /* For both global and memcg */146 /* For both global and memcg */
135 struct list_head deferred_list;147 struct list_head deferred_list;
136 };148 };
@@ -241,7 +253,13 @@ struct folio { · 104 unmodified lines
241 struct {253 struct {
242 /* public: */254 /* public: */
243 unsigned long flags;255 unsigned long flags;
244 struct list_head lru;256 union {
257 struct list_head lru;
258 struct {
259 void *__filler;
260 unsigned int mlock_count;
261 };
262 };
245 struct address_space *mapping;263 struct address_space *mapping;
246 pgoff_t index;264 pgoff_t index;
247 void *private;265 void *private;
@@ -285,7 +303,7 @@ static inline atomic_t *compound_mapcount_ptr(struct page *page) · 37 unmodified lines
285 303
286static inline atomic_t *compound_pincount_ptr(struct page *page)304static inline atomic_t *compound_pincount_ptr(struct page *page)
287{305{
288 return &page[2].hpage_pinned_refcount;306 return &page[1].compound_pincount;
289}307}
290 308
291/*309/*
include/linux/pagemap.h+133−13
@@ -18,6 +18,120 @@ · 17 unmodified lines
18 18
19struct folio_batch;19struct folio_batch;
20 20
21unsigned long invalidate_mapping_pages(struct address_space *mapping,
22 pgoff_t start, pgoff_t end);
23
24static inline void invalidate_remote_inode(struct inode *inode)
25{
26 if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
27 S_ISLNK(inode->i_mode))
28 invalidate_mapping_pages(inode->i_mapping, 0, -1);
29}
30int invalidate_inode_pages2(struct address_space *mapping);
31int invalidate_inode_pages2_range(struct address_space *mapping,
32 pgoff_t start, pgoff_t end);
33int write_inode_now(struct inode *, int sync);
34int filemap_fdatawrite(struct address_space *);
35int filemap_flush(struct address_space *);
36int filemap_fdatawait_keep_errors(struct address_space *mapping);
37int filemap_fdatawait_range(struct address_space *, loff_t lstart, loff_t lend);
38int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
39 loff_t start_byte, loff_t end_byte);
40
41static inline int filemap_fdatawait(struct address_space *mapping)
42{
43 return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
44}
45
46bool filemap_range_has_page(struct address_space *, loff_t lstart, loff_t lend);
47int filemap_write_and_wait_range(struct address_space *mapping,
48 loff_t lstart, loff_t lend);
49int __filemap_fdatawrite_range(struct address_space *mapping,
50 loff_t start, loff_t end, int sync_mode);
51int filemap_fdatawrite_range(struct address_space *mapping,
52 loff_t start, loff_t end);
53int filemap_check_errors(struct address_space *mapping);
54void __filemap_set_wb_err(struct address_space *mapping, int err);
55int filemap_fdatawrite_wbc(struct address_space *mapping,
56 struct writeback_control *wbc);
57
58static inline int filemap_write_and_wait(struct address_space *mapping)
59{
60 return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
61}
62
63/**
64 * filemap_set_wb_err - set a writeback error on an address_space
65 * @mapping: mapping in which to set writeback error
66 * @err: error to be set in mapping
67 *
68 * When writeback fails in some way, we must record that error so that
69 * userspace can be informed when fsync and the like are called. We endeavor
70 * to report errors on any file that was open at the time of the error. Some
71 * internal callers also need to know when writeback errors have occurred.
72 *
73 * When a writeback error occurs, most filesystems will want to call
74 * filemap_set_wb_err to record the error in the mapping so that it will be
75 * automatically reported whenever fsync is called on the file.
76 */
77static inline void filemap_set_wb_err(struct address_space *mapping, int err)
78{
79 /* Fastpath for common case of no error */
80 if (unlikely(err))
81 __filemap_set_wb_err(mapping, err);
82}
83
84/**
85 * filemap_check_wb_err - has an error occurred since the mark was sampled?
86 * @mapping: mapping to check for writeback errors
87 * @since: previously-sampled errseq_t
88 *
89 * Grab the errseq_t value from the mapping, and see if it has changed "since"
90 * the given value was sampled.
91 *
92 * If it has then report the latest error set, otherwise return 0.
93 */
94static inline int filemap_check_wb_err(struct address_space *mapping,
95 errseq_t since)
96{
97 return errseq_check(&mapping->wb_err, since);
98}
99
100/**
101 * filemap_sample_wb_err - sample the current errseq_t to test for later errors
102 * @mapping: mapping to be sampled
103 *
104 * Writeback errors are always reported relative to a particular sample point
105 * in the past. This function provides those sample points.
106 */
107static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
108{
109 return errseq_sample(&mapping->wb_err);
110}
111
112/**
113 * file_sample_sb_err - sample the current errseq_t to test for later errors
114 * @file: file pointer to be sampled
115 *
116 * Grab the most current superblock-level errseq_t value for the given
117 * struct file.
118 */
119static inline errseq_t file_sample_sb_err(struct file *file)
120{
121 return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
122}
123
124/*
125 * Flush file data before changing attributes. Caller must hold any locks
126 * required to prevent further writes to this file until we're done setting
127 * flags.
128 */
129static inline int inode_drain_writes(struct inode *inode)
130{
131 inode_dio_wait(inode);
132 return filemap_write_and_wait(inode->i_mapping);
133}
134
21static inline bool mapping_empty(struct address_space *mapping)135static inline bool mapping_empty(struct address_space *mapping)
22{136{
23 return xa_empty(&mapping->i_pages);137 return xa_empty(&mapping->i_pages);
@@ -192,9 +306,14 @@ static inline void mapping_set_large_folios(struct address_space *mapping) · 168 unmodified lines
192 __set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);306 __set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
193}307}
194 308
309/*
310 * Large folio support currently depends on THP. These dependencies are
311 * being worked on but are not yet fixed.
312 */
195static inline bool mapping_large_folio_support(struct address_space *mapping)313static inline bool mapping_large_folio_support(struct address_space *mapping)
196{314{
197 return test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);315 return IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
316 test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
198}317}
199 318
200static inline int filemap_nr_thps(struct address_space *mapping)319static inline int filemap_nr_thps(struct address_space *mapping)
@@ -212,7 +331,7 @@ static inline void filemap_nr_thps_inc(struct address_space *mapping) · 11 unmodified lines
212 if (!mapping_large_folio_support(mapping))331 if (!mapping_large_folio_support(mapping))
213 atomic_inc(&mapping->nr_thps);332 atomic_inc(&mapping->nr_thps);
214#else333#else
215 WARN_ON_ONCE(1);334 WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
216#endif335#endif
217}336}
218 337
@@ -222,7 +341,7 @@ static inline void filemap_nr_thps_dec(struct address_space *mapping) · 3 unmodified lines
222 if (!mapping_large_folio_support(mapping))341 if (!mapping_large_folio_support(mapping))
223 atomic_dec(&mapping->nr_thps);342 atomic_dec(&mapping->nr_thps);
224#else343#else
225 WARN_ON_ONCE(1);344 WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
226#endif345#endif
227}346}
228 347
@@ -283,16 +402,6 @@ static inline struct inode *folio_inode(struct folio *folio) · 54 unmodified lines
283 return folio->mapping->host;402 return folio->mapping->host;
284}403}
285 404
286static inline bool page_cache_add_speculative(struct page *page, int count)
287{
288 return folio_ref_try_add_rcu((struct folio *)page, count);
289}
290
291static inline bool page_cache_get_speculative(struct page *page)
292{
293 return page_cache_add_speculative(page, 1);
294}
295
296/**405/**
297 * folio_attach_private - Attach private data to a folio.406 * folio_attach_private - Attach private data to a folio.
298 * @folio: Folio to attach data to.407 * @folio: Folio to attach data to.
@@ -706,6 +815,17 @@ static inline loff_t folio_file_pos(struct folio *folio) · 407 unmodified lines
706 return page_file_offset(&folio->page);815 return page_file_offset(&folio->page);
707}816}
708 817
818/*
819 * Get the offset in PAGE_SIZE (even for hugetlb folios).
820 * (TODO: hugetlb folios should have ->index in PAGE_SIZE)
821 */
822static inline pgoff_t folio_pgoff(struct folio *folio)
823{
824 if (unlikely(folio_test_hugetlb(folio)))
825 return hugetlb_basepage_index(&folio->page);
826 return folio->index;
827}
828
709extern pgoff_t linear_hugepage_index(struct vm_area_struct *vma,829extern pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
710 unsigned long address);830 unsigned long address);
711 831
include/linux/rmap.h+47−29
@@ -11,6 +11,7 @@ · 10 unmodified lines
11#include <linux/rwsem.h>11#include <linux/rwsem.h>
12#include <linux/memcontrol.h>12#include <linux/memcontrol.h>
13#include <linux/highmem.h>13#include <linux/highmem.h>
14#include <linux/pagemap.h>
14 15
15/*16/*
16 * The anon_vma heads a list of private "related" vmas, to scan if17 * The anon_vma heads a list of private "related" vmas, to scan if
@@ -167,18 +168,19 @@ struct anon_vma *page_get_anon_vma(struct page *page); · 150 unmodified lines
167 */168 */
168void page_move_anon_rmap(struct page *, struct vm_area_struct *);169void page_move_anon_rmap(struct page *, struct vm_area_struct *);
169void page_add_anon_rmap(struct page *, struct vm_area_struct *,170void page_add_anon_rmap(struct page *, struct vm_area_struct *,
170 unsigned long, bool);171 unsigned long address, bool compound);
171void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,172void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
172 unsigned long, int);173 unsigned long address, int flags);
173void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,174void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
174 unsigned long, bool);175 unsigned long address, bool compound);
175void page_add_file_rmap(struct page *, bool);176void page_add_file_rmap(struct page *, struct vm_area_struct *,
176void page_remove_rmap(struct page *, bool);177 bool compound);
177 178void page_remove_rmap(struct page *, struct vm_area_struct *,
179 bool compound);
178void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,180void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,
179 unsigned long);181 unsigned long address);
180void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *,182void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *,
181 unsigned long);183 unsigned long address);
182 184
183static inline void page_dup_rmap(struct page *page, bool compound)185static inline void page_dup_rmap(struct page *page, bool compound)
184{186{
@@ -188,11 +190,11 @@ static inline void page_dup_rmap(struct page *page, bool compound) · 3 unmodified lines
188/*190/*
189 * Called from mm/vmscan.c to handle paging out191 * Called from mm/vmscan.c to handle paging out
190 */192 */
191int page_referenced(struct page *, int is_locked,193int folio_referenced(struct folio *, int is_locked,
192 struct mem_cgroup *memcg, unsigned long *vm_flags);194 struct mem_cgroup *memcg, unsigned long *vm_flags);
193 195
194void try_to_migrate(struct page *page, enum ttu_flags flags);196void try_to_migrate(struct folio *folio, enum ttu_flags flags);
195void try_to_unmap(struct page *, enum ttu_flags flags);197void try_to_unmap(struct folio *, enum ttu_flags flags);
196 198
197int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,199int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
198 unsigned long end, struct page **pages,200 unsigned long end, struct page **pages,
@@ -200,11 +202,13 @@ int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, · 1 unmodified line
200 202
201/* Avoid racy checks */203/* Avoid racy checks */
202#define PVMW_SYNC (1 << 0)204#define PVMW_SYNC (1 << 0)
203/* Look for migarion entries rather than present PTEs */205/* Look for migration entries rather than present PTEs */
204#define PVMW_MIGRATION (1 << 1)206#define PVMW_MIGRATION (1 << 1)
205 207
206struct page_vma_mapped_walk {208struct page_vma_mapped_walk {
207 struct page *page;209 unsigned long pfn;
210 unsigned long nr_pages;
211 pgoff_t pgoff;
208 struct vm_area_struct *vma;212 struct vm_area_struct *vma;
209 unsigned long address;213 unsigned long address;
210 pmd_t *pmd;214 pmd_t *pmd;
@@ -213,10 +217,30 @@ struct page_vma_mapped_walk { · 2 unmodified lines
213 unsigned int flags;217 unsigned int flags;
214};218};
215 219
220#define DEFINE_PAGE_VMA_WALK(name, _page, _vma, _address, _flags) \
221 struct page_vma_mapped_walk name = { \
222 .pfn = page_to_pfn(_page), \
223 .nr_pages = compound_nr(page), \
224 .pgoff = page_to_pgoff(page), \
225 .vma = _vma, \
226 .address = _address, \
227 .flags = _flags, \
228 }
229
230#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \
231 struct page_vma_mapped_walk name = { \
232 .pfn = folio_pfn(_folio), \
233 .nr_pages = folio_nr_pages(_folio), \
234 .pgoff = folio_pgoff(_folio), \
235 .vma = _vma, \
236 .address = _address, \
237 .flags = _flags, \
238 }
239
216static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)240static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
217{241{
218 /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */242 /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
219 if (pvmw->pte && !PageHuge(pvmw->page))243 if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma))
220 pte_unmap(pvmw->pte);244 pte_unmap(pvmw->pte);
221 if (pvmw->ptl)245 if (pvmw->ptl)
222 spin_unlock(pvmw->ptl);246 spin_unlock(pvmw->ptl);
@@ -237,18 +261,12 @@ unsigned long page_address_in_vma(struct page *, struct vm_area_struct *); · 14 unmodified lines
237 */261 */
238int folio_mkclean(struct folio *);262int folio_mkclean(struct folio *);
239 263
240/*264void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked);
241 * called in munlock()/munmap() path to check for other vmas holding
242 * the page mlocked.
243 */
244void page_mlock(struct page *page);
245
246void remove_migration_ptes(struct page *old, struct page *new, bool locked);
247 265
248/*266/*
249 * Called by memory-failure.c to kill processes.267 * Called by memory-failure.c to kill processes.
250 */268 */
251struct anon_vma *page_lock_anon_vma_read(struct page *page);269struct anon_vma *folio_lock_anon_vma_read(struct folio *folio);
252void page_unlock_anon_vma_read(struct anon_vma *anon_vma);270void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
253int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);271int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);
254 272
@@ -267,15 +285,15 @@ struct rmap_walk_control { · 12 unmodified lines
267 * Return false if page table scanning in rmap_walk should be stopped.285 * Return false if page table scanning in rmap_walk should be stopped.
268 * Otherwise, return true.286 * Otherwise, return true.
269 */287 */
270 bool (*rmap_one)(struct page *page, struct vm_area_struct *vma,288 bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma,
271 unsigned long addr, void *arg);289 unsigned long addr, void *arg);
272 int (*done)(struct page *page);290 int (*done)(struct folio *folio);
273 struct anon_vma *(*anon_lock)(struct page *page);291 struct anon_vma *(*anon_lock)(struct folio *folio);
274 bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);292 bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);
275};293};
276 294
277void rmap_walk(struct page *page, struct rmap_walk_control *rwc);295void rmap_walk(struct folio *folio, const struct rmap_walk_control *rwc);
278void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc);296void rmap_walk_locked(struct folio *folio, const struct rmap_walk_control *rwc);
279 297
280#else /* !CONFIG_MMU */298#else /* !CONFIG_MMU */
281 299
@@ -283,7 +301,7 @@ void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc); · 1 unmodified line
283#define anon_vma_prepare(vma) (0)301#define anon_vma_prepare(vma) (0)
284#define anon_vma_link(vma) do {} while (0)302#define anon_vma_link(vma) do {} while (0)
285 303
286static inline int page_referenced(struct page *page, int is_locked,304static inline int folio_referenced(struct folio *folio, int is_locked,
287 struct mem_cgroup *memcg,305 struct mem_cgroup *memcg,
288 unsigned long *vm_flags)306 unsigned long *vm_flags)
289{307{
@@ -291,7 +309,7 @@ static inline int page_referenced(struct page *page, int is_locked, · 1 unmodified line
291 return 0;309 return 0;
292}310}
293 311
294static inline void try_to_unmap(struct page *page, enum ttu_flags flags)312static inline void try_to_unmap(struct folio *folio, enum ttu_flags flags)
295{313{
296}314}
297 315
include/linux/swap.h+4−5
@@ -328,7 +328,7 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio) · 327 unmodified lines
328 328
329/* linux/mm/workingset.c */329/* linux/mm/workingset.c */
330void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);330void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
331void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg);331void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg);
332void workingset_refault(struct folio *folio, void *shadow);332void workingset_refault(struct folio *folio, void *shadow);
333void workingset_activation(struct folio *folio);333void workingset_activation(struct folio *folio);
334 334
@@ -375,7 +375,6 @@ extern void lru_add_drain(void); · 40 unmodified lines
375extern void lru_add_drain_cpu(int cpu);375extern void lru_add_drain_cpu(int cpu);
376extern void lru_add_drain_cpu_zone(struct zone *zone);376extern void lru_add_drain_cpu_zone(struct zone *zone);
377extern void lru_add_drain_all(void);377extern void lru_add_drain_all(void);
378extern void deactivate_file_page(struct page *page);
379extern void deactivate_page(struct page *page);378extern void deactivate_page(struct page *page);
380extern void mark_page_lazyfree(struct page *page);379extern void mark_page_lazyfree(struct page *page);
381extern void swap_setup(void);380extern void swap_setup(void);
@@ -397,7 +396,7 @@ extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, · 15 unmodified lines
397 unsigned long *nr_scanned);396 unsigned long *nr_scanned);
398extern unsigned long shrink_all_memory(unsigned long nr_pages);397extern unsigned long shrink_all_memory(unsigned long nr_pages);
399extern int vm_swappiness;398extern int vm_swappiness;
400extern int remove_mapping(struct address_space *mapping, struct page *page);399long remove_mapping(struct address_space *mapping, struct folio *folio);
401 400
402extern unsigned long reclaim_pages(struct list_head *page_list);401extern unsigned long reclaim_pages(struct list_head *page_list);
403#ifdef CONFIG_NUMA402#ifdef CONFIG_NUMA
@@ -743,7 +742,7 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · 339 unmodified lines
743#endif742#endif
744 743
745#ifdef CONFIG_MEMCG_SWAP744#ifdef CONFIG_MEMCG_SWAP
746extern void mem_cgroup_swapout(struct page *page, swp_entry_t entry);745void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry);
747extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);746extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
748static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)747static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
749{748{
@@ -763,7 +762,7 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_p · 13 unmodified lines
763extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);762extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
764extern bool mem_cgroup_swap_full(struct page *page);763extern bool mem_cgroup_swap_full(struct page *page);
765#else764#else
766static inline void mem_cgroup_swapout(struct page *page, swp_entry_t entry)765static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
767{766{
768}767}
769 768
include/trace/events/vmscan.h+5−5
@@ -327,11 +327,11 @@ TRACE_EVENT(mm_vmscan_lru_isolate, · 326 unmodified lines
327 __print_symbolic(__entry->lru, LRU_NAMES))327 __print_symbolic(__entry->lru, LRU_NAMES))
328);328);
329 329
330TRACE_EVENT(mm_vmscan_writepage,330TRACE_EVENT(mm_vmscan_write_folio,
331 331
332 TP_PROTO(struct page *page),332 TP_PROTO(struct folio *folio),
333 333
334 TP_ARGS(page),334 TP_ARGS(folio),
335 335
336 TP_STRUCT__entry(336 TP_STRUCT__entry(
337 __field(unsigned long, pfn)337 __field(unsigned long, pfn)
@@ -339,9 +339,9 @@ TRACE_EVENT(mm_vmscan_writepage, · 1 unmodified line
339 ),339 ),
340 340
341 TP_fast_assign(341 TP_fast_assign(
342 __entry->pfn = page_to_pfn(page);342 __entry->pfn = folio_pfn(folio);
343 __entry->reclaim_flags = trace_reclaim_flags(343 __entry->reclaim_flags = trace_reclaim_flags(
344 page_is_file_lru(page));344 folio_is_file_lru(folio));
345 ),345 ),
346 346
347 TP_printk("page=%p pfn=0x%lx flags=%s",347 TP_printk("page=%p pfn=0x%lx flags=%s",

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t