History

mm: introduce memdesc_flags_t

Patch series "Add and use memdesc_flags_t". At some point struct page will be separated from struct slab and struct folio. This is a step towards that by introducing a type for the 'flags' word of all three structures. This gives us a certain amount of type safety by establishing that some of these unsigned longs are different from other unsigned longs in that they contain things like node ID, section number and zone number in the upper bits. That lets us have functions that can be easily called by anyone who has a slab, folio or page (but not easily by anyone else) to get the node or zone. There's going to be some unusual merge problems with this as some odd bits of the kernel decide they want to print out the flags value or something similar by writing page->flags and now they'll need to write page->flags.f instead. That's most of the churn here. Maybe we should be removing these things from the debug output? This patch (of 11): Wrap the unsigned long flags in a typedef. In upcoming patches, this will provide a strong hint that you can't just pass a random unsigned long to functions which take this as an argument. [willy@infradead.org: s/flags/flags.f/ in several architectures] Link: https://lkml.kernel.org/r/aKMgPRLD-WnkPxYm@casper.infradead.org [nicola.vetrini@gmail.com: mips: fix compilation error] Link: https://lore.kernel.org/lkml/CA+G9fYvkpmqGr6wjBNHY=dRp71PLCoi2341JxOudi60yqaeUdg@mail.gmail.com/ Link: https://lkml.kernel.org/r/20250825214245.1838158-1-nicola.vetrini@gmail.com Link: https://lkml.kernel.org/r/20250805172307.1302730-1-willy@infradead.org Link: https://lkml.kernel.org/r/20250805172307.1302730-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org> Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org> Acked-by: Zi Yan <ziy@nvidia.com> Cc: Shakeel Butt <shakeel.butt@linux.dev> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Matthew Wilcox (Oracle) committed 1 year ago

Commit 53fbef51 parent 4e9156558 files · +195 −190Browse files

Unified · Split

58 files
  1. arch/arc/mm/cache.c +4 −4
  2. arch/arc/mm/tlb.c +1 −1
  3. arch/arm/include/asm/hugetlb.h +1 −1
  4. arch/arm/mm/copypage-v4mc.c +1 −1
  5. arch/arm/mm/copypage-v6.c +1 −1
  6. arch/arm/mm/copypage-xscale.c +1 −1
  7. arch/arm/mm/dma-mapping.c +1 −1
  8. arch/arm/mm/fault-armv.c +1 −1
  9. arch/arm/mm/flush.c +5 −5
  10. arch/arm64/include/asm/hugetlb.h +3 −3
  11. arch/arm64/include/asm/mte.h +8 −8
  12. arch/arm64/mm/flush.c +4 −4
  13. arch/csky/abiv1/cacheflush.c +3 −3
  14. arch/mips/include/asm/cacheflush.h +3 −3
  15. arch/nios2/mm/cacheflush.c +3 −3
  16. arch/openrisc/include/asm/cacheflush.h +1 −1
  17. arch/openrisc/mm/cache.c +1 −1
  18. arch/parisc/kernel/cache.c +3 −3
  19. arch/powerpc/include/asm/cacheflush.h +2 −2
  20. arch/powerpc/include/asm/kvm_ppc.h +2 −2
  21. arch/powerpc/mm/book3s64/hash_utils.c +2 −2
  22. arch/powerpc/mm/pgtable.c +6 −6
  23. arch/riscv/include/asm/cacheflush.h +2 −2
  24. arch/riscv/include/asm/hugetlb.h +1 −1
  25. arch/riscv/mm/cacheflush.c +2 −2
  26. arch/s390/include/asm/hugetlb.h +1 −1
  27. arch/s390/kernel/uv.c +6 −6
  28. arch/s390/mm/gmap.c +1 −1
  29. arch/s390/mm/hugetlbpage.c +1 −1
  30. arch/sh/include/asm/hugetlb.h +1 −1
  31. arch/sh/mm/cache-sh4.c +1 −1
  32. arch/sh/mm/cache-sh7705.c +1 −1
  33. arch/sh/mm/cache.c +7 −7
  34. arch/sh/mm/kmap.c +1 −1
  35. arch/sparc/mm/init_64.c +5 −5
  36. arch/x86/mm/pat/memtype.c +3 −3
  37. arch/xtensa/mm/cache.c +6 −6
  38. fs/fuse/dev.c +1 −1
  39. fs/gfs2/glops.c +1 −1
  40. fs/jffs2/file.c +2 −2
  41. fs/nilfs2/page.c +1 −1
  42. fs/proc/page.c +2 −2
  43. fs/ubifs/file.c +3 −3
  44. include/linux/mm.h +16 −16
  45. include/linux/mm_inline.h +6 −6
  46. include/linux/mm_types.h +6 −2
  47. include/linux/mmzone.h +1 −1
  48. include/linux/page-flags.h +20 −20
  49. include/linux/pgalloc_tag.h +4 −3
  50. include/trace/events/page_ref.h +2 −2
  51. mm/filemap.c +4 −4
  52. mm/huge_memory.c +2 −2
  53. mm/memory-failure.c +6 −6
  54. mm/mmzone.c +2 −2
  55. mm/page_alloc.c +6 −6
  56. mm/swap.c +4 −4
  57. mm/vmscan.c +9 −9
  58. mm/workingset.c +1 −1
arch/arc/mm/cache.c+4−4
@@ -704,7 +704,7 @@ static inline void arc_slc_enable(void) · 703 unmodified lines
704 704
705void flush_dcache_folio(struct folio *folio)705void flush_dcache_folio(struct folio *folio)
706{706{
707 clear_bit(PG_dc_clean, &folio->flags);707 clear_bit(PG_dc_clean, &folio->flags.f);
708 return;708 return;
709}709}
710EXPORT_SYMBOL(flush_dcache_folio);710EXPORT_SYMBOL(flush_dcache_folio);
@@ -889,8 +889,8 @@ void copy_user_highpage(struct page *to, struct page *from, · 178 unmodified lines
889 889
890 copy_page(kto, kfrom);890 copy_page(kto, kfrom);
891 891
892 clear_bit(PG_dc_clean, &dst->flags);892 clear_bit(PG_dc_clean, &dst->flags.f);
893 clear_bit(PG_dc_clean, &src->flags);893 clear_bit(PG_dc_clean, &src->flags.f);
894 894
895 kunmap_atomic(kto);895 kunmap_atomic(kto);
896 kunmap_atomic(kfrom);896 kunmap_atomic(kfrom);
@@ -900,7 +900,7 @@ void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) · 3 unmodified lines
900{900{
901 struct folio *folio = page_folio(page);901 struct folio *folio = page_folio(page);
902 clear_page(to);902 clear_page(to);
903 clear_bit(PG_dc_clean, &folio->flags);903 clear_bit(PG_dc_clean, &folio->flags.f);
904}904}
905EXPORT_SYMBOL(clear_user_page);905EXPORT_SYMBOL(clear_user_page);
906 906
arch/arc/mm/tlb.c+1−1
@@ -488,7 +488,7 @@ void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, · 487 unmodified lines
488 */488 */
489 if (vma->vm_flags & VM_EXEC) {489 if (vma->vm_flags & VM_EXEC) {
490 struct folio *folio = page_folio(page);490 struct folio *folio = page_folio(page);
491 int dirty = !test_and_set_bit(PG_dc_clean, &folio->flags);491 int dirty = !test_and_set_bit(PG_dc_clean, &folio->flags.f);
492 if (dirty) {492 if (dirty) {
493 unsigned long offset = offset_in_folio(folio, paddr);493 unsigned long offset = offset_in_folio(folio, paddr);
494 nr = folio_nr_pages(folio);494 nr = folio_nr_pages(folio);
arch/arm/include/asm/hugetlb.h+1−1
@@ -17,7 +17,7 @@ · 16 unmodified lines
17 17
18static inline void arch_clear_hugetlb_flags(struct folio *folio)18static inline void arch_clear_hugetlb_flags(struct folio *folio)
19{19{
20 clear_bit(PG_dcache_clean, &folio->flags);20 clear_bit(PG_dcache_clean, &folio->flags.f);
21}21}
22#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags22#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
23 23
arch/arm/mm/copypage-v4mc.c+1−1
@@ -67,7 +67,7 @@ void v4_mc_copy_user_highpage(struct page *to, struct page *from, · 66 unmodified lines
67 struct folio *src = page_folio(from);67 struct folio *src = page_folio(from);
68 void *kto = kmap_atomic(to);68 void *kto = kmap_atomic(to);
69 69
70 if (!test_and_set_bit(PG_dcache_clean, &src->flags))70 if (!test_and_set_bit(PG_dcache_clean, &src->flags.f))
71 __flush_dcache_folio(folio_flush_mapping(src), src);71 __flush_dcache_folio(folio_flush_mapping(src), src);
72 72
73 raw_spin_lock(&minicache_lock);73 raw_spin_lock(&minicache_lock);
arch/arm/mm/copypage-v6.c+1−1
@@ -73,7 +73,7 @@ static void v6_copy_user_highpage_aliasing(struct page *to, · 72 unmodified lines
73 unsigned int offset = CACHE_COLOUR(vaddr);73 unsigned int offset = CACHE_COLOUR(vaddr);
74 unsigned long kfrom, kto;74 unsigned long kfrom, kto;
75 75
76 if (!test_and_set_bit(PG_dcache_clean, &src->flags))76 if (!test_and_set_bit(PG_dcache_clean, &src->flags.f))
77 __flush_dcache_folio(folio_flush_mapping(src), src);77 __flush_dcache_folio(folio_flush_mapping(src), src);
78 78
79 /* FIXME: not highmem safe */79 /* FIXME: not highmem safe */
arch/arm/mm/copypage-xscale.c+1−1
@@ -87,7 +87,7 @@ void xscale_mc_copy_user_highpage(struct page *to, struct page *from, · 86 unmodified lines
87 struct folio *src = page_folio(from);87 struct folio *src = page_folio(from);
88 void *kto = kmap_atomic(to);88 void *kto = kmap_atomic(to);
89 89
90 if (!test_and_set_bit(PG_dcache_clean, &src->flags))90 if (!test_and_set_bit(PG_dcache_clean, &src->flags.f))
91 __flush_dcache_folio(folio_flush_mapping(src), src);91 __flush_dcache_folio(folio_flush_mapping(src), src);
92 92
93 raw_spin_lock(&minicache_lock);93 raw_spin_lock(&minicache_lock);
arch/arm/mm/dma-mapping.c+1−1
@@ -718,7 +718,7 @@ static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, · 717 unmodified lines
718 if (size < sz)718 if (size < sz)
719 break;719 break;
720 if (!offset)720 if (!offset)
721 set_bit(PG_dcache_clean, &folio->flags);721 set_bit(PG_dcache_clean, &folio->flags.f);
722 offset = 0;722 offset = 0;
723 size -= sz;723 size -= sz;
724 if (!size)724 if (!size)
arch/arm/mm/fault-armv.c+1−1
@@ -203,7 +203,7 @@ void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, · 202 unmodified lines
203 203
204 folio = page_folio(pfn_to_page(pfn));204 folio = page_folio(pfn_to_page(pfn));
205 mapping = folio_flush_mapping(folio);205 mapping = folio_flush_mapping(folio);
206 if (!test_and_set_bit(PG_dcache_clean, &folio->flags))206 if (!test_and_set_bit(PG_dcache_clean, &folio->flags.f))
207 __flush_dcache_folio(mapping, folio);207 __flush_dcache_folio(mapping, folio);
208 if (mapping) {208 if (mapping) {
209 if (cache_is_vivt())209 if (cache_is_vivt())
arch/arm/mm/flush.c+5−5
@@ -304,7 +304,7 @@ void __sync_icache_dcache(pte_t pteval) · 303 unmodified lines
304 else304 else
305 mapping = NULL;305 mapping = NULL;
306 306
307 if (!test_and_set_bit(PG_dcache_clean, &folio->flags))307 if (!test_and_set_bit(PG_dcache_clean, &folio->flags.f))
308 __flush_dcache_folio(mapping, folio);308 __flush_dcache_folio(mapping, folio);
309 309
310 if (pte_exec(pteval))310 if (pte_exec(pteval))
@@ -343,8 +343,8 @@ void flush_dcache_folio(struct folio *folio) · 32 unmodified lines
343 return;343 return;
344 344
345 if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) {345 if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) {
346 if (test_bit(PG_dcache_clean, &folio->flags))346 if (test_bit(PG_dcache_clean, &folio->flags.f))
347 clear_bit(PG_dcache_clean, &folio->flags);347 clear_bit(PG_dcache_clean, &folio->flags.f);
348 return;348 return;
349 }349 }
350 350
@@ -352,14 +352,14 @@ void flush_dcache_folio(struct folio *folio) · 1 unmodified line
352 352
353 if (!cache_ops_need_broadcast() &&353 if (!cache_ops_need_broadcast() &&
354 mapping && !folio_mapped(folio))354 mapping && !folio_mapped(folio))
355 clear_bit(PG_dcache_clean, &folio->flags);355 clear_bit(PG_dcache_clean, &folio->flags.f);
356 else {356 else {
357 __flush_dcache_folio(mapping, folio);357 __flush_dcache_folio(mapping, folio);
358 if (mapping && cache_is_vivt())358 if (mapping && cache_is_vivt())
359 __flush_dcache_aliases(mapping, folio);359 __flush_dcache_aliases(mapping, folio);
360 else if (mapping)360 else if (mapping)
361 __flush_icache_all();361 __flush_icache_all();
362 set_bit(PG_dcache_clean, &folio->flags);362 set_bit(PG_dcache_clean, &folio->flags.f);
363 }363 }
364}364}
365EXPORT_SYMBOL(flush_dcache_folio);365EXPORT_SYMBOL(flush_dcache_folio);
arch/arm64/include/asm/hugetlb.h+3−3
@@ -21,12 +21,12 @@ extern bool arch_hugetlb_migration_supported(struct hstate *h); · 20 unmodified lines
21 21
22static inline void arch_clear_hugetlb_flags(struct folio *folio)22static inline void arch_clear_hugetlb_flags(struct folio *folio)
23{23{
24 clear_bit(PG_dcache_clean, &folio->flags);24 clear_bit(PG_dcache_clean, &folio->flags.f);
25 25
26#ifdef CONFIG_ARM64_MTE26#ifdef CONFIG_ARM64_MTE
27 if (system_supports_mte()) {27 if (system_supports_mte()) {
28 clear_bit(PG_mte_tagged, &folio->flags);28 clear_bit(PG_mte_tagged, &folio->flags.f);
29 clear_bit(PG_mte_lock, &folio->flags);29 clear_bit(PG_mte_lock, &folio->flags.f);
30 }30 }
31#endif31#endif
32}32}
arch/arm64/include/asm/mte.h+8−8
@@ -48,12 +48,12 @@ static inline void set_page_mte_tagged(struct page *page) · 47 unmodified lines
48 * before the page flags update.48 * before the page flags update.
49 */49 */
50 smp_wmb();50 smp_wmb();
51 set_bit(PG_mte_tagged, &page->flags);51 set_bit(PG_mte_tagged, &page->flags.f);
52}52}
53 53
54static inline bool page_mte_tagged(struct page *page)54static inline bool page_mte_tagged(struct page *page)
55{55{
56 bool ret = test_bit(PG_mte_tagged, &page->flags);56 bool ret = test_bit(PG_mte_tagged, &page->flags.f);
57 57
58 VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));58 VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));
59 59
@@ -82,7 +82,7 @@ static inline bool try_page_mte_tagging(struct page *page) · 22 unmodified lines
82{82{
83 VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));83 VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));
84 84
85 if (!test_and_set_bit(PG_mte_lock, &page->flags))85 if (!test_and_set_bit(PG_mte_lock, &page->flags.f))
86 return true;86 return true;
87 87
88 /*88 /*
@@ -90,7 +90,7 @@ static inline bool try_page_mte_tagging(struct page *page) · 1 unmodified line
90 * already. Check if the PG_mte_tagged flag has been set or wait90 * already. Check if the PG_mte_tagged flag has been set or wait
91 * otherwise.91 * otherwise.
92 */92 */
93 smp_cond_load_acquire(&page->flags, VAL & (1UL << PG_mte_tagged));93 smp_cond_load_acquire(&page->flags.f, VAL & (1UL << PG_mte_tagged));
94 94
95 return false;95 return false;
96}96}
@@ -173,13 +173,13 @@ static inline void folio_set_hugetlb_mte_tagged(struct folio *folio) · 76 unmodified lines
173 * before the folio flags update.173 * before the folio flags update.
174 */174 */
175 smp_wmb();175 smp_wmb();
176 set_bit(PG_mte_tagged, &folio->flags);176 set_bit(PG_mte_tagged, &folio->flags.f);
177 177
178}178}
179 179
180static inline bool folio_test_hugetlb_mte_tagged(struct folio *folio)180static inline bool folio_test_hugetlb_mte_tagged(struct folio *folio)
181{181{
182 bool ret = test_bit(PG_mte_tagged, &folio->flags);182 bool ret = test_bit(PG_mte_tagged, &folio->flags.f);
183 183
184 VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));184 VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));
185 185
@@ -196,7 +196,7 @@ static inline bool folio_try_hugetlb_mte_tagging(struct folio *folio) · 10 unmodified lines
196{196{
197 VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));197 VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));
198 198
199 if (!test_and_set_bit(PG_mte_lock, &folio->flags))199 if (!test_and_set_bit(PG_mte_lock, &folio->flags.f))
200 return true;200 return true;
201 201
202 /*202 /*
@@ -204,7 +204,7 @@ static inline bool folio_try_hugetlb_mte_tagging(struct folio *folio) · 1 unmodified line
204 * already. Check if the PG_mte_tagged flag has been set or wait204 * already. Check if the PG_mte_tagged flag has been set or wait
205 * otherwise.205 * otherwise.
206 */206 */
207 smp_cond_load_acquire(&folio->flags, VAL & (1UL << PG_mte_tagged));207 smp_cond_load_acquire(&folio->flags.f, VAL & (1UL << PG_mte_tagged));
208 208
209 return false;209 return false;
210}210}
arch/arm64/mm/flush.c+4−4
@@ -53,11 +53,11 @@ void __sync_icache_dcache(pte_t pte) · 52 unmodified lines
53{53{
54 struct folio *folio = page_folio(pte_page(pte));54 struct folio *folio = page_folio(pte_page(pte));
55 55
56 if (!test_bit(PG_dcache_clean, &folio->flags)) {56 if (!test_bit(PG_dcache_clean, &folio->flags.f)) {
57 sync_icache_aliases((unsigned long)folio_address(folio),57 sync_icache_aliases((unsigned long)folio_address(folio),
58 (unsigned long)folio_address(folio) +58 (unsigned long)folio_address(folio) +
59 folio_size(folio));59 folio_size(folio));
60 set_bit(PG_dcache_clean, &folio->flags);60 set_bit(PG_dcache_clean, &folio->flags.f);
61 }61 }
62}62}
63EXPORT_SYMBOL_GPL(__sync_icache_dcache);63EXPORT_SYMBOL_GPL(__sync_icache_dcache);
@@ -69,8 +69,8 @@ EXPORT_SYMBOL_GPL(__sync_icache_dcache); · 5 unmodified lines
69 */69 */
70void flush_dcache_folio(struct folio *folio)70void flush_dcache_folio(struct folio *folio)
71{71{
72 if (test_bit(PG_dcache_clean, &folio->flags))72 if (test_bit(PG_dcache_clean, &folio->flags.f))
73 clear_bit(PG_dcache_clean, &folio->flags);73 clear_bit(PG_dcache_clean, &folio->flags.f);
74}74}
75EXPORT_SYMBOL(flush_dcache_folio);75EXPORT_SYMBOL(flush_dcache_folio);
76 76
arch/csky/abiv1/cacheflush.c+3−3
@@ -25,12 +25,12 @@ void flush_dcache_folio(struct folio *folio) · 24 unmodified lines
25 mapping = folio_flush_mapping(folio);25 mapping = folio_flush_mapping(folio);
26 26
27 if (mapping && !folio_mapped(folio))27 if (mapping && !folio_mapped(folio))
28 clear_bit(PG_dcache_clean, &folio->flags);28 clear_bit(PG_dcache_clean, &folio->flags.f);
29 else {29 else {
30 dcache_wbinv_all();30 dcache_wbinv_all();
31 if (mapping)31 if (mapping)
32 icache_inv_all();32 icache_inv_all();
33 set_bit(PG_dcache_clean, &folio->flags);33 set_bit(PG_dcache_clean, &folio->flags.f);
34 }34 }
35}35}
36EXPORT_SYMBOL(flush_dcache_folio);36EXPORT_SYMBOL(flush_dcache_folio);
@@ -56,7 +56,7 @@ void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, · 19 unmodified lines
56 return;56 return;
57 57
58 folio = page_folio(pfn_to_page(pfn));58 folio = page_folio(pfn_to_page(pfn));
59 if (!test_and_set_bit(PG_dcache_clean, &folio->flags))59 if (!test_and_set_bit(PG_dcache_clean, &folio->flags.f))
60 dcache_wbinv_all();60 dcache_wbinv_all();
61 61
62 if (folio_flush_mapping(folio)) {62 if (folio_flush_mapping(folio)) {
arch/mips/include/asm/cacheflush.h+3−3
@@ -37,11 +37,11 @@ · 36 unmodified lines
37#define PG_dcache_dirty PG_arch_137#define PG_dcache_dirty PG_arch_1
38 38
39#define folio_test_dcache_dirty(folio) \39#define folio_test_dcache_dirty(folio) \
40 test_bit(PG_dcache_dirty, &(folio)->flags)40 test_bit(PG_dcache_dirty, &(folio)->flags.f)
41#define folio_set_dcache_dirty(folio) \41#define folio_set_dcache_dirty(folio) \
42 set_bit(PG_dcache_dirty, &(folio)->flags)42 set_bit(PG_dcache_dirty, &(folio)->flags.f)
43#define folio_clear_dcache_dirty(folio) \43#define folio_clear_dcache_dirty(folio) \
44 clear_bit(PG_dcache_dirty, &(folio)->flags)44 clear_bit(PG_dcache_dirty, &(folio)->flags.f)
45 45
46extern void (*flush_cache_all)(void);46extern void (*flush_cache_all)(void);
47extern void (*__flush_cache_all)(void);47extern void (*__flush_cache_all)(void);
arch/nios2/mm/cacheflush.c+3−3
@@ -187,7 +187,7 @@ void flush_dcache_folio(struct folio *folio) · 186 unmodified lines
187 187
188 /* Flush this page if there are aliases. */188 /* Flush this page if there are aliases. */
189 if (mapping && !mapping_mapped(mapping)) {189 if (mapping && !mapping_mapped(mapping)) {
190 clear_bit(PG_dcache_clean, &folio->flags);190 clear_bit(PG_dcache_clean, &folio->flags.f);
191 } else {191 } else {
192 __flush_dcache_folio(folio);192 __flush_dcache_folio(folio);
193 if (mapping) {193 if (mapping) {
@@ -195,7 +195,7 @@ void flush_dcache_folio(struct folio *folio) · 1 unmodified line
195 flush_aliases(mapping, folio);195 flush_aliases(mapping, folio);
196 flush_icache_range(start, start + folio_size(folio));196 flush_icache_range(start, start + folio_size(folio));
197 }197 }
198 set_bit(PG_dcache_clean, &folio->flags);198 set_bit(PG_dcache_clean, &folio->flags.f);
199 }199 }
200}200}
201EXPORT_SYMBOL(flush_dcache_folio);201EXPORT_SYMBOL(flush_dcache_folio);
@@ -227,7 +227,7 @@ void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, · 25 unmodified lines
227 return;227 return;
228 228
229 folio = page_folio(pfn_to_page(pfn));229 folio = page_folio(pfn_to_page(pfn));
230 if (!test_and_set_bit(PG_dcache_clean, &folio->flags))230 if (!test_and_set_bit(PG_dcache_clean, &folio->flags.f))
231 __flush_dcache_folio(folio);231 __flush_dcache_folio(folio);
232 232
233 mapping = folio_flush_mapping(folio);233 mapping = folio_flush_mapping(folio);
arch/openrisc/include/asm/cacheflush.h+1−1
@@ -75,7 +75,7 @@ static inline void sync_icache_dcache(struct page *page) · 74 unmodified lines
75 75
76static inline void flush_dcache_folio(struct folio *folio)76static inline void flush_dcache_folio(struct folio *folio)
77{77{
78 clear_bit(PG_dc_clean, &folio->flags);78 clear_bit(PG_dc_clean, &folio->flags.f);
79}79}
80#define flush_dcache_folio flush_dcache_folio80#define flush_dcache_folio flush_dcache_folio
81 81
arch/openrisc/mm/cache.c+1−1
@@ -83,7 +83,7 @@ void update_cache(struct vm_area_struct *vma, unsigned long address, · 82 unmodified lines
83{83{
84 unsigned long pfn = pte_val(*pte) >> PAGE_SHIFT;84 unsigned long pfn = pte_val(*pte) >> PAGE_SHIFT;
85 struct folio *folio = page_folio(pfn_to_page(pfn));85 struct folio *folio = page_folio(pfn_to_page(pfn));
86 int dirty = !test_and_set_bit(PG_dc_clean, &folio->flags);86 int dirty = !test_and_set_bit(PG_dc_clean, &folio->flags.f);
87 87
88 /*88 /*
89 * Since icaches do not snoop for updated data on OpenRISC, we89 * Since icaches do not snoop for updated data on OpenRISC, we
arch/parisc/kernel/cache.c+3−3
@@ -122,10 +122,10 @@ void __update_cache(pte_t pte) · 121 unmodified lines
122 pfn = folio_pfn(folio);122 pfn = folio_pfn(folio);
123 nr = folio_nr_pages(folio);123 nr = folio_nr_pages(folio);
124 if (folio_flush_mapping(folio) &&124 if (folio_flush_mapping(folio) &&
125 test_bit(PG_dcache_dirty, &folio->flags)) {125 test_bit(PG_dcache_dirty, &folio->flags.f)) {
126 while (nr--)126 while (nr--)
127 flush_kernel_dcache_page_addr(pfn_va(pfn + nr));127 flush_kernel_dcache_page_addr(pfn_va(pfn + nr));
128 clear_bit(PG_dcache_dirty, &folio->flags);128 clear_bit(PG_dcache_dirty, &folio->flags.f);
129 } else if (parisc_requires_coherency())129 } else if (parisc_requires_coherency())
130 while (nr--)130 while (nr--)
131 flush_kernel_dcache_page_addr(pfn_va(pfn + nr));131 flush_kernel_dcache_page_addr(pfn_va(pfn + nr));
@@ -481,7 +481,7 @@ void flush_dcache_folio(struct folio *folio) · 349 unmodified lines
481 pgoff_t pgoff;481 pgoff_t pgoff;
482 482
483 if (mapping && !mapping_mapped(mapping)) {483 if (mapping && !mapping_mapped(mapping)) {
484 set_bit(PG_dcache_dirty, &folio->flags);484 set_bit(PG_dcache_dirty, &folio->flags.f);
485 return;485 return;
486 }486 }
487 487
arch/powerpc/include/asm/cacheflush.h+2−2
@@ -40,8 +40,8 @@ static inline void flush_dcache_folio(struct folio *folio) · 39 unmodified lines
40 if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))40 if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
41 return;41 return;
42 /* avoid an atomic op if possible */42 /* avoid an atomic op if possible */
43 if (test_bit(PG_dcache_clean, &folio->flags))43 if (test_bit(PG_dcache_clean, &folio->flags.f))
44 clear_bit(PG_dcache_clean, &folio->flags);44 clear_bit(PG_dcache_clean, &folio->flags.f);
45}45}
46#define flush_dcache_folio flush_dcache_folio46#define flush_dcache_folio flush_dcache_folio
47 47
arch/powerpc/include/asm/kvm_ppc.h+2−2
@@ -939,9 +939,9 @@ static inline void kvmppc_mmu_flush_icache(kvm_pfn_t pfn) · 938 unmodified lines
939 939
940 /* Clear i-cache for new pages */940 /* Clear i-cache for new pages */
941 folio = page_folio(pfn_to_page(pfn));941 folio = page_folio(pfn_to_page(pfn));
942 if (!test_bit(PG_dcache_clean, &folio->flags)) {942 if (!test_bit(PG_dcache_clean, &folio->flags.f)) {
943 flush_dcache_icache_folio(folio);943 flush_dcache_icache_folio(folio);
944 set_bit(PG_dcache_clean, &folio->flags);944 set_bit(PG_dcache_clean, &folio->flags.f);
945 }945 }
946}946}
947 947
arch/powerpc/mm/book3s64/hash_utils.c+2−2
@@ -1562,11 +1562,11 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap) · 1561 unmodified lines
1562 folio = page_folio(pte_page(pte));1562 folio = page_folio(pte_page(pte));
1563 1563
1564 /* page is dirty */1564 /* page is dirty */
1565 if (!test_bit(PG_dcache_clean, &folio->flags) &&1565 if (!test_bit(PG_dcache_clean, &folio->flags.f) &&
1566 !folio_test_reserved(folio)) {1566 !folio_test_reserved(folio)) {
1567 if (trap == INTERRUPT_INST_STORAGE) {1567 if (trap == INTERRUPT_INST_STORAGE) {
1568 flush_dcache_icache_folio(folio);1568 flush_dcache_icache_folio(folio);
1569 set_bit(PG_dcache_clean, &folio->flags);1569 set_bit(PG_dcache_clean, &folio->flags.f);
1570 } else1570 } else
1571 pp |= HPTE_R_N;1571 pp |= HPTE_R_N;
1572 }1572 }
arch/powerpc/mm/pgtable.c+6−6
@@ -87,9 +87,9 @@ static pte_t set_pte_filter_hash(pte_t pte, unsigned long addr) · 86 unmodified lines
87 struct folio *folio = maybe_pte_to_folio(pte);87 struct folio *folio = maybe_pte_to_folio(pte);
88 if (!folio)88 if (!folio)
89 return pte;89 return pte;
90 if (!test_bit(PG_dcache_clean, &folio->flags)) {90 if (!test_bit(PG_dcache_clean, &folio->flags.f)) {
91 flush_dcache_icache_folio(folio);91 flush_dcache_icache_folio(folio);
92 set_bit(PG_dcache_clean, &folio->flags);92 set_bit(PG_dcache_clean, &folio->flags.f);
93 }93 }
94 }94 }
95 return pte;95 return pte;
@@ -127,13 +127,13 @@ static inline pte_t set_pte_filter(pte_t pte, unsigned long addr) · 31 unmodified lines
127 return pte;127 return pte;
128 128
129 /* If the page clean, we move on */129 /* If the page clean, we move on */
130 if (test_bit(PG_dcache_clean, &folio->flags))130 if (test_bit(PG_dcache_clean, &folio->flags.f))
131 return pte;131 return pte;
132 132
133 /* If it's an exec fault, we flush the cache and make it clean */133 /* If it's an exec fault, we flush the cache and make it clean */
134 if (is_exec_fault()) {134 if (is_exec_fault()) {
135 flush_dcache_icache_folio(folio);135 flush_dcache_icache_folio(folio);
136 set_bit(PG_dcache_clean, &folio->flags);136 set_bit(PG_dcache_clean, &folio->flags.f);
137 return pte;137 return pte;
138 }138 }
139 139
@@ -175,12 +175,12 @@ static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma, · 35 unmodified lines
175 goto bail;175 goto bail;
176 176
177 /* If the page is already clean, we move on */177 /* If the page is already clean, we move on */
178 if (test_bit(PG_dcache_clean, &folio->flags))178 if (test_bit(PG_dcache_clean, &folio->flags.f))
179 goto bail;179 goto bail;
180 180
181 /* Clean the page and set PG_dcache_clean */181 /* Clean the page and set PG_dcache_clean */
182 flush_dcache_icache_folio(folio);182 flush_dcache_icache_folio(folio);
183 set_bit(PG_dcache_clean, &folio->flags);183 set_bit(PG_dcache_clean, &folio->flags.f);
184 184
185 bail:185 bail:
186 return pte_mkexec(pte);186 return pte_mkexec(pte);
arch/riscv/include/asm/cacheflush.h+2−2
@@ -23,8 +23,8 @@ static inline void local_flush_icache_range(unsigned long start, · 22 unmodified lines
23 23
24static inline void flush_dcache_folio(struct folio *folio)24static inline void flush_dcache_folio(struct folio *folio)
25{25{
26 if (test_bit(PG_dcache_clean, &folio->flags))26 if (test_bit(PG_dcache_clean, &folio->flags.f))
27 clear_bit(PG_dcache_clean, &folio->flags);27 clear_bit(PG_dcache_clean, &folio->flags.f);
28}28}
29#define flush_dcache_folio flush_dcache_folio29#define flush_dcache_folio flush_dcache_folio
30#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 130#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
arch/riscv/include/asm/hugetlb.h+1−1
@@ -7,7 +7,7 @@ · 6 unmodified lines
7 7
8static inline void arch_clear_hugetlb_flags(struct folio *folio)8static inline void arch_clear_hugetlb_flags(struct folio *folio)
9{9{
10 clear_bit(PG_dcache_clean, &folio->flags);10 clear_bit(PG_dcache_clean, &folio->flags.f);
11}11}
12#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags12#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
13 13
arch/riscv/mm/cacheflush.c+2−2
@@ -101,9 +101,9 @@ void flush_icache_pte(struct mm_struct *mm, pte_t pte) · 100 unmodified lines
101{101{
102 struct folio *folio = page_folio(pte_page(pte));102 struct folio *folio = page_folio(pte_page(pte));
103 103
104 if (!test_bit(PG_dcache_clean, &folio->flags)) {104 if (!test_bit(PG_dcache_clean, &folio->flags.f)) {
105 flush_icache_mm(mm, false);105 flush_icache_mm(mm, false);
106 set_bit(PG_dcache_clean, &folio->flags);106 set_bit(PG_dcache_clean, &folio->flags.f);
107 }107 }
108}108}
109#endif /* CONFIG_MMU */109#endif /* CONFIG_MMU */
arch/s390/include/asm/hugetlb.h+1−1
@@ -39,7 +39,7 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, · 38 unmodified lines
39 39
40static inline void arch_clear_hugetlb_flags(struct folio *folio)40static inline void arch_clear_hugetlb_flags(struct folio *folio)
41{41{
42 clear_bit(PG_arch_1, &folio->flags);42 clear_bit(PG_arch_1, &folio->flags.f);
43}43}
44#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags44#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
45 45
arch/s390/kernel/uv.c+6−6
@@ -144,7 +144,7 @@ int uv_destroy_folio(struct folio *folio) · 143 unmodified lines
144 folio_get(folio);144 folio_get(folio);
145 rc = uv_destroy(folio_to_phys(folio));145 rc = uv_destroy(folio_to_phys(folio));
146 if (!rc)146 if (!rc)
147 clear_bit(PG_arch_1, &folio->flags);147 clear_bit(PG_arch_1, &folio->flags.f);
148 folio_put(folio);148 folio_put(folio);
149 return rc;149 return rc;
150}150}
@@ -193,7 +193,7 @@ int uv_convert_from_secure_folio(struct folio *folio) · 42 unmodified lines
193 folio_get(folio);193 folio_get(folio);
194 rc = uv_convert_from_secure(folio_to_phys(folio));194 rc = uv_convert_from_secure(folio_to_phys(folio));
195 if (!rc)195 if (!rc)
196 clear_bit(PG_arch_1, &folio->flags);196 clear_bit(PG_arch_1, &folio->flags.f);
197 folio_put(folio);197 folio_put(folio);
198 return rc;198 return rc;
199}199}
@@ -289,7 +289,7 @@ static int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb) · 89 unmodified lines
289 expected = expected_folio_refs(folio) + 1;289 expected = expected_folio_refs(folio) + 1;
290 if (!folio_ref_freeze(folio, expected))290 if (!folio_ref_freeze(folio, expected))
291 return -EBUSY;291 return -EBUSY;
292 set_bit(PG_arch_1, &folio->flags);292 set_bit(PG_arch_1, &folio->flags.f);
293 /*293 /*
294 * If the UVC does not succeed or fail immediately, we don't want to294 * If the UVC does not succeed or fail immediately, we don't want to
295 * loop for long, or we might get stall notifications.295 * loop for long, or we might get stall notifications.
@@ -483,18 +483,18 @@ int arch_make_folio_accessible(struct folio *folio) · 187 unmodified lines
483 * convert_to_secure.483 * convert_to_secure.
484 * As secure pages are never large folios, both variants can co-exists.484 * As secure pages are never large folios, both variants can co-exists.
485 */485 */
486 if (!test_bit(PG_arch_1, &folio->flags))486 if (!test_bit(PG_arch_1, &folio->flags.f))
487 return 0;487 return 0;
488 488
489 rc = uv_pin_shared(folio_to_phys(folio));489 rc = uv_pin_shared(folio_to_phys(folio));
490 if (!rc) {490 if (!rc) {
491 clear_bit(PG_arch_1, &folio->flags);491 clear_bit(PG_arch_1, &folio->flags.f);
492 return 0;492 return 0;
493 }493 }
494 494
495 rc = uv_convert_from_secure(folio_to_phys(folio));495 rc = uv_convert_from_secure(folio_to_phys(folio));
496 if (!rc) {496 if (!rc) {
497 clear_bit(PG_arch_1, &folio->flags);497 clear_bit(PG_arch_1, &folio->flags.f);
498 return 0;498 return 0;
499 }499 }
500 500
arch/s390/mm/gmap.c+1−1
@@ -2272,7 +2272,7 @@ static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, · 2271 unmodified lines
2272 start = pmd_val(*pmd) & HPAGE_MASK;2272 start = pmd_val(*pmd) & HPAGE_MASK;
2273 end = start + HPAGE_SIZE;2273 end = start + HPAGE_SIZE;
2274 __storage_key_init_range(start, end);2274 __storage_key_init_range(start, end);
2275 set_bit(PG_arch_1, &folio->flags);2275 set_bit(PG_arch_1, &folio->flags.f);
2276 cond_resched();2276 cond_resched();
2277 return 0;2277 return 0;
2278}2278}
arch/s390/mm/hugetlbpage.c+1−1
@@ -155,7 +155,7 @@ static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste) · 154 unmodified lines
155 paddr = rste & PMD_MASK;155 paddr = rste & PMD_MASK;
156 }156 }
157 157
158 if (!test_and_set_bit(PG_arch_1, &folio->flags))158 if (!test_and_set_bit(PG_arch_1, &folio->flags.f))
159 __storage_key_init_range(paddr, paddr + size);159 __storage_key_init_range(paddr, paddr + size);
160}160}
161 161
arch/sh/include/asm/hugetlb.h+1−1
@@ -14,7 +14,7 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, · 13 unmodified lines
14 14
15static inline void arch_clear_hugetlb_flags(struct folio *folio)15static inline void arch_clear_hugetlb_flags(struct folio *folio)
16{16{
17 clear_bit(PG_dcache_clean, &folio->flags);17 clear_bit(PG_dcache_clean, &folio->flags.f);
18}18}
19#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags19#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
20 20
arch/sh/mm/cache-sh4.c+1−1
@@ -114,7 +114,7 @@ static void sh4_flush_dcache_folio(void *arg) · 113 unmodified lines
114 struct address_space *mapping = folio_flush_mapping(folio);114 struct address_space *mapping = folio_flush_mapping(folio);
115 115
116 if (mapping && !mapping_mapped(mapping))116 if (mapping && !mapping_mapped(mapping))
117 clear_bit(PG_dcache_clean, &folio->flags);117 clear_bit(PG_dcache_clean, &folio->flags.f);
118 else118 else
119#endif119#endif
120 {120 {
arch/sh/mm/cache-sh7705.c+1−1
@@ -138,7 +138,7 @@ static void sh7705_flush_dcache_folio(void *arg) · 137 unmodified lines
138 struct address_space *mapping = folio_flush_mapping(folio);138 struct address_space *mapping = folio_flush_mapping(folio);
139 139
140 if (mapping && !mapping_mapped(mapping))140 if (mapping && !mapping_mapped(mapping))
141 clear_bit(PG_dcache_clean, &folio->flags);141 clear_bit(PG_dcache_clean, &folio->flags.f);
142 else {142 else {
143 unsigned long pfn = folio_pfn(folio);143 unsigned long pfn = folio_pfn(folio);
144 unsigned int i, nr = folio_nr_pages(folio);144 unsigned int i, nr = folio_nr_pages(folio);
arch/sh/mm/cache.c+7−7
@@ -64,14 +64,14 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, · 63 unmodified lines
64 struct folio *folio = page_folio(page);64 struct folio *folio = page_folio(page);
65 65
66 if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) &&66 if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) &&
67 test_bit(PG_dcache_clean, &folio->flags)) {67 test_bit(PG_dcache_clean, &folio->flags.f)) {
68 void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);68 void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
69 memcpy(vto, src, len);69 memcpy(vto, src, len);
70 kunmap_coherent(vto);70 kunmap_coherent(vto);
71 } else {71 } else {
72 memcpy(dst, src, len);72 memcpy(dst, src, len);
73 if (boot_cpu_data.dcache.n_aliases)73 if (boot_cpu_data.dcache.n_aliases)
74 clear_bit(PG_dcache_clean, &folio->flags);74 clear_bit(PG_dcache_clean, &folio->flags.f);
75 }75 }
76 76
77 if (vma->vm_flags & VM_EXEC)77 if (vma->vm_flags & VM_EXEC)
@@ -85,14 +85,14 @@ void copy_from_user_page(struct vm_area_struct *vma, struct page *page, · 7 unmodified lines
85 struct folio *folio = page_folio(page);85 struct folio *folio = page_folio(page);
86 86
87 if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) &&87 if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) &&
88 test_bit(PG_dcache_clean, &folio->flags)) {88 test_bit(PG_dcache_clean, &folio->flags.f)) {
89 void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);89 void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
90 memcpy(dst, vfrom, len);90 memcpy(dst, vfrom, len);
91 kunmap_coherent(vfrom);91 kunmap_coherent(vfrom);
92 } else {92 } else {
93 memcpy(dst, src, len);93 memcpy(dst, src, len);
94 if (boot_cpu_data.dcache.n_aliases)94 if (boot_cpu_data.dcache.n_aliases)
95 clear_bit(PG_dcache_clean, &folio->flags);95 clear_bit(PG_dcache_clean, &folio->flags.f);
96 }96 }
97}97}
98 98
@@ -105,7 +105,7 @@ void copy_user_highpage(struct page *to, struct page *from, · 6 unmodified lines
105 vto = kmap_atomic(to);105 vto = kmap_atomic(to);
106 106
107 if (boot_cpu_data.dcache.n_aliases && folio_mapped(src) &&107 if (boot_cpu_data.dcache.n_aliases && folio_mapped(src) &&
108 test_bit(PG_dcache_clean, &src->flags)) {108 test_bit(PG_dcache_clean, &src->flags.f)) {
109 vfrom = kmap_coherent(from, vaddr);109 vfrom = kmap_coherent(from, vaddr);
110 copy_page(vto, vfrom);110 copy_page(vto, vfrom);
111 kunmap_coherent(vfrom);111 kunmap_coherent(vfrom);
@@ -148,7 +148,7 @@ void __update_cache(struct vm_area_struct *vma, · 36 unmodified lines
148 148
149 if (pfn_valid(pfn)) {149 if (pfn_valid(pfn)) {
150 struct folio *folio = page_folio(pfn_to_page(pfn));150 struct folio *folio = page_folio(pfn_to_page(pfn));
151 int dirty = !test_and_set_bit(PG_dcache_clean, &folio->flags);151 int dirty = !test_and_set_bit(PG_dcache_clean, &folio->flags.f);
152 if (dirty)152 if (dirty)
153 __flush_purge_region(folio_address(folio),153 __flush_purge_region(folio_address(folio),
154 folio_size(folio));154 folio_size(folio));
@@ -162,7 +162,7 @@ void __flush_anon_page(struct page *page, unsigned long vmaddr) · 7 unmodified lines
162 162
163 if (pages_do_alias(addr, vmaddr)) {163 if (pages_do_alias(addr, vmaddr)) {
164 if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) &&164 if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) &&
165 test_bit(PG_dcache_clean, &folio->flags)) {165 test_bit(PG_dcache_clean, &folio->flags.f)) {
166 void *kaddr;166 void *kaddr;
167 167
168 kaddr = kmap_coherent(page, vmaddr);168 kaddr = kmap_coherent(page, vmaddr);
arch/sh/mm/kmap.c+1−1
@@ -31,7 +31,7 @@ void *kmap_coherent(struct page *page, unsigned long addr) · 30 unmodified lines
31 enum fixed_addresses idx;31 enum fixed_addresses idx;
32 unsigned long vaddr;32 unsigned long vaddr;
33 33
34 BUG_ON(!test_bit(PG_dcache_clean, &folio->flags));34 BUG_ON(!test_bit(PG_dcache_clean, &folio->flags.f));
35 35
36 preempt_disable();36 preempt_disable();
37 pagefault_disable();37 pagefault_disable();
arch/sparc/mm/init_64.c+5−5
@@ -224,7 +224,7 @@ inline void flush_dcache_folio_impl(struct folio *folio) · 223 unmodified lines
224 ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)224 ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
225 225
226#define dcache_dirty_cpu(folio) \226#define dcache_dirty_cpu(folio) \
227 (((folio)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)227 (((folio)->flags.f >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
228 228
229static inline void set_dcache_dirty(struct folio *folio, int this_cpu)229static inline void set_dcache_dirty(struct folio *folio, int this_cpu)
230{230{
@@ -243,7 +243,7 @@ static inline void set_dcache_dirty(struct folio *folio, int this_cpu) · 12 unmodified lines
243 "bne,pn %%xcc, 1b\n\t"243 "bne,pn %%xcc, 1b\n\t"
244 " nop"244 " nop"
245 : /* no outputs */245 : /* no outputs */
246 : "r" (mask), "r" (non_cpu_bits), "r" (&folio->flags)246 : "r" (mask), "r" (non_cpu_bits), "r" (&folio->flags.f)
247 : "g1", "g7");247 : "g1", "g7");
248}248}
249 249
@@ -265,7 +265,7 @@ static inline void clear_dcache_dirty_cpu(struct folio *folio, unsigned long cpu · 15 unmodified lines
265 " nop\n"265 " nop\n"
266 "2:"266 "2:"
267 : /* no outputs */267 : /* no outputs */
268 : "r" (cpu), "r" (mask), "r" (&folio->flags),268 : "r" (cpu), "r" (mask), "r" (&folio->flags.f),
269 "i" (PG_dcache_cpu_mask),269 "i" (PG_dcache_cpu_mask),
270 "i" (PG_dcache_cpu_shift)270 "i" (PG_dcache_cpu_shift)
271 : "g1", "g7");271 : "g1", "g7");
@@ -292,7 +292,7 @@ static void flush_dcache(unsigned long pfn) · 20 unmodified lines
292 struct folio *folio = page_folio(page);292 struct folio *folio = page_folio(page);
293 unsigned long pg_flags;293 unsigned long pg_flags;
294 294
295 pg_flags = folio->flags;295 pg_flags = folio->flags.f;
296 if (pg_flags & (1UL << PG_dcache_dirty)) {296 if (pg_flags & (1UL << PG_dcache_dirty)) {
297 int cpu = ((pg_flags >> PG_dcache_cpu_shift) &297 int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
298 PG_dcache_cpu_mask);298 PG_dcache_cpu_mask);
@@ -480,7 +480,7 @@ void flush_dcache_folio(struct folio *folio) · 181 unmodified lines
480 480
481 mapping = folio_flush_mapping(folio);481 mapping = folio_flush_mapping(folio);
482 if (mapping && !mapping_mapped(mapping)) {482 if (mapping && !mapping_mapped(mapping)) {
483 bool dirty = test_bit(PG_dcache_dirty, &folio->flags);483 bool dirty = test_bit(PG_dcache_dirty, &folio->flags.f);
484 if (dirty) {484 if (dirty) {
485 int dirty_cpu = dcache_dirty_cpu(folio);485 int dirty_cpu = dcache_dirty_cpu(folio);
486 486
arch/x86/mm/pat/memtype.c+3−3
@@ -126,7 +126,7 @@ __setup("debugpat", pat_debug_setup); · 125 unmodified lines
126 126
127static inline enum page_cache_mode get_page_memtype(struct page *pg)127static inline enum page_cache_mode get_page_memtype(struct page *pg)
128{128{
129 unsigned long pg_flags = pg->flags & _PGMT_MASK;129 unsigned long pg_flags = pg->flags.f & _PGMT_MASK;
130 130
131 if (pg_flags == _PGMT_WB)131 if (pg_flags == _PGMT_WB)
132 return _PAGE_CACHE_MODE_WB;132 return _PAGE_CACHE_MODE_WB;
@@ -161,10 +161,10 @@ static inline void set_page_memtype(struct page *pg, · 28 unmodified lines
161 break;161 break;
162 }162 }
163 163
164 old_flags = READ_ONCE(pg->flags);164 old_flags = READ_ONCE(pg->flags.f);
165 do {165 do {
166 new_flags = (old_flags & _PGMT_CLEAR_MASK) | memtype_flags;166 new_flags = (old_flags & _PGMT_CLEAR_MASK) | memtype_flags;
167 } while (!try_cmpxchg(&pg->flags, &old_flags, new_flags));167 } while (!try_cmpxchg(&pg->flags.f, &old_flags, new_flags));
168}168}
169#else169#else
170static inline enum page_cache_mode get_page_memtype(struct page *pg)170static inline enum page_cache_mode get_page_memtype(struct page *pg)
arch/xtensa/mm/cache.c+6−6
@@ -134,8 +134,8 @@ void flush_dcache_folio(struct folio *folio) · 133 unmodified lines
134 */134 */
135 135
136 if (mapping && !mapping_mapped(mapping)) {136 if (mapping && !mapping_mapped(mapping)) {
137 if (!test_bit(PG_arch_1, &folio->flags))137 if (!test_bit(PG_arch_1, &folio->flags.f))
138 set_bit(PG_arch_1, &folio->flags);138 set_bit(PG_arch_1, &folio->flags.f);
139 return;139 return;
140 140
141 } else {141 } else {
@@ -232,7 +232,7 @@ void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, · 90 unmodified lines
232 232
233#if (DCACHE_WAY_SIZE > PAGE_SIZE)233#if (DCACHE_WAY_SIZE > PAGE_SIZE)
234 234
235 if (!folio_test_reserved(folio) && test_bit(PG_arch_1, &folio->flags)) {235 if (!folio_test_reserved(folio) && test_bit(PG_arch_1, &folio->flags.f)) {
236 unsigned long phys = folio_pfn(folio) * PAGE_SIZE;236 unsigned long phys = folio_pfn(folio) * PAGE_SIZE;
237 unsigned long tmp;237 unsigned long tmp;
238 238
@@ -247,10 +247,10 @@ void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, · 8 unmodified lines
247 }247 }
248 preempt_enable();248 preempt_enable();
249 249
250 clear_bit(PG_arch_1, &folio->flags);250 clear_bit(PG_arch_1, &folio->flags.f);
251 }251 }
252#else252#else
253 if (!folio_test_reserved(folio) && !test_bit(PG_arch_1, &folio->flags)253 if (!folio_test_reserved(folio) && !test_bit(PG_arch_1, &folio->flags.f)
254 && (vma->vm_flags & VM_EXEC) != 0) {254 && (vma->vm_flags & VM_EXEC) != 0) {
255 for (i = 0; i < nr; i++) {255 for (i = 0; i < nr; i++) {
256 void *paddr = kmap_local_folio(folio, i * PAGE_SIZE);256 void *paddr = kmap_local_folio(folio, i * PAGE_SIZE);
@@ -258,7 +258,7 @@ void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, · 1 unmodified line
258 __invalidate_icache_page((unsigned long)paddr);258 __invalidate_icache_page((unsigned long)paddr);
259 kunmap_local(paddr);259 kunmap_local(paddr);
260 }260 }
261 set_bit(PG_arch_1, &folio->flags);261 set_bit(PG_arch_1, &folio->flags.f);
262 }262 }
263#endif263#endif
264}264}
fs/fuse/dev.c+1−1
@@ -935,7 +935,7 @@ static int fuse_check_folio(struct folio *folio) · 934 unmodified lines
935{935{
936 if (folio_mapped(folio) ||936 if (folio_mapped(folio) ||
937 folio->mapping != NULL ||937 folio->mapping != NULL ||
938 (folio->flags & PAGE_FLAGS_CHECK_AT_PREP &938 (folio->flags.f & PAGE_FLAGS_CHECK_AT_PREP &
939 ~(1 << PG_locked |939 ~(1 << PG_locked |
940 1 << PG_referenced |940 1 << PG_referenced |
941 1 << PG_lru |941 1 << PG_lru |
fs/gfs2/glops.c+1−1
@@ -40,7 +40,7 @@ static void gfs2_ail_error(struct gfs2_glock *gl, const struct buffer_head *bh) · 39 unmodified lines
40 "AIL buffer %p: blocknr %llu state 0x%08lx mapping %p page "40 "AIL buffer %p: blocknr %llu state 0x%08lx mapping %p page "
41 "state 0x%lx\n",41 "state 0x%lx\n",
42 bh, (unsigned long long)bh->b_blocknr, bh->b_state,42 bh, (unsigned long long)bh->b_blocknr, bh->b_state,
43 bh->b_folio->mapping, bh->b_folio->flags);43 bh->b_folio->mapping, bh->b_folio->flags.f);
44 fs_err(sdp, "AIL glock %u:%llu mapping %p\n",44 fs_err(sdp, "AIL glock %u:%llu mapping %p\n",
45 gl->gl_name.ln_type, gl->gl_name.ln_number,45 gl->gl_name.ln_type, gl->gl_name.ln_number,
46 gfs2_glock2aspace(gl));46 gfs2_glock2aspace(gl));
fs/jffs2/file.c+2−2
@@ -230,7 +230,7 @@ static int jffs2_write_begin(const struct kiocb *iocb, · 229 unmodified lines
230 goto release_sem;230 goto release_sem;
231 }231 }
232 }232 }
233 jffs2_dbg(1, "end write_begin(). folio->flags %lx\n", folio->flags);233 jffs2_dbg(1, "end write_begin(). folio->flags %lx\n", folio->flags.f);
234 234
235release_sem:235release_sem:
236 mutex_unlock(&c->alloc_sem);236 mutex_unlock(&c->alloc_sem);
@@ -259,7 +259,7 @@ static int jffs2_write_end(const struct kiocb *iocb, · 22 unmodified lines
259 259
260 jffs2_dbg(1, "%s(): ino #%lu, page at 0x%llx, range %d-%d, flags %lx\n",260 jffs2_dbg(1, "%s(): ino #%lu, page at 0x%llx, range %d-%d, flags %lx\n",
261 __func__, inode->i_ino, folio_pos(folio),261 __func__, inode->i_ino, folio_pos(folio),
262 start, end, folio->flags);262 start, end, folio->flags.f);
263 263
264 /* We need to avoid deadlock with page_cache_read() in264 /* We need to avoid deadlock with page_cache_read() in
265 jffs2_garbage_collect_pass(). So the folio must be265 jffs2_garbage_collect_pass(). So the folio must be
fs/nilfs2/page.c+1−1
@@ -167,7 +167,7 @@ void nilfs_folio_bug(struct folio *folio) · 166 unmodified lines
167 printk(KERN_CRIT "NILFS_FOLIO_BUG(%p): cnt=%d index#=%llu flags=0x%lx "167 printk(KERN_CRIT "NILFS_FOLIO_BUG(%p): cnt=%d index#=%llu flags=0x%lx "
168 "mapping=%p ino=%lu\n",168 "mapping=%p ino=%lu\n",
169 folio, folio_ref_count(folio),169 folio, folio_ref_count(folio),
170 (unsigned long long)folio->index, folio->flags, m, ino);170 (unsigned long long)folio->index, folio->flags.f, m, ino);
171 171
172 head = folio_buffers(folio);172 head = folio_buffers(folio);
173 if (head) {173 if (head) {
fs/proc/page.c+2−2
@@ -163,7 +163,7 @@ u64 stable_page_flags(const struct page *page) · 162 unmodified lines
163 snapshot_page(&ps, page);163 snapshot_page(&ps, page);
164 folio = &ps.folio_snapshot;164 folio = &ps.folio_snapshot;
165 165
166 k = folio->flags;166 k = folio->flags.f;
167 mapping = (unsigned long)folio->mapping;167 mapping = (unsigned long)folio->mapping;
168 is_anon = mapping & FOLIO_MAPPING_ANON;168 is_anon = mapping & FOLIO_MAPPING_ANON;
169 169
@@ -238,7 +238,7 @@ u64 stable_page_flags(const struct page *page) · 68 unmodified lines
238 if (u & (1 << KPF_HUGE))238 if (u & (1 << KPF_HUGE))
239 u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison);239 u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison);
240 else240 else
241 u |= kpf_copy_bit(ps.page_snapshot.flags, KPF_HWPOISON, PG_hwpoison);241 u |= kpf_copy_bit(ps.page_snapshot.flags.f, KPF_HWPOISON, PG_hwpoison);
242#endif242#endif
243 243
244 u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved);244 u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved);
fs/ubifs/file.c+3−3
@@ -107,7 +107,7 @@ static int do_readpage(struct folio *folio) · 106 unmodified lines
107 size_t offset = 0;107 size_t offset = 0;
108 108
109 dbg_gen("ino %lu, pg %lu, i_size %lld, flags %#lx",109 dbg_gen("ino %lu, pg %lu, i_size %lld, flags %#lx",
110 inode->i_ino, folio->index, i_size, folio->flags);110 inode->i_ino, folio->index, i_size, folio->flags.f);
111 ubifs_assert(c, !folio_test_checked(folio));111 ubifs_assert(c, !folio_test_checked(folio));
112 ubifs_assert(c, !folio->private);112 ubifs_assert(c, !folio->private);
113 113
@@ -600,7 +600,7 @@ static int populate_page(struct ubifs_info *c, struct folio *folio, · 486 unmodified lines
600 pgoff_t end_index;600 pgoff_t end_index;
601 601
602 dbg_gen("ino %lu, pg %lu, i_size %lld, flags %#lx",602 dbg_gen("ino %lu, pg %lu, i_size %lld, flags %#lx",
603 inode->i_ino, folio->index, i_size, folio->flags);603 inode->i_ino, folio->index, i_size, folio->flags.f);
604 604
605 end_index = (i_size - 1) >> PAGE_SHIFT;605 end_index = (i_size - 1) >> PAGE_SHIFT;
606 if (!i_size || folio->index > end_index) {606 if (!i_size || folio->index > end_index) {
@@ -988,7 +988,7 @@ static int ubifs_writepage(struct folio *folio, struct writeback_control *wbc) · 381 unmodified lines
988 int err, len = folio_size(folio);988 int err, len = folio_size(folio);
989 989
990 dbg_gen("ino %lu, pg %lu, pg flags %#lx",990 dbg_gen("ino %lu, pg %lu, pg flags %#lx",
991 inode->i_ino, folio->index, folio->flags);991 inode->i_ino, folio->index, folio->flags.f);
992 ubifs_assert(c, folio->private != NULL);992 ubifs_assert(c, folio->private != NULL);
993 993
994 /* Is the folio fully outside @i_size? (truncate in progress) */994 /* Is the folio fully outside @i_size? (truncate in progress) */
include/linux/mm.h+16−16
@@ -1024,7 +1024,7 @@ static inline unsigned int compound_order(struct page *page) · 1023 unmodified lines
1024{1024{
1025 struct folio *folio = (struct folio *)page;1025 struct folio *folio = (struct folio *)page;
1026 1026
1027 if (!test_bit(PG_head, &folio->flags))1027 if (!test_bit(PG_head, &folio->flags.f))
1028 return 0;1028 return 0;
1029 return folio_large_order(folio);1029 return folio_large_order(folio);
1030}1030}
@@ -1554,7 +1554,7 @@ static inline bool is_nommu_shared_mapping(vm_flags_t flags) · 523 unmodified lines
1554 */1554 */
1555static inline int page_zone_id(struct page *page)1555static inline int page_zone_id(struct page *page)
1556{1556{
1557 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;1557 return (page->flags.f >> ZONEID_PGSHIFT) & ZONEID_MASK;
1558}1558}
1559 1559
1560#ifdef NODE_NOT_IN_PAGE_FLAGS1560#ifdef NODE_NOT_IN_PAGE_FLAGS
@@ -1562,7 +1562,7 @@ int page_to_nid(const struct page *page); · 1 unmodified line
1562#else1562#else
1563static inline int page_to_nid(const struct page *page)1563static inline int page_to_nid(const struct page *page)
1564{1564{
1565 return (PF_POISONED_CHECK(page)->flags >> NODES_PGSHIFT) & NODES_MASK;1565 return (PF_POISONED_CHECK(page)->flags.f >> NODES_PGSHIFT) & NODES_MASK;
1566}1566}
1567#endif1567#endif
1568 1568
@@ -1637,14 +1637,14 @@ static inline void page_cpupid_reset_last(struct page *page) · 68 unmodified lines
1637#else1637#else
1638static inline int folio_last_cpupid(struct folio *folio)1638static inline int folio_last_cpupid(struct folio *folio)
1639{1639{
1640 return (folio->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;1640 return (folio->flags.f >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
1641}1641}
1642 1642
1643int folio_xchg_last_cpupid(struct folio *folio, int cpupid);1643int folio_xchg_last_cpupid(struct folio *folio, int cpupid);
1644 1644
1645static inline void page_cpupid_reset_last(struct page *page)1645static inline void page_cpupid_reset_last(struct page *page)
1646{1646{
1647 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;1647 page->flags.f |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
1648}1648}
1649#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */1649#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
1650 1650
@@ -1740,7 +1740,7 @@ static inline u8 page_kasan_tag(const struct page *page) · 89 unmodified lines
1740 u8 tag = KASAN_TAG_KERNEL;1740 u8 tag = KASAN_TAG_KERNEL;
1741 1741
1742 if (kasan_enabled()) {1742 if (kasan_enabled()) {
1743 tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;1743 tag = (page->flags.f >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;
1744 tag ^= 0xff;1744 tag ^= 0xff;
1745 }1745 }
1746 1746
@@ -1755,12 +1755,12 @@ static inline void page_kasan_tag_set(struct page *page, u8 tag) · 8 unmodified lines
1755 return;1755 return;
1756 1756
1757 tag ^= 0xff;1757 tag ^= 0xff;
1758 old_flags = READ_ONCE(page->flags);1758 old_flags = READ_ONCE(page->flags.f);
1759 do {1759 do {
1760 flags = old_flags;1760 flags = old_flags;
1761 flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);1761 flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);
1762 flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;1762 flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;
1763 } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags)));1763 } while (unlikely(!try_cmpxchg(&page->flags.f, &old_flags, flags)));
1764}1764}
1765 1765
1766static inline void page_kasan_tag_reset(struct page *page)1766static inline void page_kasan_tag_reset(struct page *page)
@@ -1804,13 +1804,13 @@ static inline pg_data_t *folio_pgdat(const struct folio *folio) · 37 unmodified lines
1804#ifdef SECTION_IN_PAGE_FLAGS1804#ifdef SECTION_IN_PAGE_FLAGS
1805static inline void set_page_section(struct page *page, unsigned long section)1805static inline void set_page_section(struct page *page, unsigned long section)
1806{1806{
1807 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);1807 page->flags.f &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
1808 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;1808 page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
1809}1809}
1810 1810
1811static inline unsigned long page_to_section(const struct page *page)1811static inline unsigned long page_to_section(const struct page *page)
1812{1812{
1813 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;1813 return (page->flags.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
1814}1814}
1815#endif1815#endif
1816 1816
@@ -2015,14 +2015,14 @@ static inline bool folio_is_longterm_pinnable(struct folio *folio) · 198 unmodified lines
2015 2015
2016static inline void set_page_zone(struct page *page, enum zone_type zone)2016static inline void set_page_zone(struct page *page, enum zone_type zone)
2017{2017{
2018 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);2018 page->flags.f &= ~(ZONES_MASK << ZONES_PGSHIFT);
2019 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;2019 page->flags.f |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
2020}2020}
2021 2021
2022static inline void set_page_node(struct page *page, unsigned long node)2022static inline void set_page_node(struct page *page, unsigned long node)
2023{2023{
2024 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);2024 page->flags.f &= ~(NODES_MASK << NODES_PGSHIFT);
2025 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;2025 page->flags.f |= (node & NODES_MASK) << NODES_PGSHIFT;
2026}2026}
2027 2027
2028static inline void set_page_links(struct page *page, enum zone_type zone,2028static inline void set_page_links(struct page *page, enum zone_type zone,
@@ -2064,7 +2064,7 @@ static inline long compound_nr(struct page *page) · 35 unmodified lines
2064{2064{
2065 struct folio *folio = (struct folio *)page;2065 struct folio *folio = (struct folio *)page;
2066 2066
2067 if (!test_bit(PG_head, &folio->flags))2067 if (!test_bit(PG_head, &folio->flags.f))
2068 return 1;2068 return 1;
2069 return folio_large_nr_pages(folio);2069 return folio_large_nr_pages(folio);
2070}2070}
include/linux/mm_inline.h+6−6
@@ -143,7 +143,7 @@ static inline int lru_tier_from_refs(int refs, bool workingset) · 142 unmodified lines
143 143
144static inline int folio_lru_refs(struct folio *folio)144static inline int folio_lru_refs(struct folio *folio)
145{145{
146 unsigned long flags = READ_ONCE(folio->flags);146 unsigned long flags = READ_ONCE(folio->flags.f);
147 147
148 if (!(flags & BIT(PG_referenced)))148 if (!(flags & BIT(PG_referenced)))
149 return 0;149 return 0;
@@ -156,7 +156,7 @@ static inline int folio_lru_refs(struct folio *folio) · 6 unmodified lines
156 156
157static inline int folio_lru_gen(struct folio *folio)157static inline int folio_lru_gen(struct folio *folio)
158{158{
159 unsigned long flags = READ_ONCE(folio->flags);159 unsigned long flags = READ_ONCE(folio->flags.f);
160 160
161 return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;161 return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
162}162}
@@ -268,7 +268,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, · 105 unmodified lines
268 gen = lru_gen_from_seq(seq);268 gen = lru_gen_from_seq(seq);
269 flags = (gen + 1UL) << LRU_GEN_PGOFF;269 flags = (gen + 1UL) << LRU_GEN_PGOFF;
270 /* see the comment on MIN_NR_GENS about PG_active */270 /* see the comment on MIN_NR_GENS about PG_active */
271 set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);271 set_mask_bits(&folio->flags.f, LRU_GEN_MASK | BIT(PG_active), flags);
272 272
273 lru_gen_update_size(lruvec, folio, -1, gen);273 lru_gen_update_size(lruvec, folio, -1, gen);
274 /* for folio_rotate_reclaimable() */274 /* for folio_rotate_reclaimable() */
@@ -293,7 +293,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, · 18 unmodified lines
293 293
294 /* for folio_migrate_flags() */294 /* for folio_migrate_flags() */
295 flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;295 flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
296 flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags);296 flags = set_mask_bits(&folio->flags.f, LRU_GEN_MASK, flags);
297 gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;297 gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
298 298
299 lru_gen_update_size(lruvec, folio, gen, -1);299 lru_gen_update_size(lruvec, folio, gen, -1);
@@ -304,9 +304,9 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, · 4 unmodified lines
304 304
305static inline void folio_migrate_refs(struct folio *new, struct folio *old)305static inline void folio_migrate_refs(struct folio *new, struct folio *old)
306{306{
307 unsigned long refs = READ_ONCE(old->flags) & LRU_REFS_MASK;307 unsigned long refs = READ_ONCE(old->flags.f) & LRU_REFS_MASK;
308 308
309 set_mask_bits(&new->flags, LRU_REFS_MASK, refs);309 set_mask_bits(&new->flags.f, LRU_REFS_MASK, refs);
310}310}
311#else /* !CONFIG_LRU_GEN */311#else /* !CONFIG_LRU_GEN */
312 312
include/linux/mm_types.h+6−2
@@ -34,6 +34,10 @@ struct address_space; · 33 unmodified lines
34struct futex_private_hash;34struct futex_private_hash;
35struct mem_cgroup;35struct mem_cgroup;
36 36
37typedef struct {
38 unsigned long f;
39} memdesc_flags_t;
40
37/*41/*
38 * Each physical page in the system has a struct page associated with42 * Each physical page in the system has a struct page associated with
39 * it to keep track of whatever it is we are using the page for at the43 * it to keep track of whatever it is we are using the page for at the
@@ -72,7 +76,7 @@ struct mem_cgroup; · 32 unmodified lines
72#endif76#endif
73 77
74struct page {78struct page {
75 unsigned long flags; /* Atomic flags, some possibly79 memdesc_flags_t flags; /* Atomic flags, some possibly
76 * updated asynchronously */80 * updated asynchronously */
77 /*81 /*
78 * Five words (20/40 bytes) are available in this union.82 * Five words (20/40 bytes) are available in this union.
@@ -383,7 +387,7 @@ struct folio { · 304 unmodified lines
383 union {387 union {
384 struct {388 struct {
385 /* public: */389 /* public: */
386 unsigned long flags;390 memdesc_flags_t flags;
387 union {391 union {
388 struct list_head lru;392 struct list_head lru;
389 /* private: avoid cluttering the output */393 /* private: avoid cluttering the output */
include/linux/mmzone.h+1−1
@@ -1186,7 +1186,7 @@ static inline bool zone_is_empty(struct zone *zone) · 1185 unmodified lines
1186static inline enum zone_type page_zonenum(const struct page *page)1186static inline enum zone_type page_zonenum(const struct page *page)
1187{1187{
1188 ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT);1188 ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT);
1189 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;1189 return (page->flags.f >> ZONES_PGSHIFT) & ZONES_MASK;
1190}1190}
1191 1191
1192static inline enum zone_type folio_zonenum(const struct folio *folio)1192static inline enum zone_type folio_zonenum(const struct folio *folio)
include/linux/page-flags.h+20−20
@@ -217,7 +217,7 @@ static __always_inline const struct page *page_fixed_fake_head(const struct page · 216 unmodified lines
217 * cold cacheline in some cases.217 * cold cacheline in some cases.
218 */218 */
219 if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&219 if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
220 test_bit(PG_head, &page->flags)) {220 test_bit(PG_head, &page->flags.f)) {
221 /*221 /*
222 * We can safely access the field of the @page[1] with PG_head222 * We can safely access the field of the @page[1] with PG_head
223 * because the @page is a compound page composed with at least223 * because the @page is a compound page composed with at least
@@ -325,14 +325,14 @@ static __always_inline int PageTail(const struct page *page) · 101 unmodified lines
325 325
326static __always_inline int PageCompound(const struct page *page)326static __always_inline int PageCompound(const struct page *page)
327{327{
328 return test_bit(PG_head, &page->flags) ||328 return test_bit(PG_head, &page->flags.f) ||
329 READ_ONCE(page->compound_head) & 1;329 READ_ONCE(page->compound_head) & 1;
330}330}
331 331
332#define PAGE_POISON_PATTERN -1l332#define PAGE_POISON_PATTERN -1l
333static inline int PagePoisoned(const struct page *page)333static inline int PagePoisoned(const struct page *page)
334{334{
335 return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;335 return READ_ONCE(page->flags.f) == PAGE_POISON_PATTERN;
336}336}
337 337
338#ifdef CONFIG_DEBUG_VM338#ifdef CONFIG_DEBUG_VM
@@ -349,8 +349,8 @@ static const unsigned long *const_folio_flags(const struct folio *folio, · 10 unmodified lines
349 const struct page *page = &folio->page;349 const struct page *page = &folio->page;
350 350
351 VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);351 VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);
352 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);352 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page);
353 return &page[n].flags;353 return &page[n].flags.f;
354}354}
355 355
356static unsigned long *folio_flags(struct folio *folio, unsigned n)356static unsigned long *folio_flags(struct folio *folio, unsigned n)
@@ -358,8 +358,8 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n) · 1 unmodified line
358 struct page *page = &folio->page;358 struct page *page = &folio->page;
359 359
360 VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);360 VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);
361 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);361 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page);
362 return &page[n].flags;362 return &page[n].flags.f;
363}363}
364 364
365/*365/*
@@ -449,37 +449,37 @@ FOLIO_CLEAR_FLAG(name, page) · 83 unmodified lines
449#define TESTPAGEFLAG(uname, lname, policy) \449#define TESTPAGEFLAG(uname, lname, policy) \
450FOLIO_TEST_FLAG(lname, FOLIO_##policy) \450FOLIO_TEST_FLAG(lname, FOLIO_##policy) \
451static __always_inline int Page##uname(const struct page *page) \451static __always_inline int Page##uname(const struct page *page) \
452{ return test_bit(PG_##lname, &policy(page, 0)->flags); }452{ return test_bit(PG_##lname, &policy(page, 0)->flags.f); }
453 453
454#define SETPAGEFLAG(uname, lname, policy) \454#define SETPAGEFLAG(uname, lname, policy) \
455FOLIO_SET_FLAG(lname, FOLIO_##policy) \455FOLIO_SET_FLAG(lname, FOLIO_##policy) \
456static __always_inline void SetPage##uname(struct page *page) \456static __always_inline void SetPage##uname(struct page *page) \
457{ set_bit(PG_##lname, &policy(page, 1)->flags); }457{ set_bit(PG_##lname, &policy(page, 1)->flags.f); }
458 458
459#define CLEARPAGEFLAG(uname, lname, policy) \459#define CLEARPAGEFLAG(uname, lname, policy) \
460FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \460FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \
461static __always_inline void ClearPage##uname(struct page *page) \461static __always_inline void ClearPage##uname(struct page *page) \
462{ clear_bit(PG_##lname, &policy(page, 1)->flags); }462{ clear_bit(PG_##lname, &policy(page, 1)->flags.f); }
463 463
464#define __SETPAGEFLAG(uname, lname, policy) \464#define __SETPAGEFLAG(uname, lname, policy) \
465__FOLIO_SET_FLAG(lname, FOLIO_##policy) \465__FOLIO_SET_FLAG(lname, FOLIO_##policy) \
466static __always_inline void __SetPage##uname(struct page *page) \466static __always_inline void __SetPage##uname(struct page *page) \
467{ __set_bit(PG_##lname, &policy(page, 1)->flags); }467{ __set_bit(PG_##lname, &policy(page, 1)->flags.f); }
468 468
469#define __CLEARPAGEFLAG(uname, lname, policy) \469#define __CLEARPAGEFLAG(uname, lname, policy) \
470__FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \470__FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \
471static __always_inline void __ClearPage##uname(struct page *page) \471static __always_inline void __ClearPage##uname(struct page *page) \
472{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }472{ __clear_bit(PG_##lname, &policy(page, 1)->flags.f); }
473 473
474#define TESTSETFLAG(uname, lname, policy) \474#define TESTSETFLAG(uname, lname, policy) \
475FOLIO_TEST_SET_FLAG(lname, FOLIO_##policy) \475FOLIO_TEST_SET_FLAG(lname, FOLIO_##policy) \
476static __always_inline int TestSetPage##uname(struct page *page) \476static __always_inline int TestSetPage##uname(struct page *page) \
477{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }477{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags.f); }
478 478
479#define TESTCLEARFLAG(uname, lname, policy) \479#define TESTCLEARFLAG(uname, lname, policy) \
480FOLIO_TEST_CLEAR_FLAG(lname, FOLIO_##policy) \480FOLIO_TEST_CLEAR_FLAG(lname, FOLIO_##policy) \
481static __always_inline int TestClearPage##uname(struct page *page) \481static __always_inline int TestClearPage##uname(struct page *page) \
482{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }482{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags.f); }
483 483
484#define PAGEFLAG(uname, lname, policy) \484#define PAGEFLAG(uname, lname, policy) \
485 TESTPAGEFLAG(uname, lname, policy) \485 TESTPAGEFLAG(uname, lname, policy) \
@@ -846,7 +846,7 @@ static __always_inline bool folio_test_head(const struct folio *folio) · 360 unmodified lines
846static __always_inline int PageHead(const struct page *page)846static __always_inline int PageHead(const struct page *page)
847{847{
848 PF_POISONED_CHECK(page);848 PF_POISONED_CHECK(page);
849 return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);849 return test_bit(PG_head, &page->flags.f) && !page_is_fake_head(page);
850}850}
851 851
852__SETPAGEFLAG(Head, head, PF_ANY)852__SETPAGEFLAG(Head, head, PF_ANY)
@@ -1170,28 +1170,28 @@ static __always_inline int PageAnonExclusive(const struct page *page) · 317 unmodified lines
1170 */1170 */
1171 if (PageHuge(page))1171 if (PageHuge(page))
1172 page = compound_head(page);1172 page = compound_head(page);
1173 return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);1173 return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags.f);
1174}1174}
1175 1175
1176static __always_inline void SetPageAnonExclusive(struct page *page)1176static __always_inline void SetPageAnonExclusive(struct page *page)
1177{1177{
1178 VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);1178 VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);
1179 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);1179 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1180 set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);1180 set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags.f);
1181}1181}
1182 1182
1183static __always_inline void ClearPageAnonExclusive(struct page *page)1183static __always_inline void ClearPageAnonExclusive(struct page *page)
1184{1184{
1185 VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);1185 VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);
1186 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);1186 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1187 clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);1187 clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags.f);
1188}1188}
1189 1189
1190static __always_inline void __ClearPageAnonExclusive(struct page *page)1190static __always_inline void __ClearPageAnonExclusive(struct page *page)
1191{1191{
1192 VM_BUG_ON_PGFLAGS(!PageAnon(page), page);1192 VM_BUG_ON_PGFLAGS(!PageAnon(page), page);
1193 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);1193 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1194 __clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);1194 __clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags.f);
1195}1195}
1196 1196
1197#ifdef CONFIG_MMU1197#ifdef CONFIG_MMU
@@ -1241,7 +1241,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) · 43 unmodified lines
1241 */1241 */
1242static inline int folio_has_private(const struct folio *folio)1242static inline int folio_has_private(const struct folio *folio)
1243{1243{
1244 return !!(folio->flags & PAGE_FLAGS_PRIVATE);1244 return !!(folio->flags.f & PAGE_FLAGS_PRIVATE);
1245}1245}
1246 1246
1247#undef PF_ANY1247#undef PF_ANY
include/linux/pgalloc_tag.h+4−3
@@ -107,7 +107,8 @@ static inline bool get_page_tag_ref(struct page *page, union codetag_ref *ref, · 106 unmodified lines
107 if (static_key_enabled(&mem_profiling_compressed)) {107 if (static_key_enabled(&mem_profiling_compressed)) {
108 pgalloc_tag_idx idx;108 pgalloc_tag_idx idx;
109 109
110 idx = (page->flags >> alloc_tag_ref_offs) & alloc_tag_ref_mask;110 idx = (page->flags.f >> alloc_tag_ref_offs) &
111 alloc_tag_ref_mask;
111 idx_to_ref(idx, ref);112 idx_to_ref(idx, ref);
112 handle->page = page;113 handle->page = page;
113 } else {114 } else {
@@ -149,11 +150,11 @@ static inline void update_page_tag_ref(union pgtag_ref_handle handle, union code · 35 unmodified lines
149 idx = (unsigned long)ref_to_idx(ref);150 idx = (unsigned long)ref_to_idx(ref);
150 idx = (idx & alloc_tag_ref_mask) << alloc_tag_ref_offs;151 idx = (idx & alloc_tag_ref_mask) << alloc_tag_ref_offs;
151 do {152 do {
152 old_flags = READ_ONCE(page->flags);153 old_flags = READ_ONCE(page->flags.f);
153 flags = old_flags;154 flags = old_flags;
154 flags &= ~(alloc_tag_ref_mask << alloc_tag_ref_offs);155 flags &= ~(alloc_tag_ref_mask << alloc_tag_ref_offs);
155 flags |= idx;156 flags |= idx;
156 } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags)));157 } while (unlikely(!try_cmpxchg(&page->flags.f, &old_flags, flags)));
157 } else {158 } else {
158 if (WARN_ON(!handle.ref || !ref))159 if (WARN_ON(!handle.ref || !ref))
159 return;160 return;
include/trace/events/page_ref.h+2−2
@@ -28,7 +28,7 @@ DECLARE_EVENT_CLASS(page_ref_mod_template, · 27 unmodified lines
28 28
29 TP_fast_assign(29 TP_fast_assign(
30 __entry->pfn = page_to_pfn(page);30 __entry->pfn = page_to_pfn(page);
31 __entry->flags = page->flags;31 __entry->flags = page->flags.f;
32 __entry->count = page_ref_count(page);32 __entry->count = page_ref_count(page);
33 __entry->mapcount = atomic_read(&page->_mapcount);33 __entry->mapcount = atomic_read(&page->_mapcount);
34 __entry->mapping = page->mapping;34 __entry->mapping = page->mapping;
@@ -77,7 +77,7 @@ DECLARE_EVENT_CLASS(page_ref_mod_and_test_template, · 42 unmodified lines
77 77
78 TP_fast_assign(78 TP_fast_assign(
79 __entry->pfn = page_to_pfn(page);79 __entry->pfn = page_to_pfn(page);
80 __entry->flags = page->flags;80 __entry->flags = page->flags.f;
81 __entry->count = page_ref_count(page);81 __entry->count = page_ref_count(page);
82 __entry->mapcount = atomic_read(&page->_mapcount);82 __entry->mapcount = atomic_read(&page->_mapcount);
83 __entry->mapping = page->mapping;83 __entry->mapping = page->mapping;
mm/filemap.c+4−4
@@ -1140,10 +1140,10 @@ static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, · 1139 unmodified lines
1140 */1140 */
1141 flags = wait->flags;1141 flags = wait->flags;
1142 if (flags & WQ_FLAG_EXCLUSIVE) {1142 if (flags & WQ_FLAG_EXCLUSIVE) {
1143 if (test_bit(key->bit_nr, &key->folio->flags))1143 if (test_bit(key->bit_nr, &key->folio->flags.f))
1144 return -1;1144 return -1;
1145 if (flags & WQ_FLAG_CUSTOM) {1145 if (flags & WQ_FLAG_CUSTOM) {
1146 if (test_and_set_bit(key->bit_nr, &key->folio->flags))1146 if (test_and_set_bit(key->bit_nr, &key->folio->flags.f))
1147 return -1;1147 return -1;
1148 flags |= WQ_FLAG_DONE;1148 flags |= WQ_FLAG_DONE;
1149 }1149 }
@@ -1226,9 +1226,9 @@ static inline bool folio_trylock_flag(struct folio *folio, int bit_nr, · 76 unmodified lines
1226 struct wait_queue_entry *wait)1226 struct wait_queue_entry *wait)
1227{1227{
1228 if (wait->flags & WQ_FLAG_EXCLUSIVE) {1228 if (wait->flags & WQ_FLAG_EXCLUSIVE) {
1229 if (test_and_set_bit(bit_nr, &folio->flags))1229 if (test_and_set_bit(bit_nr, &folio->flags.f))
1230 return false;1230 return false;
1231 } else if (test_bit(bit_nr, &folio->flags))1231 } else if (test_bit(bit_nr, &folio->flags.f))
1232 return false;1232 return false;
1233 1233
1234 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE;1234 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE;
mm/huge_memory.c+2−2
@@ -3303,8 +3303,8 @@ static void __split_folio_to_order(struct folio *folio, int old_order, · 3302 unmodified lines
3303 * unreferenced sub-pages of an anonymous THP: we can simply drop3303 * unreferenced sub-pages of an anonymous THP: we can simply drop
3304 * PG_anon_exclusive (-> PG_mappedtodisk) for these here.3304 * PG_anon_exclusive (-> PG_mappedtodisk) for these here.
3305 */3305 */
3306 new_folio->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;3306 new_folio->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
3307 new_folio->flags |= (folio->flags &3307 new_folio->flags.f |= (folio->flags.f &
3308 ((1L << PG_referenced) |3308 ((1L << PG_referenced) |
3309 (1L << PG_swapbacked) |3309 (1L << PG_swapbacked) |
3310 (1L << PG_swapcache) |3310 (1L << PG_swapcache) |
mm/memory-failure.c+6−6
@@ -1707,10 +1707,10 @@ static int identify_page_state(unsigned long pfn, struct page *p, · 1706 unmodified lines
1707 * carried out only if the first check can't determine the page status.1707 * carried out only if the first check can't determine the page status.
1708 */1708 */
1709 for (ps = error_states;; ps++)1709 for (ps = error_states;; ps++)
1710 if ((p->flags & ps->mask) == ps->res)1710 if ((p->flags.f & ps->mask) == ps->res)
1711 break;1711 break;
1712 1712
1713 page_flags |= (p->flags & (1UL << PG_dirty));1713 page_flags |= (p->flags.f & (1UL << PG_dirty));
1714 1714
1715 if (!ps->mask)1715 if (!ps->mask)
1716 for (ps = error_states;; ps++)1716 for (ps = error_states;; ps++)
@@ -2137,7 +2137,7 @@ retry: · 420 unmodified lines
2137 return action_result(pfn, MF_MSG_FREE_HUGE, res);2137 return action_result(pfn, MF_MSG_FREE_HUGE, res);
2138 }2138 }
2139 2139
2140 page_flags = folio->flags;2140 page_flags = folio->flags.f;
2141 2141
2142 if (!hwpoison_user_mappings(folio, p, pfn, flags)) {2142 if (!hwpoison_user_mappings(folio, p, pfn, flags)) {
2143 folio_unlock(folio);2143 folio_unlock(folio);
@@ -2398,7 +2398,7 @@ try_again: · 254 unmodified lines
2398 * folio_remove_rmap_*() in try_to_unmap_one(). So to determine page2398 * folio_remove_rmap_*() in try_to_unmap_one(). So to determine page
2399 * status correctly, we save a copy of the page flags at this time.2399 * status correctly, we save a copy of the page flags at this time.
2400 */2400 */
2401 page_flags = folio->flags;2401 page_flags = folio->flags.f;
2402 2402
2403 /*2403 /*
2404 * __munlock_folio() may clear a writeback folio's LRU flag without2404 * __munlock_folio() may clear a writeback folio's LRU flag without
@@ -2744,13 +2744,13 @@ static int soft_offline_in_use_page(struct page *page) · 339 unmodified lines
2744 putback_movable_pages(&pagelist);2744 putback_movable_pages(&pagelist);
2745 2745
2746 pr_info("%#lx: %s migration failed %ld, type %pGp\n",2746 pr_info("%#lx: %s migration failed %ld, type %pGp\n",
2747 pfn, msg_page[huge], ret, &page->flags);2747 pfn, msg_page[huge], ret, &page->flags.f);
2748 if (ret > 0)2748 if (ret > 0)
2749 ret = -EBUSY;2749 ret = -EBUSY;
2750 }2750 }
2751 } else {2751 } else {
2752 pr_info("%#lx: %s isolation failed, page count %d, type %pGp\n",2752 pr_info("%#lx: %s isolation failed, page count %d, type %pGp\n",
2753 pfn, msg_page[huge], page_count(page), &page->flags);2753 pfn, msg_page[huge], page_count(page), &page->flags.f);
2754 ret = -EBUSY;2754 ret = -EBUSY;
2755 }2755 }
2756 return ret;2756 return ret;
mm/mmzone.c+2−2
@@ -99,14 +99,14 @@ int folio_xchg_last_cpupid(struct folio *folio, int cpupid) · 98 unmodified lines
99 unsigned long old_flags, flags;99 unsigned long old_flags, flags;
100 int last_cpupid;100 int last_cpupid;
101 101
102 old_flags = READ_ONCE(folio->flags);102 old_flags = READ_ONCE(folio->flags.f);
103 do {103 do {
104 flags = old_flags;104 flags = old_flags;
105 last_cpupid = (flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;105 last_cpupid = (flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
106 106
107 flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);107 flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
108 flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;108 flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
109 } while (unlikely(!try_cmpxchg(&folio->flags, &old_flags, flags)));109 } while (unlikely(!try_cmpxchg(&folio->flags.f, &old_flags, flags)));
110 110
111 return last_cpupid;111 return last_cpupid;
112}112}
mm/page_alloc.c+6−6
@@ -950,7 +950,7 @@ static inline void __free_one_page(struct page *page, · 949 unmodified lines
950 bool to_tail;950 bool to_tail;
951 951
952 VM_BUG_ON(!zone_is_initialized(zone));952 VM_BUG_ON(!zone_is_initialized(zone));
953 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);953 VM_BUG_ON_PAGE(page->flags.f & PAGE_FLAGS_CHECK_AT_PREP, page);
954 954
955 VM_BUG_ON(migratetype == -1);955 VM_BUG_ON(migratetype == -1);
956 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);956 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
@@ -1043,7 +1043,7 @@ static inline bool page_expected_state(struct page *page, · 86 unmodified lines
1043 page->memcg_data |1043 page->memcg_data |
1044#endif1044#endif
1045 page_pool_page_is_pp(page) |1045 page_pool_page_is_pp(page) |
1046 (page->flags & check_flags)))1046 (page->flags.f & check_flags)))
1047 return false;1047 return false;
1048 1048
1049 return true;1049 return true;
@@ -1059,7 +1059,7 @@ static const char *page_bad_reason(struct page *page, unsigned long flags) · 9 unmodified lines
1059 bad_reason = "non-NULL mapping";1059 bad_reason = "non-NULL mapping";
1060 if (unlikely(page_ref_count(page) != 0))1060 if (unlikely(page_ref_count(page) != 0))
1061 bad_reason = "nonzero _refcount";1061 bad_reason = "nonzero _refcount";
1062 if (unlikely(page->flags & flags)) {1062 if (unlikely(page->flags.f & flags)) {
1063 if (flags == PAGE_FLAGS_CHECK_AT_PREP)1063 if (flags == PAGE_FLAGS_CHECK_AT_PREP)
1064 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";1064 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
1065 else1065 else
@@ -1358,7 +1358,7 @@ __always_inline bool free_pages_prepare(struct page *page, · 292 unmodified lines
1358 int i;1358 int i;
1359 1359
1360 if (compound) {1360 if (compound) {
1361 page[1].flags &= ~PAGE_FLAGS_SECOND;1361 page[1].flags.f &= ~PAGE_FLAGS_SECOND;
1362#ifdef NR_PAGES_IN_LARGE_FOLIO1362#ifdef NR_PAGES_IN_LARGE_FOLIO
1363 folio->_nr_pages = 0;1363 folio->_nr_pages = 0;
1364#endif1364#endif
@@ -1372,7 +1372,7 @@ __always_inline bool free_pages_prepare(struct page *page, · 7 unmodified lines
1372 continue;1372 continue;
1373 }1373 }
1374 }1374 }
1375 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;1375 (page + i)->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
1376 }1376 }
1377 }1377 }
1378 if (folio_test_anon(folio)) {1378 if (folio_test_anon(folio)) {
@@ -1391,7 +1391,7 @@ __always_inline bool free_pages_prepare(struct page *page, · 12 unmodified lines
1391 }1391 }
1392 1392
1393 page_cpupid_reset_last(page);1393 page_cpupid_reset_last(page);
1394 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;1394 page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
1395 reset_page_owner(page, order);1395 reset_page_owner(page, order);
1396 page_table_check_free(page, order);1396 page_table_check_free(page, order);
1397 pgalloc_tag_sub(page, 1 << order);1397 pgalloc_tag_sub(page, 1 << order);
mm/swap.c+4−4
@@ -387,14 +387,14 @@ static void __lru_cache_activate_folio(struct folio *folio) · 386 unmodified lines
387 387
388static void lru_gen_inc_refs(struct folio *folio)388static void lru_gen_inc_refs(struct folio *folio)
389{389{
390 unsigned long new_flags, old_flags = READ_ONCE(folio->flags);390 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
391 391
392 if (folio_test_unevictable(folio))392 if (folio_test_unevictable(folio))
393 return;393 return;
394 394
395 /* see the comment on LRU_REFS_FLAGS */395 /* see the comment on LRU_REFS_FLAGS */
396 if (!folio_test_referenced(folio)) {396 if (!folio_test_referenced(folio)) {
397 set_mask_bits(&folio->flags, LRU_REFS_MASK, BIT(PG_referenced));397 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
398 return;398 return;
399 }399 }
400 400
@@ -406,7 +406,7 @@ static void lru_gen_inc_refs(struct folio *folio) · 5 unmodified lines
406 }406 }
407 407
408 new_flags = old_flags + BIT(LRU_REFS_PGOFF);408 new_flags = old_flags + BIT(LRU_REFS_PGOFF);
409 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));409 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
410}410}
411 411
412static bool lru_gen_clear_refs(struct folio *folio)412static bool lru_gen_clear_refs(struct folio *folio)
@@ -418,7 +418,7 @@ static bool lru_gen_clear_refs(struct folio *folio) · 5 unmodified lines
418 if (gen < 0)418 if (gen < 0)
419 return true;419 return true;
420 420
421 set_mask_bits(&folio->flags, LRU_REFS_FLAGS | BIT(PG_workingset), 0);421 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS | BIT(PG_workingset), 0);
422 422
423 lrugen = &folio_lruvec(folio)->lrugen;423 lrugen = &folio_lruvec(folio)->lrugen;
424 /* whether can do without shuffling under the LRU lock */424 /* whether can do without shuffling under the LRU lock */
mm/vmscan.c+9−9
@@ -888,11 +888,11 @@ static bool lru_gen_set_refs(struct folio *folio) · 887 unmodified lines
888{888{
889 /* see the comment on LRU_REFS_FLAGS */889 /* see the comment on LRU_REFS_FLAGS */
890 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {890 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
891 set_mask_bits(&folio->flags, LRU_REFS_MASK, BIT(PG_referenced));891 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
892 return false;892 return false;
893 }893 }
894 894
895 set_mask_bits(&folio->flags, LRU_REFS_FLAGS, BIT(PG_workingset));895 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
896 return true;896 return true;
897}897}
898#else898#else
@@ -3257,13 +3257,13 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) · 2358 unmodified lines
3257/* promote pages accessed through page tables */3257/* promote pages accessed through page tables */
3258static int folio_update_gen(struct folio *folio, int gen)3258static int folio_update_gen(struct folio *folio, int gen)
3259{3259{
3260 unsigned long new_flags, old_flags = READ_ONCE(folio->flags);3260 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
3261 3261
3262 VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);3262 VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
3263 3263
3264 /* see the comment on LRU_REFS_FLAGS */3264 /* see the comment on LRU_REFS_FLAGS */
3265 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {3265 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
3266 set_mask_bits(&folio->flags, LRU_REFS_MASK, BIT(PG_referenced));3266 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
3267 return -1;3267 return -1;
3268 }3268 }
3269 3269
@@ -3274,7 +3274,7 @@ static int folio_update_gen(struct folio *folio, int gen) · 4 unmodified lines
3274 3274
3275 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);3275 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
3276 new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset);3276 new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset);
3277 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));3277 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
3278 3278
3279 return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;3279 return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
3280}3280}
@@ -3285,7 +3285,7 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclai · 4 unmodified lines
3285 int type = folio_is_file_lru(folio);3285 int type = folio_is_file_lru(folio);
3286 struct lru_gen_folio *lrugen = &lruvec->lrugen;3286 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3287 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);3287 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
3288 unsigned long new_flags, old_flags = READ_ONCE(folio->flags);3288 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
3289 3289
3290 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);3290 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
3291 3291
@@ -3302,7 +3302,7 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclai · 10 unmodified lines
3302 /* for folio_end_writeback() */3302 /* for folio_end_writeback() */
3303 if (reclaiming)3303 if (reclaiming)
3304 new_flags |= BIT(PG_reclaim);3304 new_flags |= BIT(PG_reclaim);
3305 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));3305 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
3306 3306
3307 lru_gen_update_size(lruvec, folio, old_gen, new_gen);3307 lru_gen_update_size(lruvec, folio, old_gen, new_gen);
3308 3308
@@ -4553,7 +4553,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca · 1244 unmodified lines
4553 4553
4554 /* see the comment on LRU_REFS_FLAGS */4554 /* see the comment on LRU_REFS_FLAGS */
4555 if (!folio_test_referenced(folio))4555 if (!folio_test_referenced(folio))
4556 set_mask_bits(&folio->flags, LRU_REFS_MASK, 0);4556 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0);
4557 4557
4558 /* for shrink_folio_list() */4558 /* for shrink_folio_list() */
4559 folio_clear_reclaim(folio);4559 folio_clear_reclaim(folio);
@@ -4766,7 +4766,7 @@ retry: · 206 unmodified lines
4766 4766
4767 /* don't add rejected folios to the oldest generation */4767 /* don't add rejected folios to the oldest generation */
4768 if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])4768 if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
4769 set_mask_bits(&folio->flags, LRU_REFS_FLAGS, BIT(PG_active));4769 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active));
4770 }4770 }
4771 4771
4772 spin_lock_irq(&lruvec->lru_lock);4772 spin_lock_irq(&lruvec->lru_lock);
mm/workingset.c+1−1
@@ -318,7 +318,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow) · 317 unmodified lines
318 folio_set_workingset(folio);318 folio_set_workingset(folio);
319 mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);319 mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
320 } else320 } else
321 set_mask_bits(&folio->flags, LRU_REFS_MASK, (refs - 1UL) << LRU_REFS_PGOFF);321 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, (refs - 1UL) << LRU_REFS_PGOFF);
322unlock:322unlock:
323 rcu_read_unlock();323 rcu_read_unlock();
324}324}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t