History

mm/rmap: Convert rmap_walk() to take a folio

This ripples all the way through to every calling and called function from rmap.

Matthew Wilcox (Oracle) committed 4 years ago

Commit 2f031c61 parent e05b3459 files · +80 −99Browse files

Unified · Split

9 files
  1. include/linux/ksm.h +2 −2
  2. include/linux/rmap.h +5 −6
  3. mm/damon/paddr.c +7 −8
  4. mm/folio-compat.c +0 −7
  5. mm/huge_memory.c +1 −1
  6. mm/ksm.c +6 −6
  7. mm/migrate.c +4 −6
  8. mm/page_idle.c +3 −4
  9. mm/rmap.c +52 −59
include/linux/ksm.h+2−2
@@ -51,7 +51,7 @@ static inline void ksm_exit(struct mm_struct *mm) · 50 unmodified lines
51struct page *ksm_might_need_to_copy(struct page *page,51struct page *ksm_might_need_to_copy(struct page *page,
52 struct vm_area_struct *vma, unsigned long address);52 struct vm_area_struct *vma, unsigned long address);
53 53
54void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);54void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc);
55void folio_migrate_ksm(struct folio *newfolio, struct folio *folio);55void folio_migrate_ksm(struct folio *newfolio, struct folio *folio);
56 56
57#else /* !CONFIG_KSM */57#else /* !CONFIG_KSM */
@@ -78,7 +78,7 @@ static inline struct page *ksm_might_need_to_copy(struct page *page, · 20 unmodified lines
78 return page;78 return page;
79}79}
80 80
81static inline void rmap_walk_ksm(struct page *page,81static inline void rmap_walk_ksm(struct folio *folio,
82 struct rmap_walk_control *rwc)82 struct rmap_walk_control *rwc)
83{83{
84}84}
include/linux/rmap.h+5−6
@@ -266,7 +266,6 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked); · 265 unmodified lines
266/*266/*
267 * Called by memory-failure.c to kill processes.267 * Called by memory-failure.c to kill processes.
268 */268 */
269struct anon_vma *page_lock_anon_vma_read(struct page *page);
270struct anon_vma *folio_lock_anon_vma_read(struct folio *folio);269struct anon_vma *folio_lock_anon_vma_read(struct folio *folio);
271void page_unlock_anon_vma_read(struct anon_vma *anon_vma);270void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
272int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);271int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);
@@ -286,15 +285,15 @@ struct rmap_walk_control { · 13 unmodified lines
286 * Return false if page table scanning in rmap_walk should be stopped.285 * Return false if page table scanning in rmap_walk should be stopped.
287 * Otherwise, return true.286 * Otherwise, return true.
288 */287 */
289 bool (*rmap_one)(struct page *page, struct vm_area_struct *vma,288 bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma,
290 unsigned long addr, void *arg);289 unsigned long addr, void *arg);
291 int (*done)(struct page *page);290 int (*done)(struct folio *folio);
292 struct anon_vma *(*anon_lock)(struct page *page);291 struct anon_vma *(*anon_lock)(struct folio *folio);
293 bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);292 bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);
294};293};
295 294
296void rmap_walk(struct page *page, struct rmap_walk_control *rwc);295void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc);
297void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc);296void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc);
298 297
299#else /* !CONFIG_MMU */298#else /* !CONFIG_MMU */
300 299
mm/damon/paddr.c+7−8
@@ -16,10 +16,10 @@ · 15 unmodified lines
16#include "../internal.h"16#include "../internal.h"
17#include "prmtv-common.h"17#include "prmtv-common.h"
18 18
19static bool __damon_pa_mkold(struct page *page, struct vm_area_struct *vma,19static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma,
20 unsigned long addr, void *arg)20 unsigned long addr, void *arg)
21{21{
22 DEFINE_PAGE_VMA_WALK(pvmw, page, vma, addr, 0);22 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
23 23
24 while (page_vma_mapped_walk(&pvmw)) {24 while (page_vma_mapped_walk(&pvmw)) {
25 addr = pvmw.address;25 addr = pvmw.address;
@@ -37,7 +37,7 @@ static void damon_pa_mkold(unsigned long paddr) · 11 unmodified lines
37 struct page *page = damon_get_page(PHYS_PFN(paddr));37 struct page *page = damon_get_page(PHYS_PFN(paddr));
38 struct rmap_walk_control rwc = {38 struct rmap_walk_control rwc = {
39 .rmap_one = __damon_pa_mkold,39 .rmap_one = __damon_pa_mkold,
40 .anon_lock = page_lock_anon_vma_read,40 .anon_lock = folio_lock_anon_vma_read,
41 };41 };
42 bool need_lock;42 bool need_lock;
43 43
@@ -54,7 +54,7 @@ static void damon_pa_mkold(unsigned long paddr) · 10 unmodified lines
54 if (need_lock && !folio_trylock(folio))54 if (need_lock && !folio_trylock(folio))
55 goto out;55 goto out;
56 56
57 rmap_walk(&folio->page, &rwc);57 rmap_walk(folio, &rwc);
58 58
59 if (need_lock)59 if (need_lock)
60 folio_unlock(folio);60 folio_unlock(folio);
@@ -87,10 +87,9 @@ struct damon_pa_access_chk_result { · 26 unmodified lines
87 bool accessed;87 bool accessed;
88};88};
89 89
90static bool __damon_pa_young(struct page *page, struct vm_area_struct *vma,90static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma,
91 unsigned long addr, void *arg)91 unsigned long addr, void *arg)
92{92{
93 struct folio *folio = page_folio(page);
94 struct damon_pa_access_chk_result *result = arg;93 struct damon_pa_access_chk_result *result = arg;
95 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);94 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
96 95
@@ -133,7 +132,7 @@ static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz) · 36 unmodified lines
133 struct rmap_walk_control rwc = {132 struct rmap_walk_control rwc = {
134 .arg = &result,133 .arg = &result,
135 .rmap_one = __damon_pa_young,134 .rmap_one = __damon_pa_young,
136 .anon_lock = page_lock_anon_vma_read,135 .anon_lock = folio_lock_anon_vma_read,
137 };136 };
138 bool need_lock;137 bool need_lock;
139 138
@@ -156,7 +155,7 @@ static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz) · 16 unmodified lines
156 return NULL;155 return NULL;
157 }156 }
158 157
159 rmap_walk(&folio->page, &rwc);158 rmap_walk(folio, &rwc);
160 159
161 if (need_lock)160 if (need_lock)
162 folio_unlock(folio);161 folio_unlock(folio);
mm/folio-compat.c+0−7
@@ -164,10 +164,3 @@ void putback_lru_page(struct page *page) · 163 unmodified lines
164{164{
165 folio_putback_lru(page_folio(page));165 folio_putback_lru(page_folio(page));
166}166}
167
168#ifdef CONFIG_MMU
169struct anon_vma *page_lock_anon_vma_read(struct page *page)
170{
171 return folio_lock_anon_vma_read(page_folio(page));
172}
173#endif
mm/huge_memory.c+1−1
@@ -2572,7 +2572,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) · 2571 unmodified lines
2572 * The caller does not necessarily hold an mmap_lock that would2572 * The caller does not necessarily hold an mmap_lock that would
2573 * prevent the anon_vma disappearing so we first we take a2573 * prevent the anon_vma disappearing so we first we take a
2574 * reference to it and then lock the anon_vma for write. This2574 * reference to it and then lock the anon_vma for write. This
2575 * is similar to page_lock_anon_vma_read except the write lock2575 * is similar to folio_lock_anon_vma_read except the write lock
2576 * is taken to serialise against parallel split or collapse2576 * is taken to serialise against parallel split or collapse
2577 * operations.2577 * operations.
2578 */2578 */
mm/ksm.c+6−6
@@ -2588,21 +2588,21 @@ struct page *ksm_might_need_to_copy(struct page *page, · 2587 unmodified lines
2588 return new_page;2588 return new_page;
2589}2589}
2590 2590
2591void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)2591void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
2592{2592{
2593 struct stable_node *stable_node;2593 struct stable_node *stable_node;
2594 struct rmap_item *rmap_item;2594 struct rmap_item *rmap_item;
2595 int search_new_forks = 0;2595 int search_new_forks = 0;
2596 2596
2597 VM_BUG_ON_PAGE(!PageKsm(page), page);2597 VM_BUG_ON_FOLIO(!folio_test_ksm(folio), folio);
2598 2598
2599 /*2599 /*
2600 * Rely on the page lock to protect against concurrent modifications2600 * Rely on the page lock to protect against concurrent modifications
2601 * to that page's node of the stable tree.2601 * to that page's node of the stable tree.
2602 */2602 */
2603 VM_BUG_ON_PAGE(!PageLocked(page), page);2603 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2604 2604
2605 stable_node = page_stable_node(page);2605 stable_node = folio_stable_node(folio);
2606 if (!stable_node)2606 if (!stable_node)
2607 return;2607 return;
2608again:2608again:
@@ -2637,11 +2637,11 @@ again: · 28 unmodified lines
2637 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))2637 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2638 continue;2638 continue;
2639 2639
2640 if (!rwc->rmap_one(page, vma, addr, rwc->arg)) {2640 if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) {
2641 anon_vma_unlock_read(anon_vma);2641 anon_vma_unlock_read(anon_vma);
2642 return;2642 return;
2643 }2643 }
2644 if (rwc->done && rwc->done(page)) {2644 if (rwc->done && rwc->done(folio)) {
2645 anon_vma_unlock_read(anon_vma);2645 anon_vma_unlock_read(anon_vma);
2646 return;2646 return;
2647 }2647 }
mm/migrate.c+4−6
@@ -171,13 +171,11 @@ void putback_movable_pages(struct list_head *l) · 170 unmodified lines
171/*171/*
172 * Restore a potential migration pte to a working pte entry172 * Restore a potential migration pte to a working pte entry
173 */173 */
174static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,174static bool remove_migration_pte(struct folio *folio,
175 unsigned long addr, void *old)175 struct vm_area_struct *vma, unsigned long addr, void *old)
176{176{
177 struct folio *folio = page_folio(page);
178 DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION);177 DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION);
179 178
180 VM_BUG_ON_PAGE(PageTail(page), page);
181 while (page_vma_mapped_walk(&pvmw)) {179 while (page_vma_mapped_walk(&pvmw)) {
182 pte_t pte;180 pte_t pte;
183 swp_entry_t entry;181 swp_entry_t entry;
@@ -269,9 +267,9 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked) · 85 unmodified lines
269 };267 };
270 268
271 if (locked)269 if (locked)
272 rmap_walk_locked(&dst->page, &rwc);270 rmap_walk_locked(dst, &rwc);
273 else271 else
274 rmap_walk(&dst->page, &rwc);272 rmap_walk(dst, &rwc);
275}273}
276 274
277/*275/*
mm/page_idle.c+3−4
@@ -46,11 +46,10 @@ static struct page *page_idle_get_page(unsigned long pfn) · 45 unmodified lines
46 return page;46 return page;
47}47}
48 48
49static bool page_idle_clear_pte_refs_one(struct page *page,49static bool page_idle_clear_pte_refs_one(struct folio *folio,
50 struct vm_area_struct *vma,50 struct vm_area_struct *vma,
51 unsigned long addr, void *arg)51 unsigned long addr, void *arg)
52{52{
53 struct folio *folio = page_folio(page);
54 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);53 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
55 bool referenced = false;54 bool referenced = false;
56 55
@@ -93,7 +92,7 @@ static void page_idle_clear_pte_refs(struct page *page) · 36 unmodified lines
93 */92 */
94 static const struct rmap_walk_control rwc = {93 static const struct rmap_walk_control rwc = {
95 .rmap_one = page_idle_clear_pte_refs_one,94 .rmap_one = page_idle_clear_pte_refs_one,
96 .anon_lock = page_lock_anon_vma_read,95 .anon_lock = folio_lock_anon_vma_read,
97 };96 };
98 bool need_lock;97 bool need_lock;
99 98
@@ -104,7 +103,7 @@ static void page_idle_clear_pte_refs(struct page *page) · 4 unmodified lines
104 if (need_lock && !folio_trylock(folio))103 if (need_lock && !folio_trylock(folio))
105 return;104 return;
106 105
107 rmap_walk(&folio->page, (struct rmap_walk_control *)&rwc);106 rmap_walk(folio, (struct rmap_walk_control *)&rwc);
108 107
109 if (need_lock)108 if (need_lock)
110 folio_unlock(folio);109 folio_unlock(folio);
mm/rmap.c+52−59
@@ -107,15 +107,15 @@ static inline void anon_vma_free(struct anon_vma *anon_vma) · 106 unmodified lines
107 VM_BUG_ON(atomic_read(&anon_vma->refcount));107 VM_BUG_ON(atomic_read(&anon_vma->refcount));
108 108
109 /*109 /*
110 * Synchronize against page_lock_anon_vma_read() such that110 * Synchronize against folio_lock_anon_vma_read() such that
111 * we can safely hold the lock without the anon_vma getting111 * we can safely hold the lock without the anon_vma getting
112 * freed.112 * freed.
113 *113 *
114 * Relies on the full mb implied by the atomic_dec_and_test() from114 * Relies on the full mb implied by the atomic_dec_and_test() from
115 * put_anon_vma() against the acquire barrier implied by115 * put_anon_vma() against the acquire barrier implied by
116 * down_read_trylock() from page_lock_anon_vma_read(). This orders:116 * down_read_trylock() from folio_lock_anon_vma_read(). This orders:
117 *117 *
118 * page_lock_anon_vma_read() VS put_anon_vma()118 * folio_lock_anon_vma_read() VS put_anon_vma()
119 * down_read_trylock() atomic_dec_and_test()119 * down_read_trylock() atomic_dec_and_test()
120 * LOCK MB120 * LOCK MB
121 * atomic_read() rwsem_is_locked()121 * atomic_read() rwsem_is_locked()
@@ -168,7 +168,7 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, · 46 unmodified lines
168 * allocate a new one.168 * allocate a new one.
169 *169 *
170 * Anon-vma allocations are very subtle, because we may have170 * Anon-vma allocations are very subtle, because we may have
171 * optimistically looked up an anon_vma in page_lock_anon_vma_read()171 * optimistically looked up an anon_vma in folio_lock_anon_vma_read()
172 * and that may actually touch the rwsem even in the newly172 * and that may actually touch the rwsem even in the newly
173 * allocated vma (it depends on RCU to make sure that the173 * allocated vma (it depends on RCU to make sure that the
174 * anon_vma isn't actually destroyed).174 * anon_vma isn't actually destroyed).
@@ -799,10 +799,9 @@ struct folio_referenced_arg { · 624 unmodified lines
799/*799/*
800 * arg: folio_referenced_arg will be passed800 * arg: folio_referenced_arg will be passed
801 */801 */
802static bool folio_referenced_one(struct page *page, struct vm_area_struct *vma,802static bool folio_referenced_one(struct folio *folio,
803 unsigned long address, void *arg)803 struct vm_area_struct *vma, unsigned long address, void *arg)
804{804{
805 struct folio *folio = page_folio(page);
806 struct folio_referenced_arg *pra = arg;805 struct folio_referenced_arg *pra = arg;
807 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);806 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
808 int referenced = 0;807 int referenced = 0;
@@ -894,7 +893,7 @@ int folio_referenced(struct folio *folio, int is_locked, · 85 unmodified lines
894 struct rmap_walk_control rwc = {893 struct rmap_walk_control rwc = {
895 .rmap_one = folio_referenced_one,894 .rmap_one = folio_referenced_one,
896 .arg = (void *)&pra,895 .arg = (void *)&pra,
897 .anon_lock = page_lock_anon_vma_read,896 .anon_lock = folio_lock_anon_vma_read,
898 };897 };
899 898
900 *vm_flags = 0;899 *vm_flags = 0;
@@ -919,7 +918,7 @@ int folio_referenced(struct folio *folio, int is_locked, · 18 unmodified lines
919 rwc.invalid_vma = invalid_folio_referenced_vma;918 rwc.invalid_vma = invalid_folio_referenced_vma;
920 }919 }
921 920
922 rmap_walk(&folio->page, &rwc);921 rmap_walk(folio, &rwc);
923 *vm_flags = pra.vm_flags;922 *vm_flags = pra.vm_flags;
924 923
925 if (we_locked)924 if (we_locked)
@@ -928,10 +927,9 @@ int folio_referenced(struct folio *folio, int is_locked, · 2 unmodified lines
928 return pra.referenced;927 return pra.referenced;
929}928}
930 929
931static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,930static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma,
932 unsigned long address, void *arg)931 unsigned long address, void *arg)
933{932{
934 struct folio *folio = page_folio(page);
935 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);933 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
936 struct mmu_notifier_range range;934 struct mmu_notifier_range range;
937 int *cleaned = arg;935 int *cleaned = arg;
@@ -1025,7 +1023,7 @@ int folio_mkclean(struct folio *folio) · 87 unmodified lines
1025 if (!mapping)1023 if (!mapping)
1026 return 0;1024 return 0;
1027 1025
1028 rmap_walk(&folio->page, &rwc);1026 rmap_walk(folio, &rwc);
1029 1027
1030 return cleaned;1028 return cleaned;
1031}1029}
@@ -1410,10 +1408,9 @@ out: · 378 unmodified lines
1410/*1408/*
1411 * @arg: enum ttu_flags will be passed to this argument1409 * @arg: enum ttu_flags will be passed to this argument
1412 */1410 */
1413static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,1411static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
1414 unsigned long address, void *arg)1412 unsigned long address, void *arg)
1415{1413{
1416 struct folio *folio = page_folio(page);
1417 struct mm_struct *mm = vma->vm_mm;1414 struct mm_struct *mm = vma->vm_mm;
1418 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);1415 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1419 pte_t pteval;1416 pte_t pteval;
@@ -1667,9 +1664,9 @@ static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) · 247 unmodified lines
1667 return vma_is_temporary_stack(vma);1664 return vma_is_temporary_stack(vma);
1668}1665}
1669 1666
1670static int page_not_mapped(struct page *page)1667static int page_not_mapped(struct folio *folio)
1671{1668{
1672 return !page_mapped(page);1669 return !folio_mapped(folio);
1673}1670}
1674 1671
1675/**1672/**
@@ -1689,13 +1686,13 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) · 13 unmodified lines
1689 .rmap_one = try_to_unmap_one,1686 .rmap_one = try_to_unmap_one,
1690 .arg = (void *)flags,1687 .arg = (void *)flags,
1691 .done = page_not_mapped,1688 .done = page_not_mapped,
1692 .anon_lock = page_lock_anon_vma_read,1689 .anon_lock = folio_lock_anon_vma_read,
1693 };1690 };
1694 1691
1695 if (flags & TTU_RMAP_LOCKED)1692 if (flags & TTU_RMAP_LOCKED)
1696 rmap_walk_locked(&folio->page, &rwc);1693 rmap_walk_locked(folio, &rwc);
1697 else1694 else
1698 rmap_walk(&folio->page, &rwc);1695 rmap_walk(folio, &rwc);
1699}1696}
1700 1697
1701/*1698/*
@@ -1704,10 +1701,9 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) · 2 unmodified lines
1704 * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs1701 * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
1705 * containing migration entries.1702 * containing migration entries.
1706 */1703 */
1707static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,1704static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
1708 unsigned long address, void *arg)1705 unsigned long address, void *arg)
1709{1706{
1710 struct folio *folio = page_folio(page);
1711 struct mm_struct *mm = vma->vm_mm;1707 struct mm_struct *mm = vma->vm_mm;
1712 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);1708 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1713 pte_t pteval;1709 pte_t pteval;
@@ -1951,7 +1947,7 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags) · 237 unmodified lines
1951 .rmap_one = try_to_migrate_one,1947 .rmap_one = try_to_migrate_one,
1952 .arg = (void *)flags,1948 .arg = (void *)flags,
1953 .done = page_not_mapped,1949 .done = page_not_mapped,
1954 .anon_lock = page_lock_anon_vma_read,1950 .anon_lock = folio_lock_anon_vma_read,
1955 };1951 };
1956 1952
1957 /*1953 /*
@@ -1977,9 +1973,9 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags) · 19 unmodified lines
1977 rwc.invalid_vma = invalid_migration_vma;1973 rwc.invalid_vma = invalid_migration_vma;
1978 1974
1979 if (flags & TTU_RMAP_LOCKED)1975 if (flags & TTU_RMAP_LOCKED)
1980 rmap_walk_locked(&folio->page, &rwc);1976 rmap_walk_locked(folio, &rwc);
1981 else1977 else
1982 rmap_walk(&folio->page, &rwc);1978 rmap_walk(folio, &rwc);
1983}1979}
1984 1980
1985#ifdef CONFIG_DEVICE_PRIVATE1981#ifdef CONFIG_DEVICE_PRIVATE
@@ -1990,10 +1986,9 @@ struct make_exclusive_args { · 4 unmodified lines
1990 bool valid;1986 bool valid;
1991};1987};
1992 1988
1993static bool page_make_device_exclusive_one(struct page *page,1989static bool page_make_device_exclusive_one(struct folio *folio,
1994 struct vm_area_struct *vma, unsigned long address, void *priv)1990 struct vm_area_struct *vma, unsigned long address, void *priv)
1995{1991{
1996 struct folio *folio = page_folio(page);
1997 struct mm_struct *mm = vma->vm_mm;1992 struct mm_struct *mm = vma->vm_mm;
1998 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);1993 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1999 struct make_exclusive_args *args = priv;1994 struct make_exclusive_args *args = priv;
@@ -2098,7 +2093,7 @@ static bool folio_make_device_exclusive(struct folio *folio, · 98 unmodified lines
2098 struct rmap_walk_control rwc = {2093 struct rmap_walk_control rwc = {
2099 .rmap_one = page_make_device_exclusive_one,2094 .rmap_one = page_make_device_exclusive_one,
2100 .done = page_not_mapped,2095 .done = page_not_mapped,
2101 .anon_lock = page_lock_anon_vma_read,2096 .anon_lock = folio_lock_anon_vma_read,
2102 .arg = &args,2097 .arg = &args,
2103 };2098 };
2104 2099
@@ -2109,7 +2104,7 @@ static bool folio_make_device_exclusive(struct folio *folio, · 4 unmodified lines
2109 if (!folio_test_anon(folio))2104 if (!folio_test_anon(folio))
2110 return false;2105 return false;
2111 2106
2112 rmap_walk(&folio->page, &rwc);2107 rmap_walk(folio, &rwc);
2113 2108
2114 return args.valid && !folio_mapcount(folio);2109 return args.valid && !folio_mapcount(folio);
2115}2110}
@@ -2177,17 +2172,16 @@ void __put_anon_vma(struct anon_vma *anon_vma) · 61 unmodified lines
2177 anon_vma_free(root);2172 anon_vma_free(root);
2178}2173}
2179 2174
2180static struct anon_vma *rmap_walk_anon_lock(struct page *page,2175static struct anon_vma *rmap_walk_anon_lock(struct folio *folio,
2181 struct rmap_walk_control *rwc)2176 struct rmap_walk_control *rwc)
2182{2177{
2183 struct folio *folio = page_folio(page);
2184 struct anon_vma *anon_vma;2178 struct anon_vma *anon_vma;
2185 2179
2186 if (rwc->anon_lock)2180 if (rwc->anon_lock)
2187 return rwc->anon_lock(page);2181 return rwc->anon_lock(folio);
2188 2182
2189 /*2183 /*
2190 * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read()2184 * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
2191 * because that depends on page_mapped(); but not all its usages2185 * because that depends on page_mapped(); but not all its usages
2192 * are holding mmap_lock. Users without mmap_lock are required to2186 * are holding mmap_lock. Users without mmap_lock are required to
2193 * take a reference count to prevent the anon_vma disappearing2187 * take a reference count to prevent the anon_vma disappearing
@@ -2209,10 +2203,9 @@ static struct anon_vma *rmap_walk_anon_lock(struct page *page, · 15 unmodified lines
2209 * Find all the mappings of a page using the mapping pointer and the vma chains2203 * Find all the mappings of a page using the mapping pointer and the vma chains
2210 * contained in the anon_vma struct it points to.2204 * contained in the anon_vma struct it points to.
2211 */2205 */
2212static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc,2206static void rmap_walk_anon(struct folio *folio, struct rmap_walk_control *rwc,
2213 bool locked)2207 bool locked)
2214{2208{
2215 struct folio *folio = page_folio(page);
2216 struct anon_vma *anon_vma;2209 struct anon_vma *anon_vma;
2217 pgoff_t pgoff_start, pgoff_end;2210 pgoff_t pgoff_start, pgoff_end;
2218 struct anon_vma_chain *avc;2211 struct anon_vma_chain *avc;
@@ -2222,17 +2215,17 @@ static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, · 3 unmodified lines
2222 /* anon_vma disappear under us? */2215 /* anon_vma disappear under us? */
2223 VM_BUG_ON_FOLIO(!anon_vma, folio);2216 VM_BUG_ON_FOLIO(!anon_vma, folio);
2224 } else {2217 } else {
2225 anon_vma = rmap_walk_anon_lock(page, rwc);2218 anon_vma = rmap_walk_anon_lock(folio, rwc);
2226 }2219 }
2227 if (!anon_vma)2220 if (!anon_vma)
2228 return;2221 return;
2229 2222
2230 pgoff_start = page_to_pgoff(page);2223 pgoff_start = folio_pgoff(folio);
2231 pgoff_end = pgoff_start + thp_nr_pages(page) - 1;2224 pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
2232 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,2225 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
2233 pgoff_start, pgoff_end) {2226 pgoff_start, pgoff_end) {
2234 struct vm_area_struct *vma = avc->vma;2227 struct vm_area_struct *vma = avc->vma;
2235 unsigned long address = vma_address(page, vma);2228 unsigned long address = vma_address(&folio->page, vma);
2236 2229
2237 VM_BUG_ON_VMA(address == -EFAULT, vma);2230 VM_BUG_ON_VMA(address == -EFAULT, vma);
2238 cond_resched();2231 cond_resched();
@@ -2240,9 +2233,9 @@ static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, · 1 unmodified line
2240 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))2233 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2241 continue;2234 continue;
2242 2235
2243 if (!rwc->rmap_one(page, vma, address, rwc->arg))2236 if (!rwc->rmap_one(folio, vma, address, rwc->arg))
2244 break;2237 break;
2245 if (rwc->done && rwc->done(page))2238 if (rwc->done && rwc->done(folio))
2246 break;2239 break;
2247 }2240 }
2248 2241
@@ -2258,10 +2251,10 @@ static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, · 9 unmodified lines
2258 * Find all the mappings of a page using the mapping pointer and the vma chains2251 * Find all the mappings of a page using the mapping pointer and the vma chains
2259 * contained in the address_space struct it points to.2252 * contained in the address_space struct it points to.
2260 */2253 */
2261static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc,2254static void rmap_walk_file(struct folio *folio, struct rmap_walk_control *rwc,
2262 bool locked)2255 bool locked)
2263{2256{
2264 struct address_space *mapping = page_mapping(page);2257 struct address_space *mapping = folio_mapping(folio);
2265 pgoff_t pgoff_start, pgoff_end;2258 pgoff_t pgoff_start, pgoff_end;
2266 struct vm_area_struct *vma;2259 struct vm_area_struct *vma;
2267 2260
@@ -2271,18 +2264,18 @@ static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, · 3 unmodified lines
2271 * structure at mapping cannot be freed and reused yet,2264 * structure at mapping cannot be freed and reused yet,
2272 * so we can safely take mapping->i_mmap_rwsem.2265 * so we can safely take mapping->i_mmap_rwsem.
2273 */2266 */
2274 VM_BUG_ON_PAGE(!PageLocked(page), page);2267 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2275 2268
2276 if (!mapping)2269 if (!mapping)
2277 return;2270 return;
2278 2271
2279 pgoff_start = page_to_pgoff(page);2272 pgoff_start = folio_pgoff(folio);
2280 pgoff_end = pgoff_start + thp_nr_pages(page) - 1;2273 pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
2281 if (!locked)2274 if (!locked)
2282 i_mmap_lock_read(mapping);2275 i_mmap_lock_read(mapping);
2283 vma_interval_tree_foreach(vma, &mapping->i_mmap,2276 vma_interval_tree_foreach(vma, &mapping->i_mmap,
2284 pgoff_start, pgoff_end) {2277 pgoff_start, pgoff_end) {
2285 unsigned long address = vma_address(page, vma);2278 unsigned long address = vma_address(&folio->page, vma);
2286 2279
2287 VM_BUG_ON_VMA(address == -EFAULT, vma);2280 VM_BUG_ON_VMA(address == -EFAULT, vma);
2288 cond_resched();2281 cond_resched();
@@ -2290,9 +2283,9 @@ static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, · 1 unmodified line
2290 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))2283 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2291 continue;2284 continue;
2292 2285
2293 if (!rwc->rmap_one(page, vma, address, rwc->arg))2286 if (!rwc->rmap_one(folio, vma, address, rwc->arg))
2294 goto done;2287 goto done;
2295 if (rwc->done && rwc->done(page))2288 if (rwc->done && rwc->done(folio))
2296 goto done;2289 goto done;
2297 }2290 }
2298 2291
@@ -2301,25 +2294,25 @@ done: · 2 unmodified lines
2301 i_mmap_unlock_read(mapping);2294 i_mmap_unlock_read(mapping);
2302}2295}
2303 2296
2304void rmap_walk(struct page *page, struct rmap_walk_control *rwc)2297void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc)
2305{2298{
2306 if (unlikely(PageKsm(page)))2299 if (unlikely(folio_test_ksm(folio)))
2307 rmap_walk_ksm(page, rwc);2300 rmap_walk_ksm(folio, rwc);
2308 else if (PageAnon(page))2301 else if (folio_test_anon(folio))
2309 rmap_walk_anon(page, rwc, false);2302 rmap_walk_anon(folio, rwc, false);
2310 else2303 else
2311 rmap_walk_file(page, rwc, false);2304 rmap_walk_file(folio, rwc, false);
2312}2305}
2313 2306
2314/* Like rmap_walk, but caller holds relevant rmap lock */2307/* Like rmap_walk, but caller holds relevant rmap lock */
2315void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc)2308void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc)
2316{2309{
2317 /* no ksm support for now */2310 /* no ksm support for now */
2318 VM_BUG_ON_PAGE(PageKsm(page), page);2311 VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
2319 if (PageAnon(page))2312 if (folio_test_anon(folio))
2320 rmap_walk_anon(page, rwc, true);2313 rmap_walk_anon(folio, rwc, true);
2321 else2314 else
2322 rmap_walk_file(page, rwc, true);2315 rmap_walk_file(folio, rwc, true);
2323}2316}
2324 2317
2325#ifdef CONFIG_HUGETLB_PAGE2318#ifdef CONFIG_HUGETLB_PAGE

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t