mm/rmap: Convert rmap_walk() to take a folio

This ripples all the way through to every calling and called function from rmap.

Matthew Wilcox (Oracle) committed 4 years ago

Commit 2f031c61 parent e05b3459 files · +80 −99Browse files

Unified · Split

9 files

include/linux/ksm.h +2 −2
include/linux/rmap.h +5 −6
mm/damon/paddr.c +7 −8
mm/folio-compat.c +0 −7
mm/huge_memory.c +1 −1
mm/ksm.c +6 −6
mm/migrate.c +4 −6
mm/page_idle.c +3 −4
mm/rmap.c +52 −59

@@ -51,7 +51,7 @@ static inline void ksm_exit(struct mm_struct *mm) · 50 unmodified lines
51	51	struct page ksm_might_need_to_copy(struct page page,
52	52	struct vm_area_struct *vma, unsigned long address);
53	53
54		−void rmap_walk_ksm(struct page page, struct rmap_walk_control rwc);
	54	+void rmap_walk_ksm(struct folio folio, struct rmap_walk_control rwc);
55	55	void folio_migrate_ksm(struct folio newfolio, struct folio folio);
56	56
57	57	#else /* !CONFIG_KSM */
@@ -78,7 +78,7 @@ static inline struct page ksm_might_need_to_copy(struct page page, · 20 unmodified lines
78	78	return page;
79	79	}
80	80
81		−static inline void rmap_walk_ksm(struct page *page,
	81	+static inline void rmap_walk_ksm(struct folio *folio,
82	82	struct rmap_walk_control *rwc)
83	83	{
84	84	}

include/linux/rmap.h+5−6

@@ -266,7 +266,6 @@ void remove_migration_ptes(struct folio src, struct folio dst, bool locked); · 265 unmodified lines
266	266	/*
267	267	* Called by memory-failure.c to kill processes.
268	268	*/
269		−struct anon_vma page_lock_anon_vma_read(struct page page);
270	269	struct anon_vma folio_lock_anon_vma_read(struct folio folio);
271	270	void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
272	271	int page_mapped_in_vma(struct page page, struct vm_area_struct vma);
@@ -286,15 +285,15 @@ struct rmap_walk_control { · 13 unmodified lines
286	285	* Return false if page table scanning in rmap_walk should be stopped.
287	286	* Otherwise, return true.
288	287	*/
289		− bool (rmap_one)(struct page page, struct vm_area_struct *vma,
	288	+ bool (rmap_one)(struct folio folio, struct vm_area_struct *vma,
290	289	unsigned long addr, void *arg);
291		− int (done)(struct page page);
292		− struct anon_vma (anon_lock)(struct page *page);
	290	+ int (done)(struct folio folio);
	291	+ struct anon_vma (anon_lock)(struct folio *folio);
293	292	bool (invalid_vma)(struct vm_area_struct vma, void *arg);
294	293	};
295	294
296		−void rmap_walk(struct page page, struct rmap_walk_control rwc);
297		−void rmap_walk_locked(struct page page, struct rmap_walk_control rwc);
	295	+void rmap_walk(struct folio folio, struct rmap_walk_control rwc);
	296	+void rmap_walk_locked(struct folio folio, struct rmap_walk_control rwc);
298	297
299	298	#else /* !CONFIG_MMU */
300	299

mm/damon/paddr.c+7−8

@@ -16,10 +16,10 @@ · 15 unmodified lines
16	16	#include "../internal.h"
17	17	#include "prmtv-common.h"
18	18
19		−static bool __damon_pa_mkold(struct page page, struct vm_area_struct vma,
	19	+static bool __damon_pa_mkold(struct folio folio, struct vm_area_struct vma,
20	20	unsigned long addr, void *arg)
21	21	{
22		− DEFINE_PAGE_VMA_WALK(pvmw, page, vma, addr, 0);
	22	+ DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
23	23
24	24	while (page_vma_mapped_walk(&pvmw)) {
25	25	addr = pvmw.address;
@@ -37,7 +37,7 @@ static void damon_pa_mkold(unsigned long paddr) · 11 unmodified lines
37	37	struct page *page = damon_get_page(PHYS_PFN(paddr));
38	38	struct rmap_walk_control rwc = {
39	39	.rmap_one = __damon_pa_mkold,
40		− .anon_lock = page_lock_anon_vma_read,
	40	+ .anon_lock = folio_lock_anon_vma_read,
41	41	};
42	42	bool need_lock;
43	43
@@ -54,7 +54,7 @@ static void damon_pa_mkold(unsigned long paddr) · 10 unmodified lines
54	54	if (need_lock && !folio_trylock(folio))
55	55	goto out;
56	56
57		− rmap_walk(&folio->page, &rwc);
	57	+ rmap_walk(folio, &rwc);
58	58
59	59	if (need_lock)
60	60	folio_unlock(folio);
@@ -87,10 +87,9 @@ struct damon_pa_access_chk_result { · 26 unmodified lines
87	87	bool accessed;
88	88	};
89	89
90		−static bool __damon_pa_young(struct page page, struct vm_area_struct vma,
	90	+static bool __damon_pa_young(struct folio folio, struct vm_area_struct vma,
91	91	unsigned long addr, void *arg)
92	92	{
93		− struct folio *folio = page_folio(page);
94	93	struct damon_pa_access_chk_result *result = arg;
95	94	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
96	95
@@ -133,7 +132,7 @@ static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz) · 36 unmodified lines
133	132	struct rmap_walk_control rwc = {
134	133	.arg = &result,
135	134	.rmap_one = __damon_pa_young,
136		− .anon_lock = page_lock_anon_vma_read,
	135	+ .anon_lock = folio_lock_anon_vma_read,
137	136	};
138	137	bool need_lock;
139	138
@@ -156,7 +155,7 @@ static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz) · 16 unmodified lines
156	155	return NULL;
157	156	}
158	157
159		− rmap_walk(&folio->page, &rwc);
	158	+ rmap_walk(folio, &rwc);
160	159
161	160	if (need_lock)
162	161	folio_unlock(folio);

mm/folio-compat.c+0−7

@@ -164,10 +164,3 @@ void putback_lru_page(struct page *page) · 163 unmodified lines
164	164	{
165	165	folio_putback_lru(page_folio(page));
166	166	}
167		−
168		−#ifdef CONFIG_MMU
169		−struct anon_vma page_lock_anon_vma_read(struct page page)
170		−{
171		− return folio_lock_anon_vma_read(page_folio(page));
172		−}
173		−#endif

mm/huge_memory.c+1−1

@@ -2572,7 +2572,7 @@ int split_huge_page_to_list(struct page page, struct list_head list) · 2571 unmodified lines
2572	2572	* The caller does not necessarily hold an mmap_lock that would
2573	2573	* prevent the anon_vma disappearing so we first we take a
2574	2574	* reference to it and then lock the anon_vma for write. This
2575		− * is similar to page_lock_anon_vma_read except the write lock
	2575	+ * is similar to folio_lock_anon_vma_read except the write lock
2576	2576	* is taken to serialise against parallel split or collapse
2577	2577	* operations.
2578	2578	*/

mm/ksm.c+6−6

@@ -2588,21 +2588,21 @@ struct page ksm_might_need_to_copy(struct page page, · 2587 unmodified lines
2588	2588	return new_page;
2589	2589	}
2590	2590
2591		−void rmap_walk_ksm(struct page page, struct rmap_walk_control rwc)
	2591	+void rmap_walk_ksm(struct folio folio, struct rmap_walk_control rwc)
2592	2592	{
2593	2593	struct stable_node *stable_node;
2594	2594	struct rmap_item *rmap_item;
2595	2595	int search_new_forks = 0;
2596	2596
2597		− VM_BUG_ON_PAGE(!PageKsm(page), page);
	2597	+ VM_BUG_ON_FOLIO(!folio_test_ksm(folio), folio);
2598	2598
2599	2599	/*
2600	2600	* Rely on the page lock to protect against concurrent modifications
2601	2601	* to that page's node of the stable tree.
2602	2602	*/
2603		− VM_BUG_ON_PAGE(!PageLocked(page), page);
	2603	+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2604	2604
2605		− stable_node = page_stable_node(page);
	2605	+ stable_node = folio_stable_node(folio);
2606	2606	if (!stable_node)
2607	2607	return;
2608	2608	again:
@@ -2637,11 +2637,11 @@ again: · 28 unmodified lines
2637	2637	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2638	2638	continue;
2639	2639
2640		− if (!rwc->rmap_one(page, vma, addr, rwc->arg)) {
	2640	+ if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) {
2641	2641	anon_vma_unlock_read(anon_vma);
2642	2642	return;
2643	2643	}
2644		− if (rwc->done && rwc->done(page)) {
	2644	+ if (rwc->done && rwc->done(folio)) {
2645	2645	anon_vma_unlock_read(anon_vma);
2646	2646	return;
2647	2647	}

mm/migrate.c+4−6

@@ -171,13 +171,11 @@ void putback_movable_pages(struct list_head *l) · 170 unmodified lines
171	171	/*
172	172	* Restore a potential migration pte to a working pte entry
173	173	*/
174		−static bool remove_migration_pte(struct page page, struct vm_area_struct vma,
175		− unsigned long addr, void *old)
	174	+static bool remove_migration_pte(struct folio *folio,
	175	+ struct vm_area_struct vma, unsigned long addr, void old)
176	176	{
177		− struct folio *folio = page_folio(page);
178	177	DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC \| PVMW_MIGRATION);
179	178
180		− VM_BUG_ON_PAGE(PageTail(page), page);
181	179	while (page_vma_mapped_walk(&pvmw)) {
182	180	pte_t pte;
183	181	swp_entry_t entry;
@@ -269,9 +267,9 @@ void remove_migration_ptes(struct folio src, struct folio dst, bool locked) · 85 unmodified lines
269	267	};
270	268
271	269	if (locked)
272		− rmap_walk_locked(&dst->page, &rwc);
	270	+ rmap_walk_locked(dst, &rwc);
273	271	else
274		− rmap_walk(&dst->page, &rwc);
	272	+ rmap_walk(dst, &rwc);
275	273	}
276	274
277	275	/*

mm/page_idle.c+3−4

@@ -46,11 +46,10 @@ static struct page *page_idle_get_page(unsigned long pfn) · 45 unmodified lines
46	46	return page;
47	47	}
48	48
49		−static bool page_idle_clear_pte_refs_one(struct page *page,
	49	+static bool page_idle_clear_pte_refs_one(struct folio *folio,
50	50	struct vm_area_struct *vma,
51	51	unsigned long addr, void *arg)
52	52	{
53		− struct folio *folio = page_folio(page);
54	53	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
55	54	bool referenced = false;
56	55
@@ -93,7 +92,7 @@ static void page_idle_clear_pte_refs(struct page *page) · 36 unmodified lines
93	92	*/
94	93	static const struct rmap_walk_control rwc = {
95	94	.rmap_one = page_idle_clear_pte_refs_one,
96		− .anon_lock = page_lock_anon_vma_read,
	95	+ .anon_lock = folio_lock_anon_vma_read,
97	96	};
98	97	bool need_lock;
99	98
@@ -104,7 +103,7 @@ static void page_idle_clear_pte_refs(struct page *page) · 4 unmodified lines
104	103	if (need_lock && !folio_trylock(folio))
105	104	return;
106	105
107		− rmap_walk(&folio->page, (struct rmap_walk_control *)&rwc);
	106	+ rmap_walk(folio, (struct rmap_walk_control *)&rwc);
108	107
109	108	if (need_lock)
110	109	folio_unlock(folio);

mm/rmap.c+52−59

@@ -107,15 +107,15 @@ static inline void anon_vma_free(struct anon_vma *anon_vma) · 106 unmodified lines
107	107	VM_BUG_ON(atomic_read(&anon_vma->refcount));
108	108
109	109	/*
110		− * Synchronize against page_lock_anon_vma_read() such that
	110	+ * Synchronize against folio_lock_anon_vma_read() such that
111	111	* we can safely hold the lock without the anon_vma getting
112	112	* freed.
113	113	*
114	114	* Relies on the full mb implied by the atomic_dec_and_test() from
115	115	* put_anon_vma() against the acquire barrier implied by
116		− * down_read_trylock() from page_lock_anon_vma_read(). This orders:
	116	+ * down_read_trylock() from folio_lock_anon_vma_read(). This orders:
117	117	*
118		− * page_lock_anon_vma_read() VS put_anon_vma()
	118	+ * folio_lock_anon_vma_read() VS put_anon_vma()
119	119	* down_read_trylock() atomic_dec_and_test()
120	120	* LOCK MB
121	121	* atomic_read() rwsem_is_locked()
@@ -168,7 +168,7 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, · 46 unmodified lines
168	168	* allocate a new one.
169	169	*
170	170	* Anon-vma allocations are very subtle, because we may have
171		− * optimistically looked up an anon_vma in page_lock_anon_vma_read()
	171	+ * optimistically looked up an anon_vma in folio_lock_anon_vma_read()
172	172	* and that may actually touch the rwsem even in the newly
173	173	* allocated vma (it depends on RCU to make sure that the
174	174	* anon_vma isn't actually destroyed).
@@ -799,10 +799,9 @@ struct folio_referenced_arg { · 624 unmodified lines
799	799	/*
800	800	* arg: folio_referenced_arg will be passed
801	801	*/
802		−static bool folio_referenced_one(struct page page, struct vm_area_struct vma,
803		− unsigned long address, void *arg)
	802	+static bool folio_referenced_one(struct folio *folio,
	803	+ struct vm_area_struct vma, unsigned long address, void arg)
804	804	{
805		− struct folio *folio = page_folio(page);
806	805	struct folio_referenced_arg *pra = arg;
807	806	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
808	807	int referenced = 0;
@@ -894,7 +893,7 @@ int folio_referenced(struct folio *folio, int is_locked, · 85 unmodified lines
894	893	struct rmap_walk_control rwc = {
895	894	.rmap_one = folio_referenced_one,
896	895	.arg = (void *)&pra,
897		− .anon_lock = page_lock_anon_vma_read,
	896	+ .anon_lock = folio_lock_anon_vma_read,
898	897	};
899	898
900	899	*vm_flags = 0;
@@ -919,7 +918,7 @@ int folio_referenced(struct folio *folio, int is_locked, · 18 unmodified lines
919	918	rwc.invalid_vma = invalid_folio_referenced_vma;
920	919	}
921	920
922		− rmap_walk(&folio->page, &rwc);
	921	+ rmap_walk(folio, &rwc);
923	922	*vm_flags = pra.vm_flags;
924	923
925	924	if (we_locked)
@@ -928,10 +927,9 @@ int folio_referenced(struct folio *folio, int is_locked, · 2 unmodified lines
928	927	return pra.referenced;
929	928	}
930	929
931		−static bool page_mkclean_one(struct page page, struct vm_area_struct vma,
	930	+static bool page_mkclean_one(struct folio folio, struct vm_area_struct vma,
932	931	unsigned long address, void *arg)
933	932	{
934		− struct folio *folio = page_folio(page);
935	933	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
936	934	struct mmu_notifier_range range;
937	935	int *cleaned = arg;
@@ -1025,7 +1023,7 @@ int folio_mkclean(struct folio *folio) · 87 unmodified lines
1025	1023	if (!mapping)
1026	1024	return 0;
1027	1025
1028		− rmap_walk(&folio->page, &rwc);
	1026	+ rmap_walk(folio, &rwc);
1029	1027
1030	1028	return cleaned;
1031	1029	}
@@ -1410,10 +1408,9 @@ out: · 378 unmodified lines
1410	1408	/*
1411	1409	* @arg: enum ttu_flags will be passed to this argument
1412	1410	*/
1413		−static bool try_to_unmap_one(struct page page, struct vm_area_struct vma,
	1411	+static bool try_to_unmap_one(struct folio folio, struct vm_area_struct vma,
1414	1412	unsigned long address, void *arg)
1415	1413	{
1416		− struct folio *folio = page_folio(page);
1417	1414	struct mm_struct *mm = vma->vm_mm;
1418	1415	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1419	1416	pte_t pteval;
@@ -1667,9 +1664,9 @@ static bool invalid_migration_vma(struct vm_area_struct vma, void arg) · 247 unmodified lines
1667	1664	return vma_is_temporary_stack(vma);
1668	1665	}
1669	1666
1670		−static int page_not_mapped(struct page *page)
	1667	+static int page_not_mapped(struct folio *folio)
1671	1668	{
1672		− return !page_mapped(page);
	1669	+ return !folio_mapped(folio);
1673	1670	}
1674	1671
1675	1672	/**
@@ -1689,13 +1686,13 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) · 13 unmodified lines
1689	1686	.rmap_one = try_to_unmap_one,
1690	1687	.arg = (void *)flags,
1691	1688	.done = page_not_mapped,
1692		− .anon_lock = page_lock_anon_vma_read,
	1689	+ .anon_lock = folio_lock_anon_vma_read,
1693	1690	};
1694	1691
1695	1692	if (flags & TTU_RMAP_LOCKED)
1696		− rmap_walk_locked(&folio->page, &rwc);
	1693	+ rmap_walk_locked(folio, &rwc);
1697	1694	else
1698		− rmap_walk(&folio->page, &rwc);
	1695	+ rmap_walk(folio, &rwc);
1699	1696	}
1700	1697
1701	1698	/*
@@ -1704,10 +1701,9 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) · 2 unmodified lines
1704	1701	* If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
1705	1702	* containing migration entries.
1706	1703	*/
1707		−static bool try_to_migrate_one(struct page page, struct vm_area_struct vma,
	1704	+static bool try_to_migrate_one(struct folio folio, struct vm_area_struct vma,
1708	1705	unsigned long address, void *arg)
1709	1706	{
1710		− struct folio *folio = page_folio(page);
1711	1707	struct mm_struct *mm = vma->vm_mm;
1712	1708	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1713	1709	pte_t pteval;
@@ -1951,7 +1947,7 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags) · 237 unmodified lines
1951	1947	.rmap_one = try_to_migrate_one,
1952	1948	.arg = (void *)flags,
1953	1949	.done = page_not_mapped,
1954		− .anon_lock = page_lock_anon_vma_read,
	1950	+ .anon_lock = folio_lock_anon_vma_read,
1955	1951	};
1956	1952
1957	1953	/*
@@ -1977,9 +1973,9 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags) · 19 unmodified lines
1977	1973	rwc.invalid_vma = invalid_migration_vma;
1978	1974
1979	1975	if (flags & TTU_RMAP_LOCKED)
1980		− rmap_walk_locked(&folio->page, &rwc);
	1976	+ rmap_walk_locked(folio, &rwc);
1981	1977	else
1982		− rmap_walk(&folio->page, &rwc);
	1978	+ rmap_walk(folio, &rwc);
1983	1979	}
1984	1980
1985	1981	#ifdef CONFIG_DEVICE_PRIVATE
@@ -1990,10 +1986,9 @@ struct make_exclusive_args { · 4 unmodified lines
1990	1986	bool valid;
1991	1987	};
1992	1988
1993		−static bool page_make_device_exclusive_one(struct page *page,
	1989	+static bool page_make_device_exclusive_one(struct folio *folio,
1994	1990	struct vm_area_struct vma, unsigned long address, void priv)
1995	1991	{
1996		− struct folio *folio = page_folio(page);
1997	1992	struct mm_struct *mm = vma->vm_mm;
1998	1993	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
1999	1994	struct make_exclusive_args *args = priv;
@@ -2098,7 +2093,7 @@ static bool folio_make_device_exclusive(struct folio *folio, · 98 unmodified lines
2098	2093	struct rmap_walk_control rwc = {
2099	2094	.rmap_one = page_make_device_exclusive_one,
2100	2095	.done = page_not_mapped,
2101		− .anon_lock = page_lock_anon_vma_read,
	2096	+ .anon_lock = folio_lock_anon_vma_read,
2102	2097	.arg = &args,
2103	2098	};
2104	2099
@@ -2109,7 +2104,7 @@ static bool folio_make_device_exclusive(struct folio *folio, · 4 unmodified lines
2109	2104	if (!folio_test_anon(folio))
2110	2105	return false;
2111	2106
2112		− rmap_walk(&folio->page, &rwc);
	2107	+ rmap_walk(folio, &rwc);
2113	2108
2114	2109	return args.valid && !folio_mapcount(folio);
2115	2110	}
@@ -2177,17 +2172,16 @@ void __put_anon_vma(struct anon_vma *anon_vma) · 61 unmodified lines
2177	2172	anon_vma_free(root);
2178	2173	}
2179	2174
2180		−static struct anon_vma rmap_walk_anon_lock(struct page page,
	2175	+static struct anon_vma rmap_walk_anon_lock(struct folio folio,
2181	2176	struct rmap_walk_control *rwc)
2182	2177	{
2183		− struct folio *folio = page_folio(page);
2184	2178	struct anon_vma *anon_vma;
2185	2179
2186	2180	if (rwc->anon_lock)
2187		− return rwc->anon_lock(page);
	2181	+ return rwc->anon_lock(folio);
2188	2182
2189	2183	/*
2190		− * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read()
	2184	+ * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
2191	2185	* because that depends on page_mapped(); but not all its usages
2192	2186	* are holding mmap_lock. Users without mmap_lock are required to
2193	2187	* take a reference count to prevent the anon_vma disappearing
@@ -2209,10 +2203,9 @@ static struct anon_vma rmap_walk_anon_lock(struct page page, · 15 unmodified lines
2209	2203	* Find all the mappings of a page using the mapping pointer and the vma chains
2210	2204	* contained in the anon_vma struct it points to.
2211	2205	*/
2212		−static void rmap_walk_anon(struct page page, struct rmap_walk_control rwc,
	2206	+static void rmap_walk_anon(struct folio folio, struct rmap_walk_control rwc,
2213	2207	bool locked)
2214	2208	{
2215		− struct folio *folio = page_folio(page);
2216	2209	struct anon_vma *anon_vma;
2217	2210	pgoff_t pgoff_start, pgoff_end;
2218	2211	struct anon_vma_chain *avc;
@@ -2222,17 +2215,17 @@ static void rmap_walk_anon(struct page page, struct rmap_walk_control rwc, · 3 unmodified lines
2222	2215	/* anon_vma disappear under us? */
2223	2216	VM_BUG_ON_FOLIO(!anon_vma, folio);
2224	2217	} else {
2225		− anon_vma = rmap_walk_anon_lock(page, rwc);
	2218	+ anon_vma = rmap_walk_anon_lock(folio, rwc);
2226	2219	}
2227	2220	if (!anon_vma)
2228	2221	return;
2229	2222
2230		− pgoff_start = page_to_pgoff(page);
2231		− pgoff_end = pgoff_start + thp_nr_pages(page) - 1;
	2223	+ pgoff_start = folio_pgoff(folio);
	2224	+ pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
2232	2225	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
2233	2226	pgoff_start, pgoff_end) {
2234	2227	struct vm_area_struct *vma = avc->vma;
2235		− unsigned long address = vma_address(page, vma);
	2228	+ unsigned long address = vma_address(&folio->page, vma);
2236	2229
2237	2230	VM_BUG_ON_VMA(address == -EFAULT, vma);
2238	2231	cond_resched();
@@ -2240,9 +2233,9 @@ static void rmap_walk_anon(struct page page, struct rmap_walk_control rwc, · 1 unmodified line
2240	2233	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2241	2234	continue;
2242	2235
2243		− if (!rwc->rmap_one(page, vma, address, rwc->arg))
	2236	+ if (!rwc->rmap_one(folio, vma, address, rwc->arg))
2244	2237	break;
2245		− if (rwc->done && rwc->done(page))
	2238	+ if (rwc->done && rwc->done(folio))
2246	2239	break;
2247	2240	}
2248	2241
@@ -2258,10 +2251,10 @@ static void rmap_walk_anon(struct page page, struct rmap_walk_control rwc, · 9 unmodified lines
2258	2251	* Find all the mappings of a page using the mapping pointer and the vma chains
2259	2252	* contained in the address_space struct it points to.
2260	2253	*/
2261		−static void rmap_walk_file(struct page page, struct rmap_walk_control rwc,
	2254	+static void rmap_walk_file(struct folio folio, struct rmap_walk_control rwc,
2262	2255	bool locked)
2263	2256	{
2264		− struct address_space *mapping = page_mapping(page);
	2257	+ struct address_space *mapping = folio_mapping(folio);
2265	2258	pgoff_t pgoff_start, pgoff_end;
2266	2259	struct vm_area_struct *vma;
2267	2260
@@ -2271,18 +2264,18 @@ static void rmap_walk_file(struct page page, struct rmap_walk_control rwc, · 3 unmodified lines
2271	2264	* structure at mapping cannot be freed and reused yet,
2272	2265	* so we can safely take mapping->i_mmap_rwsem.
2273	2266	*/
2274		− VM_BUG_ON_PAGE(!PageLocked(page), page);
	2267	+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2275	2268
2276	2269	if (!mapping)
2277	2270	return;
2278	2271
2279		− pgoff_start = page_to_pgoff(page);
2280		− pgoff_end = pgoff_start + thp_nr_pages(page) - 1;
	2272	+ pgoff_start = folio_pgoff(folio);
	2273	+ pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
2281	2274	if (!locked)
2282	2275	i_mmap_lock_read(mapping);
2283	2276	vma_interval_tree_foreach(vma, &mapping->i_mmap,
2284	2277	pgoff_start, pgoff_end) {
2285		− unsigned long address = vma_address(page, vma);
	2278	+ unsigned long address = vma_address(&folio->page, vma);
2286	2279
2287	2280	VM_BUG_ON_VMA(address == -EFAULT, vma);
2288	2281	cond_resched();
@@ -2290,9 +2283,9 @@ static void rmap_walk_file(struct page page, struct rmap_walk_control rwc, · 1 unmodified line
2290	2283	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
2291	2284	continue;
2292	2285
2293		− if (!rwc->rmap_one(page, vma, address, rwc->arg))
	2286	+ if (!rwc->rmap_one(folio, vma, address, rwc->arg))
2294	2287	goto done;
2295		− if (rwc->done && rwc->done(page))
	2288	+ if (rwc->done && rwc->done(folio))
2296	2289	goto done;
2297	2290	}
2298	2291
@@ -2301,25 +2294,25 @@ done: · 2 unmodified lines
2301	2294	i_mmap_unlock_read(mapping);
2302	2295	}
2303	2296
2304		−void rmap_walk(struct page page, struct rmap_walk_control rwc)
	2297	+void rmap_walk(struct folio folio, struct rmap_walk_control rwc)
2305	2298	{
2306		− if (unlikely(PageKsm(page)))
2307		− rmap_walk_ksm(page, rwc);
2308		− else if (PageAnon(page))
2309		− rmap_walk_anon(page, rwc, false);
	2299	+ if (unlikely(folio_test_ksm(folio)))
	2300	+ rmap_walk_ksm(folio, rwc);
	2301	+ else if (folio_test_anon(folio))
	2302	+ rmap_walk_anon(folio, rwc, false);
2310	2303	else
2311		− rmap_walk_file(page, rwc, false);
	2304	+ rmap_walk_file(folio, rwc, false);
2312	2305	}
2313	2306
2314	2307	/* Like rmap_walk, but caller holds relevant rmap lock */
2315		−void rmap_walk_locked(struct page page, struct rmap_walk_control rwc)
	2308	+void rmap_walk_locked(struct folio folio, struct rmap_walk_control rwc)
2316	2309	{
2317	2310	/* no ksm support for now */
2318		− VM_BUG_ON_PAGE(PageKsm(page), page);
2319		− if (PageAnon(page))
2320		− rmap_walk_anon(page, rwc, true);
	2311	+ VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
	2312	+ if (folio_test_anon(folio))
	2313	+ rmap_walk_anon(folio, rwc, true);
2321	2314	else
2322		− rmap_walk_file(page, rwc, true);
	2315	+ rmap_walk_file(folio, rwc, true);
2323	2316	}
2324	2317
2325	2318	#ifdef CONFIG_HUGETLB_PAGE

mm/rmap: Convert rmap_walk() to take a folio

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code