mm: Convert page_vma_mapped_walk to work on PFNs

page_mapped_in_vma() really just wants to walk one page, but as the code stands, if passed the head page of a compound page, it will walk every page in the compound page. Extract pfn/nr_pages/pgoff from the struct page early, so they can be overridden by page_mapped_in_vma().

Matthew Wilcox (Oracle) committed 4 years ago

Commit 2aff7a41 parent aef13de6 files · +58 −50Browse files

Unified · Split

6 files

include/linux/hugetlb.h +5 −0
include/linux/rmap.h +12 −5
mm/internal.h +9 −6
mm/migrate.c +3 −2
mm/page_vma_mapped.c +26 −32
mm/rmap.c +3 −5

include/linux/hugetlb.h+5−0

@@ -970,6 +970,11 @@ static inline struct hstate page_hstate(struct page page) · 969 unmodified lines
970	970	return NULL;
971	971	}
972	972
	973	+static inline struct hstate *size_to_hstate(unsigned long size)
	974	+{
	975	+ return NULL;
	976	+}
	977	+
973	978	static inline unsigned long huge_page_size(struct hstate *h)
974	979	{
975	980	return PAGE_SIZE;

include/linux/rmap.h+12−5

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include <linux/rwsem.h>
12	12	#include <linux/memcontrol.h>
13	13	#include <linux/highmem.h>
	14	+#include <linux/pagemap.h>
14	15
15	16	/*
16	17	* The anon_vma heads a list of private "related" vmas, to scan if
@@ -201,11 +202,13 @@ int make_device_exclusive_range(struct mm_struct *mm, unsigned long start, · 184 unmodified lines
201	202
202	203	/* Avoid racy checks */
203	204	#define PVMW_SYNC (1 << 0)
204		−/* Look for migarion entries rather than present PTEs */
	205	+/* Look for migration entries rather than present PTEs */
205	206	#define PVMW_MIGRATION (1 << 1)
206	207
207	208	struct page_vma_mapped_walk {
208		− struct page *page;
	209	+ unsigned long pfn;
	210	+ unsigned long nr_pages;
	211	+ pgoff_t pgoff;
209	212	struct vm_area_struct *vma;
210	213	unsigned long address;
211	214	pmd_t *pmd;
@@ -216,7 +219,9 @@ struct page_vma_mapped_walk { · 4 unmodified lines
216	219
217	220	#define DEFINE_PAGE_VMA_WALK(name, _page, _vma, _address, _flags) \
218	221	struct page_vma_mapped_walk name = { \
219		− .page = _page, \
	222	+ .pfn = page_to_pfn(_page), \
	223	+ .nr_pages = compound_nr(page), \
	224	+ .pgoff = page_to_pgoff(page), \
220	225	.vma = _vma, \
221	226	.address = _address, \
222	227	.flags = _flags, \
@@ -224,7 +229,9 @@ struct page_vma_mapped_walk { · 1 unmodified line
224	229
225	230	#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \
226	231	struct page_vma_mapped_walk name = { \
227		− .page = &_folio->page, \
	232	+ .pfn = folio_pfn(_folio), \
	233	+ .nr_pages = folio_nr_pages(_folio), \
	234	+ .pgoff = folio_pgoff(_folio), \
228	235	.vma = _vma, \
229	236	.address = _address, \
230	237	.flags = _flags, \
@@ -233,7 +240,7 @@ struct page_vma_mapped_walk { · 2 unmodified lines
233	240	static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
234	241	{
235	242	/* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
236		− if (pvmw->pte && !PageHuge(pvmw->page))
	243	+ if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma))
237	244	pte_unmap(pvmw->pte);
238	245	if (pvmw->ptl)
239	246	spin_unlock(pvmw->ptl);

mm/internal.h+9−6

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/fs.h>
11	11	#include <linux/mm.h>
12	12	#include <linux/pagemap.h>
	13	+#include <linux/rmap.h>
13	14	#include <linux/tracepoint-defs.h>
14	15
15	16	struct folio_batch;
@@ -475,18 +476,20 @@ vma_address(struct page page, struct vm_area_struct vma) · 459 unmodified lines
475	476	}
476	477
477	478	/*
478		− * Then at what user virtual address will none of the page be found in vma?
	479	+ * Then at what user virtual address will none of the range be found in vma?
479	480	* Assumes that vma_address() already returned a good starting address.
480		− * If page is a compound head, the entire compound page is considered.
481	481	*/
482		−static inline unsigned long
483		−vma_address_end(struct page page, struct vm_area_struct vma)
	482	+static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw)
484	483	{
	484	+ struct vm_area_struct *vma = pvmw->vma;
485	485	pgoff_t pgoff;
486	486	unsigned long address;
487	487
488		− VM_BUG_ON_PAGE(PageKsm(page), page); /* KSM page->index unusable */
489		− pgoff = page_to_pgoff(page) + compound_nr(page);
	488	+ /* Common case, plus ->pgoff is invalid for KSM */
	489	+ if (pvmw->nr_pages == 1)
	490	+ return pvmw->address + PAGE_SIZE;
	491	+
	492	+ pgoff = pvmw->pgoff + pvmw->nr_pages;
490	493	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
491	494	/* Check for address beyond vma (or wrapped through 0?) */
492	495	if (address < vma->vm_start \|\| address > vma->vm_end)

mm/migrate.c+3−2

@@ -174,7 +174,8 @@ void putback_movable_pages(struct list_head *l) · 173 unmodified lines
174	174	static bool remove_migration_pte(struct page page, struct vm_area_struct vma,
175	175	unsigned long addr, void *old)
176	176	{
177		− DEFINE_PAGE_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC \| PVMW_MIGRATION);
	177	+ DEFINE_PAGE_VMA_WALK(pvmw, (struct page *)old, vma, addr,
	178	+ PVMW_SYNC \| PVMW_MIGRATION);
178	179	struct page *new;
179	180	pte_t pte;
180	181	swp_entry_t entry;
@@ -184,7 +185,7 @@ static bool remove_migration_pte(struct page page, struct vm_area_struct vma, · 3 unmodified lines
184	185	if (PageKsm(page))
185	186	new = page;
186	187	else
187		− new = page - pvmw.page->index +
	188	+ new = page - pvmw.pgoff +
188	189	linear_page_index(vma, pvmw.address);
189	190
190	191	#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION

mm/page_vma_mapped.c+26−32

@@ -53,18 +53,6 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw) · 52 unmodified lines
53	53	return true;
54	54	}
55	55
56		−static inline bool pfn_is_match(struct page *page, unsigned long pfn)
57		−{
58		− unsigned long page_pfn = page_to_pfn(page);
59		−
60		− /* normal page and hugetlbfs page */
61		− if (!PageTransCompound(page) \|\| PageHuge(page))
62		− return page_pfn == pfn;
63		−
64		− /* THP can be referenced by any subpage */
65		− return pfn >= page_pfn && pfn - page_pfn < thp_nr_pages(page);
66		−}
67		−
68	56	/**
69	57	* check_pte - check if @pvmw->page is mapped at the @pvmw->pte
70	58	* @pvmw: page_vma_mapped_walk struct, includes a pair pte and page for checking
@@ -116,7 +104,17 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw) · 45 unmodified lines
116	104	pfn = pte_pfn(*pvmw->pte);
117	105	}
118	106
119		− return pfn_is_match(pvmw->page, pfn);
	107	+ return (pfn - pvmw->pfn) < pvmw->nr_pages;
	108	+}
	109	+
	110	+/* Returns true if the two ranges overlap. Careful to not overflow. */
	111	+static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw)
	112	+{
	113	+ if ((pfn + HPAGE_PMD_NR - 1) < pvmw->pfn)
	114	+ return false;
	115	+ if (pfn > pvmw->pfn + pvmw->nr_pages - 1)
	116	+ return false;
	117	+ return true;
120	118	}
121	119
122	120	static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size)
@@ -127,7 +125,7 @@ static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size) · 4 unmodified lines
127	125	}
128	126
129	127	/**
130		− * page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at
	128	+ * page_vma_mapped_walk - check if @pvmw->pfn is mapped in @pvmw->vma at
131	129	* @pvmw->address
132	130	* @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
133	131	* must be set. pmd, pte and ptl must be NULL.
@@ -152,8 +150,8 @@ static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size) · 18 unmodified lines
152	150	*/
153	151	bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
154	152	{
155		− struct mm_struct *mm = pvmw->vma->vm_mm;
156		− struct page *page = pvmw->page;
	153	+ struct vm_area_struct *vma = pvmw->vma;
	154	+ struct mm_struct *mm = vma->vm_mm;
157	155	unsigned long end;
158	156	pgd_t *pgd;
159	157	p4d_t *p4d;
@@ -164,32 +162,26 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) · 4 unmodified lines
164	162	if (pvmw->pmd && !pvmw->pte)
165	163	return not_found(pvmw);
166	164
167		− if (unlikely(PageHuge(page))) {
	165	+ if (unlikely(is_vm_hugetlb_page(vma))) {
	166	+ unsigned long size = pvmw->nr_pages * PAGE_SIZE;
168	167	/* The only possible mapping was handled on last iteration */
169	168	if (pvmw->pte)
170	169	return not_found(pvmw);
171	170
172	171	/* when pud is not present, pte will be NULL */
173		− pvmw->pte = huge_pte_offset(mm, pvmw->address, page_size(page));
	172	+ pvmw->pte = huge_pte_offset(mm, pvmw->address, size);
174	173	if (!pvmw->pte)
175	174	return false;
176	175
177		− pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte);
	176	+ pvmw->ptl = huge_pte_lockptr(size_to_hstate(size), mm,
	177	+ pvmw->pte);
178	178	spin_lock(pvmw->ptl);
179	179	if (!check_pte(pvmw))
180	180	return not_found(pvmw);
181	181	return true;
182	182	}
183	183
184		− /*
185		− * Seek to next pte only makes sense for THP.
186		− * But more important than that optimization, is to filter out
187		− * any PageKsm page: whose page->index misleads vma_address()
188		− * and vma_address_end() to disaster.
189		− */
190		− end = PageTransCompound(page) ?
191		− vma_address_end(page, pvmw->vma) :
192		− pvmw->address + PAGE_SIZE;
	184	+ end = vma_address_end(pvmw);
193	185	if (pvmw->pte)
194	186	goto next_pte;
195	187	restart:
@@ -224,7 +216,7 @@ restart: · 28 unmodified lines
224	216	if (likely(pmd_trans_huge(pmde))) {
225	217	if (pvmw->flags & PVMW_MIGRATION)
226	218	return not_found(pvmw);
227		− if (pmd_page(pmde) != page)
	219	+ if (!check_pmd(pmd_pfn(pmde), pvmw))
228	220	return not_found(pvmw);
229	221	return true;
230	222	}
@@ -236,7 +228,7 @@ restart: · 5 unmodified lines
236	228	return not_found(pvmw);
237	229	entry = pmd_to_swp_entry(pmde);
238	230	if (!is_migration_entry(entry) \|\|
239		− pfn_swap_entry_to_page(entry) != page)
	231	+ !check_pmd(swp_offset(entry), pvmw))
240	232	return not_found(pvmw);
241	233	return true;
242	234	}
@@ -250,7 +242,8 @@ restart: · 7 unmodified lines
250	242	* cleared *pmd but not decremented compound_mapcount().
251	243	*/
252	244	if ((pvmw->flags & PVMW_SYNC) &&
253		− PageTransCompound(page)) {
	245	+ transparent_hugepage_active(vma) &&
	246	+ (pvmw->nr_pages >= HPAGE_PMD_NR)) {
254	247	spinlock_t *ptl = pmd_lock(mm, pvmw->pmd);
255	248
256	249	spin_unlock(ptl);
@@ -307,7 +300,8 @@ next_pte: · 50 unmodified lines
307	300	int page_mapped_in_vma(struct page page, struct vm_area_struct vma)
308	301	{
309	302	struct page_vma_mapped_walk pvmw = {
310		− .page = page,
	303	+ .pfn = page_to_pfn(page),
	304	+ .nr_pages = 1,
311	305	.vma = vma,
312	306	.flags = PVMW_SYNC,
313	307	};

mm/rmap.c+3−5

@@ -940,7 +940,7 @@ static bool page_mkclean_one(struct page page, struct vm_area_struct vma, · 939 unmodified lines
940	940	*/
941	941	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
942	942	0, vma, vma->vm_mm, address,
943		− vma_address_end(page, vma));
	943	+ vma_address_end(&pvmw));
944	944	mmu_notifier_invalidate_range_start(&range);
945	945
946	946	while (page_vma_mapped_walk(&pvmw)) {
@@ -1437,8 +1437,7 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 490 unmodified lines
1437	1437	* Note that the page can not be free in this function as call of
1438	1438	* try_to_unmap() must hold a reference on the page.
1439	1439	*/
1440		− range.end = PageKsm(page) ?
1441		− address + PAGE_SIZE : vma_address_end(page, vma);
	1440	+ range.end = vma_address_end(&pvmw);
1442	1441	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
1443	1442	address, range.end);
1444	1443	if (PageHuge(page)) {
@@ -1732,8 +1731,7 @@ static bool try_to_migrate_one(struct page page, struct vm_area_struct vma, · 287 unmodified lines
1732	1731	* Note that the page can not be free in this function as call of
1733	1732	* try_to_unmap() must hold a reference on the page.
1734	1733	*/
1735		− range.end = PageKsm(page) ?
1736		− address + PAGE_SIZE : vma_address_end(page, vma);
	1734	+ range.end = vma_address_end(&pvmw);
1737	1735	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
1738	1736	address, range.end);
1739	1737	if (PageHuge(page)) {

mm: Convert page_vma_mapped_walk to work on PFNs

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code