thp: keep huge zero page pinned until tlb flush

Andrea has found[1] a race condition on MMU-gather based TLB flush vs split_huge_page() or shrinker which frees huge zero under us (patch 1/2 and 2/2 respectively). With new THP refcounting, we don't need patch 1/2: mmu_gather keeps the page pinned until flush is complete and the pin prevents the page from being split under us. We still need patch 2/2. This is simplified version of Andrea's patch. We don't need fancy encoding. [1] http://lkml.kernel.org/r/1447938052-22165-1-git-send-email-aarcange@redhat.com

Kirill A. Shutemov committed 10 years ago

Commit aa88b681 parent 66ee95d3 files · +13 −3Browse files

Unified · Split

3 files

include/linux/huge_mm.h +5 −0
mm/huge_memory.c +3 −3
mm/swap.c +5 −0

include/linux/huge_mm.h+5−0

@@ -152,6 +152,7 @@ static inline bool is_huge_zero_pmd(pmd_t pmd) · 151 unmodified lines
152	152	}
153	153
154	154	struct page *get_huge_zero_page(void);
	155	+void put_huge_zero_page(void);
155	156
156	157	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
157	158	#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
@@ -208,6 +209,10 @@ static inline bool is_huge_zero_page(struct page *page) · 50 unmodified lines
208	209	return false;
209	210	}
210	211
	212	+static inline void put_huge_zero_page(void)
	213	+{
	214	+ BUILD_BUG();
	215	+}
211	216
212	217	static inline struct page follow_devmap_pmd(struct vm_area_struct vma,
213	218	unsigned long addr, pmd_t *pmd, int flags)

mm/huge_memory.c+3−3

@@ -232,7 +232,7 @@ retry: · 231 unmodified lines
232	232	return READ_ONCE(huge_zero_page);
233	233	}
234	234
235		−static void put_huge_zero_page(void)
	235	+void put_huge_zero_page(void)
236	236	{
237	237	/*
238	238	* Counter should never go to zero here. Only shrinker can put
@@ -1684,12 +1684,12 @@ int zap_huge_pmd(struct mmu_gather tlb, struct vm_area_struct vma, · 1445 unmodified lines
1684	1684	if (vma_is_dax(vma)) {
1685	1685	spin_unlock(ptl);
1686	1686	if (is_huge_zero_pmd(orig_pmd))
1687		− put_huge_zero_page();
	1687	+ tlb_remove_page(tlb, pmd_page(orig_pmd));
1688	1688	} else if (is_huge_zero_pmd(orig_pmd)) {
1689	1689	pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
1690	1690	atomic_long_dec(&tlb->mm->nr_ptes);
1691	1691	spin_unlock(ptl);
1692		− put_huge_zero_page();
	1692	+ tlb_remove_page(tlb, pmd_page(orig_pmd));
1693	1693	} else {
1694	1694	struct page *page = pmd_page(orig_pmd);
1695	1695	page_remove_rmap(page, true);

mm/swap.c+5−0

@@ -728,6 +728,11 @@ void release_pages(struct page **pages, int nr, bool cold) · 727 unmodified lines
728	728	zone = NULL;
729	729	}
730	730
	731	+ if (is_huge_zero_page(page)) {
	732	+ put_huge_zero_page();
	733	+ continue;
	734	+ }
	735	+
731	736	page = compound_head(page);
732	737	if (!put_page_testzero(page))
733	738	continue;