mm: implement split page table lock for PMD level

The basic idea is the same as with PTE level: the lock is embedded into struct page of table's page. We can't use mm->pmd_huge_pte to store pgtables for THP, since we don't take mm->page_table_lock anymore. Let's reuse page->lru of table's page for that. pgtable_pmd_page_ctor() returns true, if initialization is successful and false otherwise. Current implementation never fails, but assumption that constructor can fail will help to port it to -rt where spinlock_t is rather huge and cannot be embedded into struct page -- dynamic allocation is required.

Kirill A. Shutemov committed 13 years ago

Commit e009bb31 parent c4088eb4 files · +43 −3Browse files

Unified · Split

4 files

include/linux/mm.h +32 −0
include/linux/mm_types.h +6 −1
kernel/fork.c +2 −2
mm/Kconfig +3 −0

include/linux/mm.h+32−0

@@ -1378,13 +1378,45 @@ static inline void pgtable_page_dtor(struct page *page) · 1377 unmodified lines
1378	1378	((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1379	1379	NULL: pte_offset_kernel(pmd, address))
1380	1380
	1381	+#if USE_SPLIT_PMD_PTLOCKS
	1382	+
	1383	+static inline spinlock_t pmd_lockptr(struct mm_struct mm, pmd_t *pmd)
	1384	+{
	1385	+ return &virt_to_page(pmd)->ptl;
	1386	+}
	1387	+
	1388	+static inline bool pgtable_pmd_page_ctor(struct page *page)
	1389	+{
	1390	+ spin_lock_init(&page->ptl);
	1391	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	1392	+ page->pmd_huge_pte = NULL;
	1393	+#endif
	1394	+ return true;
	1395	+}
	1396	+
	1397	+static inline void pgtable_pmd_page_dtor(struct page *page)
	1398	+{
	1399	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	1400	+ VM_BUG_ON(page->pmd_huge_pte);
	1401	+#endif
	1402	+}
	1403	+
	1404	+#define pmd_huge_pte(mm, pmd) (virt_to_page(pmd)->pmd_huge_pte)
	1405	+
	1406	+#else
	1407	+
1381	1408	static inline spinlock_t pmd_lockptr(struct mm_struct mm, pmd_t *pmd)
1382	1409	{
1383	1410	return &mm->page_table_lock;
1384	1411	}
1385	1412
	1413	+static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
	1414	+static inline void pgtable_pmd_page_dtor(struct page *page) {}
	1415	+
1386	1416	#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
1387	1417
	1418	+#endif
	1419	+
1388	1420	static inline spinlock_t pmd_lock(struct mm_struct mm, pmd_t *pmd)
1389	1421	{
1390	1422	spinlock_t *ptl = pmd_lockptr(mm, pmd);

include/linux/mm_types.h+6−1

@@ -24,6 +24,8 @@ · 23 unmodified lines
24	24	struct address_space;
25	25
26	26	#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
	27	+#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
	28	+ IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
27	29
28	30	/*
29	31	* Each physical page in the system has a struct page associated with
@@ -63,6 +65,9 @@ struct page { · 33 unmodified lines
63	65	* this page is only used to
64	66	* free other pages.
65	67	*/
	68	+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
	69	+ pgtable_t pmd_huge_pte; /* protected by page->ptl */
	70	+#endif
66	71	};
67	72
68	73	union {
@@ -406,7 +411,7 @@ struct mm_struct { · 337 unmodified lines
406	411	#ifdef CONFIG_MMU_NOTIFIER
407	412	struct mmu_notifier_mm *mmu_notifier_mm;
408	413	#endif
409		−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	414	+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
410	415	pgtable_t pmd_huge_pte; /* protected by page_table_lock */
411	416	#endif
412	417	#ifdef CONFIG_CPUMASK_OFFSTACK

kernel/fork.c+2−2

@@ -560,7 +560,7 @@ static void check_mm(struct mm_struct *mm) · 559 unmodified lines
560	560	"mm:%p idx:%d val:%ld\n", mm, i, x);
561	561	}
562	562
563		−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	563	+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
564	564	VM_BUG_ON(mm->pmd_huge_pte);
565	565	#endif
566	566	}
@@ -814,7 +814,7 @@ struct mm_struct dup_mm(struct task_struct tsk) · 247 unmodified lines
814	814	memcpy(mm, oldmm, sizeof(*mm));
815	815	mm_init_cpumask(mm);
816	816
817		−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	817	+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
818	818	mm->pmd_huge_pte = NULL;
819	819	#endif
820	820	if (!mm_init(mm, tsk))

mm/Kconfig+3−0

@@ -222,6 +222,9 @@ config SPLIT_PTLOCK_CPUS · 221 unmodified lines
222	222	default "999999" if !64BIT && GENERIC_LOCKBREAK
223	223	default "4"
224	224
	225	+config ARCH_ENABLE_SPLIT_PMD_PTLOCK
	226	+ boolean
	227	+
225	228	#
226	229	# support for memory balloon compaction
227	230	config BALLOON_COMPACTION