The basic idea is the same as with PTE level: the lock is embedded into struct page of table's page. We can't use mm->pmd_huge_pte to store pgtables for THP, since we don't take mm->page_table_lock anymore. Let's reuse page->lru of table's page for that. pgtable_pmd_page_ctor() returns true, if initialization is successful and false otherwise. Current implementation never fails, but assumption that constructor can fail will help to port it to -rt where spinlock_t is rather huge and cannot be embedded into struct page -- dynamic allocation is required.
Unified · Split
| @@ -1378,13 +1378,45 @@ static inline void pgtable_page_dtor(struct page *page) · 1377 unmodified lines | |||
| 1378 | ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \ | 1378 | ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \ |
| 1379 | NULL: pte_offset_kernel(pmd, address)) | 1379 | NULL: pte_offset_kernel(pmd, address)) |
| 1380 | 1380 | ||
| 1381 | #if USE_SPLIT_PMD_PTLOCKS | ||
| 1382 | |||
| 1383 | static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) | ||
| 1384 | { | ||
| 1385 | return &virt_to_page(pmd)->ptl; | ||
| 1386 | } | ||
| 1387 | |||
| 1388 | static inline bool pgtable_pmd_page_ctor(struct page *page) | ||
| 1389 | { | ||
| 1390 | spin_lock_init(&page->ptl); | ||
| 1391 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | ||
| 1392 | page->pmd_huge_pte = NULL; | ||
| 1393 | #endif | ||
| 1394 | return true; | ||
| 1395 | } | ||
| 1396 | |||
| 1397 | static inline void pgtable_pmd_page_dtor(struct page *page) | ||
| 1398 | { | ||
| 1399 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | ||
| 1400 | VM_BUG_ON(page->pmd_huge_pte); | ||
| 1401 | #endif | ||
| 1402 | } | ||
| 1403 | |||
| 1404 | #define pmd_huge_pte(mm, pmd) (virt_to_page(pmd)->pmd_huge_pte) | ||
| 1405 | |||
| 1406 | #else | ||
| 1407 | |||
| 1381 | static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) | 1408 | static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) |
| 1382 | { | 1409 | { |
| 1383 | return &mm->page_table_lock; | 1410 | return &mm->page_table_lock; |
| 1384 | } | 1411 | } |
| 1385 | 1412 | ||
| 1413 | static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; } | ||
| 1414 | static inline void pgtable_pmd_page_dtor(struct page *page) {} | ||
| 1415 | |||
| 1386 | #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) | 1416 | #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) |
| 1387 | 1417 | ||
| 1418 | #endif | ||
| 1419 | |||
| 1388 | static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) | 1420 | static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) |
| 1389 | { | 1421 | { |
| 1390 | spinlock_t *ptl = pmd_lockptr(mm, pmd); | 1422 | spinlock_t *ptl = pmd_lockptr(mm, pmd); |
| @@ -24,6 +24,8 @@ · 23 unmodified lines | |||
| 24 | struct address_space; | 24 | struct address_space; |
| 25 | 25 | ||
| 26 | #define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) | 26 | #define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) |
| 27 | #define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \ | ||
| 28 | IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) | ||
| 27 | 29 | ||
| 28 | /* | 30 | /* |
| 29 | * Each physical page in the system has a struct page associated with | 31 | * Each physical page in the system has a struct page associated with |
| @@ -63,6 +65,9 @@ struct page { · 33 unmodified lines | |||
| 63 | * this page is only used to | 65 | * this page is only used to |
| 64 | * free other pages. | 66 | * free other pages. |
| 65 | */ | 67 | */ |
| 68 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS | ||
| 69 | pgtable_t pmd_huge_pte; /* protected by page->ptl */ | ||
| 70 | #endif | ||
| 66 | }; | 71 | }; |
| 67 | 72 | ||
| 68 | union { | 73 | union { |
| @@ -406,7 +411,7 @@ struct mm_struct { · 337 unmodified lines | |||
| 406 | #ifdef CONFIG_MMU_NOTIFIER | 411 | #ifdef CONFIG_MMU_NOTIFIER |
| 407 | struct mmu_notifier_mm *mmu_notifier_mm; | 412 | struct mmu_notifier_mm *mmu_notifier_mm; |
| 408 | #endif | 413 | #endif |
| 409 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 414 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS |
| 410 | pgtable_t pmd_huge_pte; /* protected by page_table_lock */ | 415 | pgtable_t pmd_huge_pte; /* protected by page_table_lock */ |
| 411 | #endif | 416 | #endif |
| 412 | #ifdef CONFIG_CPUMASK_OFFSTACK | 417 | #ifdef CONFIG_CPUMASK_OFFSTACK |
| @@ -560,7 +560,7 @@ static void check_mm(struct mm_struct *mm) · 559 unmodified lines | |||
| 560 | "mm:%p idx:%d val:%ld\n", mm, i, x); | 560 | "mm:%p idx:%d val:%ld\n", mm, i, x); |
| 561 | } | 561 | } |
| 562 | 562 | ||
| 563 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 563 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS |
| 564 | VM_BUG_ON(mm->pmd_huge_pte); | 564 | VM_BUG_ON(mm->pmd_huge_pte); |
| 565 | #endif | 565 | #endif |
| 566 | } | 566 | } |
| @@ -814,7 +814,7 @@ struct mm_struct *dup_mm(struct task_struct *tsk) · 247 unmodified lines | |||
| 814 | memcpy(mm, oldmm, sizeof(*mm)); | 814 | memcpy(mm, oldmm, sizeof(*mm)); |
| 815 | mm_init_cpumask(mm); | 815 | mm_init_cpumask(mm); |
| 816 | 816 | ||
| 817 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 817 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS |
| 818 | mm->pmd_huge_pte = NULL; | 818 | mm->pmd_huge_pte = NULL; |
| 819 | #endif | 819 | #endif |
| 820 | if (!mm_init(mm, tsk)) | 820 | if (!mm_init(mm, tsk)) |
| @@ -222,6 +222,9 @@ config SPLIT_PTLOCK_CPUS · 221 unmodified lines | |||
| 222 | default "999999" if !64BIT && GENERIC_LOCKBREAK | 222 | default "999999" if !64BIT && GENERIC_LOCKBREAK |
| 223 | default "4" | 223 | default "4" |
| 224 | 224 | ||
| 225 | config ARCH_ENABLE_SPLIT_PMD_PTLOCK | ||
| 226 | boolean | ||
| 227 | |||
| 225 | # | 228 | # |
| 226 | # support for memory balloon compaction | 229 | # support for memory balloon compaction |
| 227 | config BALLOON_COMPACTION | 230 | config BALLOON_COMPACTION |