Pull NUMA balancing memory corruption fixes from Ingo Molnar: "So these fixes are definitely not something I'd like to sit on, but as I said to Mel at the KS the timing is quite tight, with Linus planning v3.12-final within a week. Fedora-19 is affected: comet:~> grep NUMA_BALANCING /boot/config-3.11.3-201.fc19.x86_64 CONFIG_ARCH_SUPPORTS_NUMA_BALANCING=y CONFIG_NUMA_BALANCING_DEFAULT_ENABLED=y CONFIG_NUMA_BALANCING=y AFAICS Ubuntu will be affected as well, once it updates the kernel: hubble:~> grep NUMA_BALANCING /boot/config-3.8.0-32-generic CONFIG_ARCH_SUPPORTS_NUMA_BALANCING=y CONFIG_NUMA_BALANCING_DEFAULT_ENABLED=y CONFIG_NUMA_BALANCING=y These 6 commits are a minimalized set of cherry-picks needed to fix the memory corruption bugs. All commits are fixes, except "mm: numa: Sanitize task_numa_fault() callsites" which is a cleanup that made two followup fixes simpler. I've done targeted testing with just this SHA1 to try to make sure there are no cherry-picking artifacts. The original non-cherry-picked set of fixes were exposed to linux-next for a couple of weeks" * 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: mm: Account for a THP NUMA hinting update as one PTE update mm: Close races between THP migration and PMD numa clearing mm: numa: Sanitize task_numa_fault() callsites mm: Prevent parallel splits during THP migration mm: Wait for THP migrations to complete during NUMA hinting faults mm: numa: Do not account for a hinting fault if we raced
Unified · Split
| @@ -1278,64 +1278,90 @@ out: · 1277 unmodified lines | ||
| 1278 | 1278 | int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, |
| 1279 | 1279 | unsigned long addr, pmd_t pmd, pmd_t *pmdp) |
| 1280 | 1280 | { |
| 1281 | + struct anon_vma *anon_vma = NULL; | |
| 1281 | 1282 | struct page *page; |
| 1282 | 1283 | unsigned long haddr = addr & HPAGE_PMD_MASK; |
| 1284 | + int page_nid = -1, this_nid = numa_node_id(); | |
| 1283 | 1285 | int target_nid; |
| 1284 | − int current_nid = -1; | |
| 1285 | − bool migrated; | |
| 1286 | + bool page_locked; | |
| 1287 | + bool migrated = false; | |
| 1286 | 1288 | |
| 1287 | 1289 | spin_lock(&mm->page_table_lock); |
| 1288 | 1290 | if (unlikely(!pmd_same(pmd, *pmdp))) |
| 1289 | 1291 | goto out_unlock; |
| 1290 | 1292 | |
| 1291 | 1293 | page = pmd_page(pmd); |
| 1292 | − get_page(page); | |
| 1293 | − current_nid = page_to_nid(page); | |
| 1294 | + page_nid = page_to_nid(page); | |
| 1294 | 1295 | count_vm_numa_event(NUMA_HINT_FAULTS); |
| 1295 | − if (current_nid == numa_node_id()) | |
| 1296 | + if (page_nid == this_nid) | |
| 1296 | 1297 | count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); |
| 1297 | 1298 | |
| 1299 | + /* | |
| 1300 | + * Acquire the page lock to serialise THP migrations but avoid dropping | |
| 1301 | + * page_table_lock if at all possible | |
| 1302 | + */ | |
| 1303 | + page_locked = trylock_page(page); | |
| 1298 | 1304 | target_nid = mpol_misplaced(page, vma, haddr); |
| 1299 | 1305 | if (target_nid == -1) { |
| 1300 | − put_page(page); | |
| 1301 | − goto clear_pmdnuma; | |
| 1306 | + /* If the page was locked, there are no parallel migrations */ | |
| 1307 | + if (page_locked) | |
| 1308 | + goto clear_pmdnuma; | |
| 1309 | + | |
| 1310 | + /* | |
| 1311 | + * Otherwise wait for potential migrations and retry. We do | |
| 1312 | + * relock and check_same as the page may no longer be mapped. | |
| 1313 | + * As the fault is being retried, do not account for it. | |
| 1314 | + */ | |
| 1315 | + spin_unlock(&mm->page_table_lock); | |
| 1316 | + wait_on_page_locked(page); | |
| 1317 | + page_nid = -1; | |
| 1318 | + goto out; | |
| 1302 | 1319 | } |
| 1303 | 1320 | |
| 1304 | − /* Acquire the page lock to serialise THP migrations */ | |
| 1321 | + /* Page is misplaced, serialise migrations and parallel THP splits */ | |
| 1322 | + get_page(page); | |
| 1305 | 1323 | spin_unlock(&mm->page_table_lock); |
| 1306 | − lock_page(page); | |
| 1324 | + if (!page_locked) | |
| 1325 | + lock_page(page); | |
| 1326 | + anon_vma = page_lock_anon_vma_read(page); | |
| 1307 | 1327 | |
| 1308 | 1328 | /* Confirm the PTE did not while locked */ |
| 1309 | 1329 | spin_lock(&mm->page_table_lock); |
| 1310 | 1330 | if (unlikely(!pmd_same(pmd, *pmdp))) { |
| 1311 | 1331 | unlock_page(page); |
| 1312 | 1332 | put_page(page); |
| 1333 | + page_nid = -1; | |
| 1313 | 1334 | goto out_unlock; |
| 1314 | 1335 | } |
| 1315 | − spin_unlock(&mm->page_table_lock); | |
| 1316 | 1336 | |
| 1317 | − /* Migrate the THP to the requested node */ | |
| 1337 | + /* | |
| 1338 | + * Migrate the THP to the requested node, returns with page unlocked | |
| 1339 | + * and pmd_numa cleared. | |
| 1340 | + */ | |
| 1341 | + spin_unlock(&mm->page_table_lock); | |
| 1318 | 1342 | migrated = migrate_misplaced_transhuge_page(mm, vma, |
| 1319 | 1343 | pmdp, pmd, addr, page, target_nid); |
| 1320 | − if (!migrated) | |
| 1321 | − goto check_same; | |
| 1322 | − | |
| 1323 | − task_numa_fault(target_nid, HPAGE_PMD_NR, true); | |
| 1324 | − return 0; | |
| 1344 | + if (migrated) | |
| 1345 | + page_nid = target_nid; | |
| 1325 | 1346 | |
| 1326 | −check_same: | |
| 1327 | − spin_lock(&mm->page_table_lock); | |
| 1328 | − if (unlikely(!pmd_same(pmd, *pmdp))) | |
| 1329 | − goto out_unlock; | |
| 1347 | + goto out; | |
| 1330 | 1348 | clear_pmdnuma: |
| 1349 | + BUG_ON(!PageLocked(page)); | |
| 1331 | 1350 | pmd = pmd_mknonnuma(pmd); |
| 1332 | 1351 | set_pmd_at(mm, haddr, pmdp, pmd); |
| 1333 | 1352 | VM_BUG_ON(pmd_numa(*pmdp)); |
| 1334 | 1353 | update_mmu_cache_pmd(vma, addr, pmdp); |
| 1354 | + unlock_page(page); | |
| 1335 | 1355 | out_unlock: |
| 1336 | 1356 | spin_unlock(&mm->page_table_lock); |
| 1337 | − if (current_nid != -1) | |
| 1338 | − task_numa_fault(current_nid, HPAGE_PMD_NR, false); | |
| 1357 | + | |
| 1358 | +out: | |
| 1359 | + if (anon_vma) | |
| 1360 | + page_unlock_anon_vma_read(anon_vma); | |
| 1361 | + | |
| 1362 | + if (page_nid != -1) | |
| 1363 | + task_numa_fault(page_nid, HPAGE_PMD_NR, migrated); | |
| 1364 | + | |
| 1339 | 1365 | return 0; |
| 1340 | 1366 | } |
| 1341 | 1367 | |
| @@ -3521,12 +3521,12 @@ static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, · 3520 unmodified lines | ||
| 3521 | 3521 | } |
| 3522 | 3522 | |
| 3523 | 3523 | int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, |
| 3524 | − unsigned long addr, int current_nid) | |
| 3524 | + unsigned long addr, int page_nid) | |
| 3525 | 3525 | { |
| 3526 | 3526 | get_page(page); |
| 3527 | 3527 | |
| 3528 | 3528 | count_vm_numa_event(NUMA_HINT_FAULTS); |
| 3529 | − if (current_nid == numa_node_id()) | |
| 3529 | + if (page_nid == numa_node_id()) | |
| 3530 | 3530 | count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); |
| 3531 | 3531 | |
| 3532 | 3532 | return mpol_misplaced(page, vma, addr); |
| @@ -3537,7 +3537,7 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, · 4 unmodified lines | ||
| 3537 | 3537 | { |
| 3538 | 3538 | struct page *page = NULL; |
| 3539 | 3539 | spinlock_t *ptl; |
| 3540 | − int current_nid = -1; | |
| 3540 | + int page_nid = -1; | |
| 3541 | 3541 | int target_nid; |
| 3542 | 3542 | bool migrated = false; |
| 3543 | 3543 | |
| @@ -3567,15 +3567,10 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, · 23 unmodified lines | ||
| 3567 | 3567 | return 0; |
| 3568 | 3568 | } |
| 3569 | 3569 | |
| 3570 | − current_nid = page_to_nid(page); | |
| 3571 | − target_nid = numa_migrate_prep(page, vma, addr, current_nid); | |
| 3570 | + page_nid = page_to_nid(page); | |
| 3571 | + target_nid = numa_migrate_prep(page, vma, addr, page_nid); | |
| 3572 | 3572 | pte_unmap_unlock(ptep, ptl); |
| 3573 | 3573 | if (target_nid == -1) { |
| 3574 | − /* | |
| 3575 | − * Account for the fault against the current node if it not | |
| 3576 | − * being replaced regardless of where the page is located. | |
| 3577 | − */ | |
| 3578 | − current_nid = numa_node_id(); | |
| 3579 | 3574 | put_page(page); |
| 3580 | 3575 | goto out; |
| 3581 | 3576 | } |
| @@ -3583,11 +3578,11 @@ int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, · 1 unmodified line | ||
| 3583 | 3578 | /* Migrate to the requested node */ |
| 3584 | 3579 | migrated = migrate_misplaced_page(page, target_nid); |
| 3585 | 3580 | if (migrated) |
| 3586 | − current_nid = target_nid; | |
| 3581 | + page_nid = target_nid; | |
| 3587 | 3582 | |
| 3588 | 3583 | out: |
| 3589 | − if (current_nid != -1) | |
| 3590 | − task_numa_fault(current_nid, 1, migrated); | |
| 3584 | + if (page_nid != -1) | |
| 3585 | + task_numa_fault(page_nid, 1, migrated); | |
| 3591 | 3586 | return 0; |
| 3592 | 3587 | } |
| 3593 | 3588 | |
| @@ -3602,7 +3597,6 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, · 8 unmodified lines | ||
| 3602 | 3597 | unsigned long offset; |
| 3603 | 3598 | spinlock_t *ptl; |
| 3604 | 3599 | bool numa = false; |
| 3605 | − int local_nid = numa_node_id(); | |
| 3606 | 3600 | |
| 3607 | 3601 | spin_lock(&mm->page_table_lock); |
| 3608 | 3602 | pmd = *pmdp; |
| @@ -3625,9 +3619,10 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, · 16 unmodified lines | ||
| 3625 | 3619 | for (addr = _addr + offset; addr < _addr + PMD_SIZE; pte++, addr += PAGE_SIZE) { |
| 3626 | 3620 | pte_t pteval = *pte; |
| 3627 | 3621 | struct page *page; |
| 3628 | − int curr_nid = local_nid; | |
| 3622 | + int page_nid = -1; | |
| 3629 | 3623 | int target_nid; |
| 3630 | − bool migrated; | |
| 3624 | + bool migrated = false; | |
| 3625 | + | |
| 3631 | 3626 | if (!pte_present(pteval)) |
| 3632 | 3627 | continue; |
| 3633 | 3628 | if (!pte_numa(pteval)) |
| @@ -3649,25 +3644,19 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, · 15 unmodified lines | ||
| 3649 | 3644 | if (unlikely(page_mapcount(page) != 1)) |
| 3650 | 3645 | continue; |
| 3651 | 3646 | |
| 3652 | − /* | |
| 3653 | − * Note that the NUMA fault is later accounted to either | |
| 3654 | − * the node that is currently running or where the page is | |
| 3655 | − * migrated to. | |
| 3656 | − */ | |
| 3657 | − curr_nid = local_nid; | |
| 3658 | − target_nid = numa_migrate_prep(page, vma, addr, | |
| 3659 | − page_to_nid(page)); | |
| 3660 | − if (target_nid == -1) { | |
| 3647 | + page_nid = page_to_nid(page); | |
| 3648 | + target_nid = numa_migrate_prep(page, vma, addr, page_nid); | |
| 3649 | + pte_unmap_unlock(pte, ptl); | |
| 3650 | + if (target_nid != -1) { | |
| 3651 | + migrated = migrate_misplaced_page(page, target_nid); | |
| 3652 | + if (migrated) | |
| 3653 | + page_nid = target_nid; | |
| 3654 | + } else { | |
| 3661 | 3655 | put_page(page); |
| 3662 | − continue; | |
| 3663 | 3656 | } |
| 3664 | 3657 | |
| 3665 | − /* Migrate to the requested node */ | |
| 3666 | − pte_unmap_unlock(pte, ptl); | |
| 3667 | − migrated = migrate_misplaced_page(page, target_nid); | |
| 3668 | − if (migrated) | |
| 3669 | − curr_nid = target_nid; | |
| 3670 | − task_numa_fault(curr_nid, 1, migrated); | |
| 3658 | + if (page_nid != -1) | |
| 3659 | + task_numa_fault(page_nid, 1, migrated); | |
| 3671 | 3660 | |
| 3672 | 3661 | pte = pte_offset_map_lock(mm, pmdp, addr, &ptl); |
| 3673 | 3662 | } |
| @@ -1715,12 +1715,12 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, · 1714 unmodified lines | ||
| 1715 | 1715 | unlock_page(new_page); |
| 1716 | 1716 | put_page(new_page); /* Free it */ |
| 1717 | 1717 | |
| 1718 | − unlock_page(page); | |
| 1718 | + /* Retake the callers reference and putback on LRU */ | |
| 1719 | + get_page(page); | |
| 1719 | 1720 | putback_lru_page(page); |
| 1720 | − | |
| 1721 | − count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR); | |
| 1722 | − isolated = 0; | |
| 1723 | − goto out; | |
| 1721 | + mod_zone_page_state(page_zone(page), | |
| 1722 | + NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR); | |
| 1723 | + goto out_fail; | |
| 1724 | 1724 | } |
| 1725 | 1725 | |
| 1726 | 1726 | /* |
| @@ -1737,9 +1737,9 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, · 10 unmodified lines | ||
| 1737 | 1737 | entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); |
| 1738 | 1738 | entry = pmd_mkhuge(entry); |
| 1739 | 1739 | |
| 1740 | − page_add_new_anon_rmap(new_page, vma, haddr); | |
| 1741 | − | |
| 1740 | + pmdp_clear_flush(vma, haddr, pmd); | |
| 1742 | 1741 | set_pmd_at(mm, haddr, pmd, entry); |
| 1742 | + page_add_new_anon_rmap(new_page, vma, haddr); | |
| 1743 | 1743 | update_mmu_cache_pmd(vma, address, &entry); |
| 1744 | 1744 | page_remove_rmap(page); |
| 1745 | 1745 | /* |
| @@ -1758,7 +1758,6 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, · 12 unmodified lines | ||
| 1758 | 1758 | count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR); |
| 1759 | 1759 | count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR); |
| 1760 | 1760 | |
| 1761 | −out: | |
| 1762 | 1761 | mod_zone_page_state(page_zone(page), |
| 1763 | 1762 | NR_ISOLATED_ANON + page_lru, |
| 1764 | 1763 | -HPAGE_PMD_NR); |
| @@ -1767,6 +1766,10 @@ out: · 2 unmodified lines | ||
| 1767 | 1766 | out_fail: |
| 1768 | 1767 | count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR); |
| 1769 | 1768 | out_dropref: |
| 1769 | + entry = pmd_mknonnuma(entry); | |
| 1770 | + set_pmd_at(mm, haddr, pmd, entry); | |
| 1771 | + update_mmu_cache_pmd(vma, address, &entry); | |
| 1772 | + | |
| 1770 | 1773 | unlock_page(page); |
| 1771 | 1774 | put_page(page); |
| 1772 | 1775 | return 0; |
| @@ -148,7 +148,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma, · 147 unmodified lines | ||
| 148 | 148 | split_huge_page_pmd(vma, addr, pmd); |
| 149 | 149 | else if (change_huge_pmd(vma, pmd, addr, newprot, |
| 150 | 150 | prot_numa)) { |
| 151 | − pages += HPAGE_PMD_NR; | |
| 151 | + pages++; | |
| 152 | 152 | continue; |
| 153 | 153 | } |
| 154 | 154 | /* fall through */ |