// SPDX-License-Identifier: GPL-2.0/* * Memory Migration functionality - linux/mm/migrate.c * * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter * * Page migration was first developed in the context of the memory hotplug * project. The main authors of the migration code are: * * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> * Hirokazu Takahashi <taka@valinux.co.jp> * Dave Hansen <haveblue@us.ibm.com> * Christoph Lameter */ #include <linux/migrate.h>#include <linux/export.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/pagemap.h>#include <linux/buffer_head.h>#include <linux/mm_inline.h>#include <linux/nsproxy.h>#include <linux/pagevec.h>#include <linux/ksm.h>#include <linux/rmap.h>#include <linux/topology.h>#include <linux/cpu.h>#include <linux/cpuset.h>#include <linux/writeback.h>#include <linux/mempolicy.h>#include <linux/vmalloc.h>#include <linux/security.h>#include <linux/backing-dev.h>#include <linux/compaction.h>#include <linux/syscalls.h>#include <linux/compat.h>#include <linux/hugetlb.h>#include <linux/hugetlb_cgroup.h>#include <linux/gfp.h>#include <linux/pagewalk.h>#include <linux/pfn_t.h>#include <linux/memremap.h>#include <linux/userfaultfd_k.h>#include <linux/balloon_compaction.h>#include <linux/mmu_notifier.h>#include <linux/page_idle.h>#include <linux/page_owner.h>#include <linux/sched/mm.h>#include <linux/ptrace.h>#include <linux/oom.h> #include <asm/tlbflush.h> #define CREATE_TRACE_POINTS#include <trace/events/migrate.h> #include "internal.h" /* * migrate_prep() needs to be called before we start compiling a list of pages * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is * undesirable, use migrate_prep_local() */int migrate_prep(void){ /* * Clear the LRU lists so pages can be isolated. * Note that pages may be moved off the LRU after we have * drained them. Those pages will fail to migrate like other * pages that may be busy. */ lru_add_drain_all(); return 0;} /* Do the necessary work of migrate_prep but not if it involves other CPUs */int migrate_prep_local(void){ lru_add_drain(); return 0;} int isolate_movable_page(struct page *page, isolate_mode_t mode){ struct address_space *mapping; /* * Avoid burning cycles with pages that are yet under __free_pages(), * or just got freed under us. * * In case we 'win' a race for a movable page being freed under us and * raise its refcount preventing __free_pages() from doing its job * the put_page() at the end of this block will take care of * release this page, thus avoiding a nasty leakage. */ if (unlikely(!get_page_unless_zero(page))) goto out; /* * Check PageMovable before holding a PG_lock because page's owner * assumes anybody doesn't touch PG_lock of newly allocated page * so unconditionally grabbing the lock ruins page's owner side. */ if (unlikely(!__PageMovable(page))) goto out_putpage; /* * As movable pages are not isolated from LRU lists, concurrent * compaction threads can race against page migration functions * as well as race against the releasing a page. * * In order to avoid having an already isolated movable page * being (wrongly) re-isolated while it is under migration, * or to avoid attempting to isolate pages being released, * lets be sure we have the page lock * before proceeding with the movable page isolation steps. */ if (unlikely(!trylock_page(page))) goto out_putpage; if (!PageMovable(page) || PageIsolated(page)) goto out_no_isolated; mapping = page_mapping(page); VM_BUG_ON_PAGE(!mapping, page); if (!mapping->a_ops->isolate_page(page, mode)) goto out_no_isolated; /* Driver shouldn't use PG_isolated bit of page->flags */ WARN_ON_ONCE(PageIsolated(page)); __SetPageIsolated(page); unlock_page(page); return 0; out_no_isolated: unlock_page(page);out_putpage: put_page(page);out: return -EBUSY;} /* It should be called on page which is PG_movable */void putback_movable_page(struct page *page){ struct address_space *mapping; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageMovable(page), page); VM_BUG_ON_PAGE(!PageIsolated(page), page); mapping = page_mapping(page); mapping->a_ops->putback_page(page); __ClearPageIsolated(page);} /* * Put previously isolated pages back onto the appropriate lists * from where they were once taken off for compaction/migration. * * This function shall be used whenever the isolated pageset has been * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() * and isolate_huge_page(). */void putback_movable_pages(struct list_head *l){ struct page *page; struct page *page2; list_for_each_entry_safe(page, page2, l, lru) { if (unlikely(PageHuge(page))) { putback_active_hugepage(page); continue; } list_del(&page->lru); /* * We isolated non-lru movable page so here we can use * __PageMovable because LRU page's mapping cannot have * PAGE_MAPPING_MOVABLE. */ if (unlikely(__PageMovable(page))) { VM_BUG_ON_PAGE(!PageIsolated(page), page); lock_page(page); if (PageMovable(page)) putback_movable_page(page); else __ClearPageIsolated(page); unlock_page(page); put_page(page); } else { mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_is_file_lru(page), -thp_nr_pages(page)); putback_lru_page(page); } }} /* * Restore a potential migration pte to a working pte entry */static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, unsigned long addr, void *old){ struct page_vma_mapped_walk pvmw = { .page = old, .vma = vma, .address = addr, .flags = PVMW_SYNC | PVMW_MIGRATION, }; struct page *new; pte_t pte; swp_entry_t entry; VM_BUG_ON_PAGE(PageTail(page), page); while (page_vma_mapped_walk(&pvmw)) { if (PageKsm(page)) new = page; else new = page - pvmw.page->index + linear_page_index(vma, pvmw.address); #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); remove_migration_pmd(&pvmw, new); continue; }#endif get_page(new); pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot))); if (pte_swp_soft_dirty(*pvmw.pte)) pte = pte_mksoft_dirty(pte); /* * Recheck VMA as permissions can change since migration started */ entry = pte_to_swp_entry(*pvmw.pte); if (is_write_migration_entry(entry)) pte = maybe_mkwrite(pte, vma); else if (pte_swp_uffd_wp(*pvmw.pte)) pte = pte_mkuffd_wp(pte); if (unlikely(is_zone_device_page(new))) { if (is_device_private_page(new)) { entry = make_device_private_entry(new, pte_write(pte)); pte = swp_entry_to_pte(entry); if (pte_swp_uffd_wp(*pvmw.pte)) pte = pte_mkuffd_wp(pte); } } #ifdef CONFIG_HUGETLB_PAGE if (PageHuge(new)) { pte = pte_mkhuge(pte); pte = arch_make_huge_pte(pte, vma, new, 0); set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); if (PageAnon(new)) hugepage_add_anon_rmap(new, vma, pvmw.address); else page_dup_rmap(new, true); } else#endif { set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); if (PageAnon(new)) page_add_anon_rmap(new, vma, pvmw.address, false); else page_add_file_rmap(new, false); } if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new)) mlock_vma_page(new); if (PageTransHuge(page) && PageMlocked(page)) clear_page_mlock(page); /* No need to invalidate - it was non-present before */ update_mmu_cache(vma, pvmw.address, pvmw.pte); } return true;} /* * Get rid of all migration entries and replace them by * references to the indicated page. */void remove_migration_ptes(struct page *old, struct page *new, bool locked){ struct rmap_walk_control rwc = { .rmap_one = remove_migration_pte, .arg = old, }; if (locked) rmap_walk_locked(new, &rwc); else rmap_walk(new, &rwc);} /* * Something used the pte of a page under migration. We need to * get to the page and wait until migration is finished. * When we return from this function the fault will be retried. */void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, spinlock_t *ptl){ pte_t pte; swp_entry_t entry; struct page *page; spin_lock(ptl); pte = *ptep; if (!is_swap_pte(pte)) goto out; entry = pte_to_swp_entry(pte); if (!is_migration_entry(entry)) goto out; page = migration_entry_to_page(entry); /* * Once page cache replacement of page migration started, page_count * is zero; but we must not call put_and_wait_on_page_locked() without * a ref. Use get_page_unless_zero(), and just fault again if it fails. */ if (!get_page_unless_zero(page)) goto out; pte_unmap_unlock(ptep, ptl); put_and_wait_on_page_locked(page); return;out: pte_unmap_unlock(ptep, ptl);} void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, unsigned long address){ spinlock_t *ptl = pte_lockptr(mm, pmd); pte_t *ptep = pte_offset_map(pmd, address); __migration_entry_wait(mm, ptep, ptl);} void migration_entry_wait_huge(struct vm_area_struct *vma, struct mm_struct *mm, pte_t *pte){ spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte); __migration_entry_wait(mm, pte, ptl);} #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATIONvoid pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd){ spinlock_t *ptl; struct page *page; ptl = pmd_lock(mm, pmd); if (!is_pmd_migration_entry(*pmd)) goto unlock; page = migration_entry_to_page(pmd_to_swp_entry(*pmd)); if (!get_page_unless_zero(page)) goto unlock; spin_unlock(ptl); put_and_wait_on_page_locked(page); return;unlock: spin_unlock(ptl);}#endif static int expected_page_refs(struct address_space *mapping, struct page *page){ int expected_count = 1; /* * Device public or private pages have an extra refcount as they are * ZONE_DEVICE pages. */ expected_count += is_device_private_page(page); if (mapping) expected_count += thp_nr_pages(page) + page_has_private(page); return expected_count;} /* * Replace the page in the mapping. * * The number of remaining references must be: * 1 for anonymous pages without a mapping * 2 for pages with a mapping * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. */int migrate_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page, int extra_count){ XA_STATE(xas, &mapping->i_pages, page_index(page)); struct zone *oldzone, *newzone; int dirty; int expected_count = expected_page_refs(mapping, page) + extra_count; if (!mapping) { /* Anonymous page without mapping */ if (page_count(page) != expected_count) return -EAGAIN; /* No turning back from here */ newpage->index = page->index; newpage->mapping = page->mapping; if (PageSwapBacked(page)) __SetPageSwapBacked(newpage); return MIGRATEPAGE_SUCCESS; } oldzone = page_zone(page); newzone = page_zone(newpage); xas_lock_irq(&xas); if (page_count(page) != expected_count || xas_load(&xas) != page) { xas_unlock_irq(&xas); return -EAGAIN; } if (!page_ref_freeze(page, expected_count)) { xas_unlock_irq(&xas); return -EAGAIN; } /* * Now we know that no one else is looking at the page: * no turning back from here. */ newpage->index = page->index; newpage->mapping = page->mapping; page_ref_add(newpage, thp_nr_pages(page)); /* add cache reference */ if (PageSwapBacked(page)) { __SetPageSwapBacked(newpage); if (PageSwapCache(page)) { SetPageSwapCache(newpage); set_page_private(newpage, page_private(page)); } } else { VM_BUG_ON_PAGE(PageSwapCache(page), page); } /* Move dirty while page refs frozen and newpage not yet exposed */ dirty = PageDirty(page); if (dirty) { ClearPageDirty(page); SetPageDirty(newpage); } xas_store(&xas, newpage); if (PageTransHuge(page)) { int i; for (i = 1; i < HPAGE_PMD_NR; i++) { xas_next(&xas); xas_store(&xas, newpage); } } /* * Drop cache reference from old page by unfreezing * to one less reference. * We know this isn't the last reference. */ page_ref_unfreeze(page, expected_count - thp_nr_pages(page)); xas_unlock(&xas); /* Leave irq disabled to prevent preemption while updating stats */ /* * If moved to a different zone then also account * the page for that zone. Other VM counters will be * taken care of when we establish references to the * new page and drop references to the old page. * * Note that anonymous pages are accounted for * via NR_FILE_PAGES and NR_ANON_MAPPED if they * are mapped to swap space. */ if (newzone != oldzone) { struct lruvec *old_lruvec, *new_lruvec; struct mem_cgroup *memcg; memcg = page_memcg(page); old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat); new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat); __dec_lruvec_state(old_lruvec, NR_FILE_PAGES);Lines 1–500 of 3090. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Memory Migration functionality - linux/mm/migrate.c * * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter * * Page migration was first developed in the context of the memory hotplug * project. The main authors of the migration code are: * * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> * Hirokazu Takahashi <taka@valinux.co.jp> * Dave Hansen <haveblue@us.ibm.com> * Christoph Lameter */ #include <linux/migrate.h> #include <linux/export.h> #include <linux/swap.h> #include <linux/swapops.h> #include <linux/pagemap.h> #include <linux/buffer_head.h> #include <linux/mm_inline.h> #include <linux/nsproxy.h> #include <linux/pagevec.h> #include <linux/ksm.h> #include <linux/rmap.h> #include <linux/topology.h> #include <linux/cpu.h> #include <linux/cpuset.h> #include <linux/writeback.h> #include <linux/mempolicy.h> #include <linux/vmalloc.h> #include <linux/security.h> #include <linux/backing-dev.h> #include <linux/compaction.h> #include <linux/syscalls.h> #include <linux/compat.h> #include <linux/hugetlb.h> #include <linux/hugetlb_cgroup.h> #include <linux/gfp.h> #include <linux/pagewalk.h> #include <linux/pfn_t.h> #include <linux/memremap.h> #include <linux/userfaultfd_k.h> #include <linux/balloon_compaction.h> #include <linux/mmu_notifier.h> #include <linux/page_idle.h> #include <linux/page_owner.h> #include <linux/sched/mm.h> #include <linux/ptrace.h> #include <linux/oom.h> #include <asm/tlbflush.h> #define CREATE_TRACE_POINTS #include <trace/events/migrate.h> #include "internal.h" /* * migrate_prep() needs to be called before we start compiling a list of pages * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is * undesirable, use migrate_prep_local() */ int migrate_prep(void) { /* * Clear the LRU lists so pages can be isolated. * Note that pages may be moved off the LRU after we have * drained them. Those pages will fail to migrate like other * pages that may be busy. */ lru_add_drain_all(); return 0; } /* Do the necessary work of migrate_prep but not if it involves other CPUs */ int migrate_prep_local(void) { lru_add_drain(); return 0; } int isolate_movable_page(struct page *page, isolate_mode_t mode) { struct address_space *mapping; /* * Avoid burning cycles with pages that are yet under __free_pages(), * or just got freed under us. * * In case we 'win' a race for a movable page being freed under us and * raise its refcount preventing __free_pages() from doing its job * the put_page() at the end of this block will take care of * release this page, thus avoiding a nasty leakage. */ if (unlikely(!get_page_unless_zero(page))) goto out; /* * Check PageMovable before holding a PG_lock because page's owner * assumes anybody doesn't touch PG_lock of newly allocated page * so unconditionally grabbing the lock ruins page's owner side. */ if (unlikely(!__PageMovable(page))) goto out_putpage; /* * As movable pages are not isolated from LRU lists, concurrent * compaction threads can race against page migration functions * as well as race against the releasing a page. * * In order to avoid having an already isolated movable page * being (wrongly) re-isolated while it is under migration, * or to avoid attempting to isolate pages being released, * lets be sure we have the page lock * before proceeding with the movable page isolation steps. */ if (unlikely(!trylock_page(page))) goto out_putpage; if (!PageMovable(page) || PageIsolated(page)) goto out_no_isolated; mapping = page_mapping(page); VM_BUG_ON_PAGE(!mapping, page); if (!mapping->a_ops->isolate_page(page, mode)) goto out_no_isolated; /* Driver shouldn't use PG_isolated bit of page->flags */ WARN_ON_ONCE(PageIsolated(page)); __SetPageIsolated(page); unlock_page(page); return 0; out_no_isolated: unlock_page(page); out_putpage: put_page(page); out: return -EBUSY; } /* It should be called on page which is PG_movable */ void putback_movable_page(struct page *page) { struct address_space *mapping; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageMovable(page), page); VM_BUG_ON_PAGE(!PageIsolated(page), page); mapping = page_mapping(page); mapping->a_ops->putback_page(page); __ClearPageIsolated(page); } /* * Put previously isolated pages back onto the appropriate lists * from where they were once taken off for compaction/migration. * * This function shall be used whenever the isolated pageset has been * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() * and isolate_huge_page(). */ void putback_movable_pages(struct list_head *l) { struct page *page; struct page *page2; list_for_each_entry_safe(page, page2, l, lru) { if (unlikely(PageHuge(page))) { putback_active_hugepage(page); continue; } list_del(&page->lru); /* * We isolated non-lru movable page so here we can use * __PageMovable because LRU page's mapping cannot have * PAGE_MAPPING_MOVABLE. */ if (unlikely(__PageMovable(page))) { VM_BUG_ON_PAGE(!PageIsolated(page), page); lock_page(page); if (PageMovable(page)) putback_movable_page(page); else __ClearPageIsolated(page); unlock_page(page); put_page(page); } else { mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_is_file_lru(page), -thp_nr_pages(page)); putback_lru_page(page); } } } /* * Restore a potential migration pte to a working pte entry */ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, unsigned long addr, void *old) { struct page_vma_mapped_walk pvmw = { .page = old, .vma = vma, .address = addr, .flags = PVMW_SYNC | PVMW_MIGRATION, }; struct page *new; pte_t pte; swp_entry_t entry; VM_BUG_ON_PAGE(PageTail(page), page); while (page_vma_mapped_walk(&pvmw)) { if (PageKsm(page)) new = page; else new = page - pvmw.page->index + linear_page_index(vma, pvmw.address); #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); remove_migration_pmd(&pvmw, new); continue; } #endif get_page(new); pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot))); if (pte_swp_soft_dirty(*pvmw.pte)) pte = pte_mksoft_dirty(pte); /* * Recheck VMA as permissions can change since migration started */ entry = pte_to_swp_entry(*pvmw.pte); if (is_write_migration_entry(entry)) pte = maybe_mkwrite(pte, vma); else if (pte_swp_uffd_wp(*pvmw.pte)) pte = pte_mkuffd_wp(pte); if (unlikely(is_zone_device_page(new))) { if (is_device_private_page(new)) { entry = make_device_private_entry(new, pte_write(pte)); pte = swp_entry_to_pte(entry); if (pte_swp_uffd_wp(*pvmw.pte)) pte = pte_mkuffd_wp(pte); } } #ifdef CONFIG_HUGETLB_PAGE if (PageHuge(new)) { pte = pte_mkhuge(pte); pte = arch_make_huge_pte(pte, vma, new, 0); set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); if (PageAnon(new)) hugepage_add_anon_rmap(new, vma, pvmw.address); else page_dup_rmap(new, true); } else #endif { set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); if (PageAnon(new)) page_add_anon_rmap(new, vma, pvmw.address, false); else page_add_file_rmap(new, false); } if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new)) mlock_vma_page(new); if (PageTransHuge(page) && PageMlocked(page)) clear_page_mlock(page); /* No need to invalidate - it was non-present before */ update_mmu_cache(vma, pvmw.address, pvmw.pte); } return true; } /* * Get rid of all migration entries and replace them by * references to the indicated page. */ void remove_migration_ptes(struct page *old, struct page *new, bool locked) { struct rmap_walk_control rwc = { .rmap_one = remove_migration_pte, .arg = old, }; if (locked) rmap_walk_locked(new, &rwc); else rmap_walk(new, &rwc); } /* * Something used the pte of a page under migration. We need to * get to the page and wait until migration is finished. * When we return from this function the fault will be retried. */ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, spinlock_t *ptl) { pte_t pte; swp_entry_t entry; struct page *page; spin_lock(ptl); pte = *ptep; if (!is_swap_pte(pte)) goto out; entry = pte_to_swp_entry(pte); if (!is_migration_entry(entry)) goto out; page = migration_entry_to_page(entry); /* * Once page cache replacement of page migration started, page_count * is zero; but we must not call put_and_wait_on_page_locked() without * a ref. Use get_page_unless_zero(), and just fault again if it fails. */ if (!get_page_unless_zero(page)) goto out; pte_unmap_unlock(ptep, ptl); put_and_wait_on_page_locked(page); return; out: pte_unmap_unlock(ptep, ptl); } void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, unsigned long address) { spinlock_t *ptl = pte_lockptr(mm, pmd); pte_t *ptep = pte_offset_map(pmd, address); __migration_entry_wait(mm, ptep, ptl); } void migration_entry_wait_huge(struct vm_area_struct *vma, struct mm_struct *mm, pte_t *pte) { spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte); __migration_entry_wait(mm, pte, ptl); } #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; struct page *page; ptl = pmd_lock(mm, pmd); if (!is_pmd_migration_entry(*pmd)) goto unlock; page = migration_entry_to_page(pmd_to_swp_entry(*pmd)); if (!get_page_unless_zero(page)) goto unlock; spin_unlock(ptl); put_and_wait_on_page_locked(page); return; unlock: spin_unlock(ptl); } #endif static int expected_page_refs(struct address_space *mapping, struct page *page) { int expected_count = 1; /* * Device public or private pages have an extra refcount as they are * ZONE_DEVICE pages. */ expected_count += is_device_private_page(page); if (mapping) expected_count += thp_nr_pages(page) + page_has_private(page); return expected_count; } /* * Replace the page in the mapping. * * The number of remaining references must be: * 1 for anonymous pages without a mapping * 2 for pages with a mapping * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. */ int migrate_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page, int extra_count) { XA_STATE(xas, &mapping->i_pages, page_index(page)); struct zone *oldzone, *newzone; int dirty; int expected_count = expected_page_refs(mapping, page) + extra_count; if (!mapping) { /* Anonymous page without mapping */ if (page_count(page) != expected_count) return -EAGAIN; /* No turning back from here */ newpage->index = page->index; newpage->mapping = page->mapping; if (PageSwapBacked(page)) __SetPageSwapBacked(newpage); return MIGRATEPAGE_SUCCESS; } oldzone = page_zone(page); newzone = page_zone(newpage); xas_lock_irq(&xas); if (page_count(page) != expected_count || xas_load(&xas) != page) { xas_unlock_irq(&xas); return -EAGAIN; } if (!page_ref_freeze(page, expected_count)) { xas_unlock_irq(&xas); return -EAGAIN; } /* * Now we know that no one else is looking at the page: * no turning back from here. */ newpage->index = page->index; newpage->mapping = page->mapping; page_ref_add(newpage, thp_nr_pages(page)); /* add cache reference */ if (PageSwapBacked(page)) { __SetPageSwapBacked(newpage); if (PageSwapCache(page)) { SetPageSwapCache(newpage); set_page_private(newpage, page_private(page)); } } else { VM_BUG_ON_PAGE(PageSwapCache(page), page); } /* Move dirty while page refs frozen and newpage not yet exposed */ dirty = PageDirty(page); if (dirty) { ClearPageDirty(page); SetPageDirty(newpage); } xas_store(&xas, newpage); if (PageTransHuge(page)) { int i; for (i = 1; i < HPAGE_PMD_NR; i++) { xas_next(&xas); xas_store(&xas, newpage); } } /* * Drop cache reference from old page by unfreezing * to one less reference. * We know this isn't the last reference. */ page_ref_unfreeze(page, expected_count - thp_nr_pages(page)); xas_unlock(&xas); /* Leave irq disabled to prevent preemption while updating stats */ /* * If moved to a different zone then also account * the page for that zone. Other VM counters will be * taken care of when we establish references to the * new page and drop references to the old page. * * Note that anonymous pages are accounted for * via NR_FILE_PAGES and NR_ANON_MAPPED if they * are mapped to swap space. */ if (newzone != oldzone) { struct lruvec *old_lruvec, *new_lruvec; struct mem_cgroup *memcg; memcg = page_memcg(page); old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat); new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat); __dec_lruvec_state(old_lruvec, NR_FILE_PAGES);