/* * Memory Migration functionality - linux/mm/migrate.c * * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter * * Page migration was first developed in the context of the memory hotplug * project. The main authors of the migration code are: * * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> * Hirokazu Takahashi <taka@valinux.co.jp> * Dave Hansen <haveblue@us.ibm.com> * Christoph Lameter */ #include <linux/migrate.h>#include <linux/export.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/pagemap.h>#include <linux/buffer_head.h>#include <linux/mm_inline.h>#include <linux/nsproxy.h>#include <linux/pagevec.h>#include <linux/ksm.h>#include <linux/rmap.h>#include <linux/topology.h>#include <linux/cpu.h>#include <linux/cpuset.h>#include <linux/writeback.h>#include <linux/mempolicy.h>#include <linux/vmalloc.h>#include <linux/security.h>#include <linux/backing-dev.h>#include <linux/compaction.h>#include <linux/syscalls.h>#include <linux/hugetlb.h>#include <linux/hugetlb_cgroup.h>#include <linux/gfp.h>#include <linux/balloon_compaction.h>#include <linux/mmu_notifier.h>#include <linux/page_idle.h>#include <linux/page_owner.h>#include <linux/sched/mm.h>#include <linux/ptrace.h> #include <asm/tlbflush.h> #define CREATE_TRACE_POINTS#include <trace/events/migrate.h> #include "internal.h" /* * migrate_prep() needs to be called before we start compiling a list of pages * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is * undesirable, use migrate_prep_local() */int migrate_prep(void){ /* * Clear the LRU lists so pages can be isolated. * Note that pages may be moved off the LRU after we have * drained them. Those pages will fail to migrate like other * pages that may be busy. */ lru_add_drain_all(); return 0;} /* Do the necessary work of migrate_prep but not if it involves other CPUs */int migrate_prep_local(void){ lru_add_drain(); return 0;} int isolate_movable_page(struct page *page, isolate_mode_t mode){ struct address_space *mapping; /* * Avoid burning cycles with pages that are yet under __free_pages(), * or just got freed under us. * * In case we 'win' a race for a movable page being freed under us and * raise its refcount preventing __free_pages() from doing its job * the put_page() at the end of this block will take care of * release this page, thus avoiding a nasty leakage. */ if (unlikely(!get_page_unless_zero(page))) goto out; /* * Check PageMovable before holding a PG_lock because page's owner * assumes anybody doesn't touch PG_lock of newly allocated page * so unconditionally grapping the lock ruins page's owner side. */ if (unlikely(!__PageMovable(page))) goto out_putpage; /* * As movable pages are not isolated from LRU lists, concurrent * compaction threads can race against page migration functions * as well as race against the releasing a page. * * In order to avoid having an already isolated movable page * being (wrongly) re-isolated while it is under migration, * or to avoid attempting to isolate pages being released, * lets be sure we have the page lock * before proceeding with the movable page isolation steps. */ if (unlikely(!trylock_page(page))) goto out_putpage; if (!PageMovable(page) || PageIsolated(page)) goto out_no_isolated; mapping = page_mapping(page); VM_BUG_ON_PAGE(!mapping, page); if (!mapping->a_ops->isolate_page(page, mode)) goto out_no_isolated; /* Driver shouldn't use PG_isolated bit of page->flags */ WARN_ON_ONCE(PageIsolated(page)); __SetPageIsolated(page); unlock_page(page); return 0; out_no_isolated: unlock_page(page);out_putpage: put_page(page);out: return -EBUSY;} /* It should be called on page which is PG_movable */void putback_movable_page(struct page *page){ struct address_space *mapping; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageMovable(page), page); VM_BUG_ON_PAGE(!PageIsolated(page), page); mapping = page_mapping(page); mapping->a_ops->putback_page(page); __ClearPageIsolated(page);} /* * Put previously isolated pages back onto the appropriate lists * from where they were once taken off for compaction/migration. * * This function shall be used whenever the isolated pageset has been * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() * and isolate_huge_page(). */void putback_movable_pages(struct list_head *l){ struct page *page; struct page *page2; list_for_each_entry_safe(page, page2, l, lru) { if (unlikely(PageHuge(page))) { putback_active_hugepage(page); continue; } list_del(&page->lru); /* * We isolated non-lru movable page so here we can use * __PageMovable because LRU page's mapping cannot have * PAGE_MAPPING_MOVABLE. */ if (unlikely(__PageMovable(page))) { VM_BUG_ON_PAGE(!PageIsolated(page), page); lock_page(page); if (PageMovable(page)) putback_movable_page(page); else __ClearPageIsolated(page); unlock_page(page); put_page(page); } else { dec_node_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page)); putback_lru_page(page); } }} /* * Restore a potential migration pte to a working pte entry */static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, unsigned long addr, void *old){ struct page_vma_mapped_walk pvmw = { .page = old, .vma = vma, .address = addr, .flags = PVMW_SYNC | PVMW_MIGRATION, }; struct page *new; pte_t pte; swp_entry_t entry; VM_BUG_ON_PAGE(PageTail(page), page); while (page_vma_mapped_walk(&pvmw)) { if (PageKsm(page)) new = page; else new = page - pvmw.page->index + linear_page_index(vma, pvmw.address); get_page(new); pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot))); if (pte_swp_soft_dirty(*pvmw.pte)) pte = pte_mksoft_dirty(pte); /* * Recheck VMA as permissions can change since migration started */ entry = pte_to_swp_entry(*pvmw.pte); if (is_write_migration_entry(entry)) pte = maybe_mkwrite(pte, vma); flush_dcache_page(new);#ifdef CONFIG_HUGETLB_PAGE if (PageHuge(new)) { pte = pte_mkhuge(pte); pte = arch_make_huge_pte(pte, vma, new, 0); set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); if (PageAnon(new)) hugepage_add_anon_rmap(new, vma, pvmw.address); else page_dup_rmap(new, true); } else#endif { set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); if (PageAnon(new)) page_add_anon_rmap(new, vma, pvmw.address, false); else page_add_file_rmap(new, false); } if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new)) mlock_vma_page(new); /* No need to invalidate - it was non-present before */ update_mmu_cache(vma, pvmw.address, pvmw.pte); } return true;} /* * Get rid of all migration entries and replace them by * references to the indicated page. */void remove_migration_ptes(struct page *old, struct page *new, bool locked){ struct rmap_walk_control rwc = { .rmap_one = remove_migration_pte, .arg = old, }; if (locked) rmap_walk_locked(new, &rwc); else rmap_walk(new, &rwc);} /* * Something used the pte of a page under migration. We need to * get to the page and wait until migration is finished. * When we return from this function the fault will be retried. */void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, spinlock_t *ptl){ pte_t pte; swp_entry_t entry; struct page *page; spin_lock(ptl); pte = *ptep; if (!is_swap_pte(pte)) goto out; entry = pte_to_swp_entry(pte); if (!is_migration_entry(entry)) goto out; page = migration_entry_to_page(entry); /* * Once radix-tree replacement of page migration started, page_count * *must* be zero. And, we don't want to call wait_on_page_locked() * against a page without get_page(). * So, we use get_page_unless_zero(), here. Even failed, page fault * will occur again. */ if (!get_page_unless_zero(page)) goto out; pte_unmap_unlock(ptep, ptl); wait_on_page_locked(page); put_page(page); return;out: pte_unmap_unlock(ptep, ptl);} void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, unsigned long address){ spinlock_t *ptl = pte_lockptr(mm, pmd); pte_t *ptep = pte_offset_map(pmd, address); __migration_entry_wait(mm, ptep, ptl);} void migration_entry_wait_huge(struct vm_area_struct *vma, struct mm_struct *mm, pte_t *pte){ spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte); __migration_entry_wait(mm, pte, ptl);} #ifdef CONFIG_BLOCK/* Returns true if all buffers are successfully locked */static bool buffer_migrate_lock_buffers(struct buffer_head *head, enum migrate_mode mode){ struct buffer_head *bh = head; /* Simple case, sync compaction */ if (mode != MIGRATE_ASYNC) { do { get_bh(bh); lock_buffer(bh); bh = bh->b_this_page; } while (bh != head); return true; } /* async case, we cannot block on lock_buffer so use trylock_buffer */ do { get_bh(bh); if (!trylock_buffer(bh)) { /* * We failed to lock the buffer and cannot stall in * async migration. Release the taken locks */ struct buffer_head *failed_bh = bh; put_bh(failed_bh); bh = head; while (bh != failed_bh) { unlock_buffer(bh); put_bh(bh); bh = bh->b_this_page; } return false; } bh = bh->b_this_page; } while (bh != head); return true;}#elsestatic inline bool buffer_migrate_lock_buffers(struct buffer_head *head, enum migrate_mode mode){ return true;}#endif /* CONFIG_BLOCK */ /* * Replace the page in the mapping. * * The number of remaining references must be: * 1 for anonymous pages without a mapping * 2 for pages with a mapping * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. */int migrate_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page, struct buffer_head *head, enum migrate_mode mode, int extra_count){ struct zone *oldzone, *newzone; int dirty; int expected_count = 1 + extra_count; void **pslot; if (!mapping) { /* Anonymous page without mapping */ if (page_count(page) != expected_count) return -EAGAIN; /* No turning back from here */ newpage->index = page->index; newpage->mapping = page->mapping; if (PageSwapBacked(page)) __SetPageSwapBacked(newpage); return MIGRATEPAGE_SUCCESS; } oldzone = page_zone(page); newzone = page_zone(newpage); spin_lock_irq(&mapping->tree_lock); pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page)); expected_count += 1 + page_has_private(page); if (page_count(page) != expected_count || radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } if (!page_ref_freeze(page, expected_count)) { spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } /* * In the async migration case of moving a page with buffers, lock the * buffers using trylock before the mapping is moved. If the mapping * was moved, we later failed to lock the buffers and could not move * the mapping back due to an elevated page count, we would have to * block waiting on other references to be dropped. */ if (mode == MIGRATE_ASYNC && head && !buffer_migrate_lock_buffers(head, mode)) { page_ref_unfreeze(page, expected_count); spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } /* * Now we know that no one else is looking at the page: * no turning back from here. */ newpage->index = page->index; newpage->mapping = page->mapping; get_page(newpage); /* add cache reference */ if (PageSwapBacked(page)) { __SetPageSwapBacked(newpage); if (PageSwapCache(page)) { SetPageSwapCache(newpage); set_page_private(newpage, page_private(page)); } } else { VM_BUG_ON_PAGE(PageSwapCache(page), page); } /* Move dirty while page refs frozen and newpage not yet exposed */ dirty = PageDirty(page); if (dirty) { ClearPageDirty(page); SetPageDirty(newpage); } radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); /* * Drop cache reference from old page by unfreezing * to one less reference. * We know this isn't the last reference. */ page_ref_unfreeze(page, expected_count - 1); spin_unlock(&mapping->tree_lock); /* Leave irq disabled to prevent preemption while updating stats */ /* * If moved to a different zone then also account * the page for that zone. Other VM counters will be * taken care of when we establish references to the * new page and drop references to the old page. * * Note that anonymous pages are accounted for * via NR_FILE_PAGES and NR_ANON_MAPPED if they * are mapped to swap space. */ if (newzone != oldzone) { __dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES); __inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES); if (PageSwapBacked(page) && !PageSwapCache(page)) { __dec_node_state(oldzone->zone_pgdat, NR_SHMEM); __inc_node_state(newzone->zone_pgdat, NR_SHMEM);Lines 1–500 of 2031. Load more
/* * Memory Migration functionality - linux/mm/migrate.c * * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter * * Page migration was first developed in the context of the memory hotplug * project. The main authors of the migration code are: * * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> * Hirokazu Takahashi <taka@valinux.co.jp> * Dave Hansen <haveblue@us.ibm.com> * Christoph Lameter */ #include <linux/migrate.h> #include <linux/export.h> #include <linux/swap.h> #include <linux/swapops.h> #include <linux/pagemap.h> #include <linux/buffer_head.h> #include <linux/mm_inline.h> #include <linux/nsproxy.h> #include <linux/pagevec.h> #include <linux/ksm.h> #include <linux/rmap.h> #include <linux/topology.h> #include <linux/cpu.h> #include <linux/cpuset.h> #include <linux/writeback.h> #include <linux/mempolicy.h> #include <linux/vmalloc.h> #include <linux/security.h> #include <linux/backing-dev.h> #include <linux/compaction.h> #include <linux/syscalls.h> #include <linux/hugetlb.h> #include <linux/hugetlb_cgroup.h> #include <linux/gfp.h> #include <linux/balloon_compaction.h> #include <linux/mmu_notifier.h> #include <linux/page_idle.h> #include <linux/page_owner.h> #include <linux/sched/mm.h> #include <linux/ptrace.h> #include <asm/tlbflush.h> #define CREATE_TRACE_POINTS #include <trace/events/migrate.h> #include "internal.h" /* * migrate_prep() needs to be called before we start compiling a list of pages * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is * undesirable, use migrate_prep_local() */ int migrate_prep(void) { /* * Clear the LRU lists so pages can be isolated. * Note that pages may be moved off the LRU after we have * drained them. Those pages will fail to migrate like other * pages that may be busy. */ lru_add_drain_all(); return 0; } /* Do the necessary work of migrate_prep but not if it involves other CPUs */ int migrate_prep_local(void) { lru_add_drain(); return 0; } int isolate_movable_page(struct page *page, isolate_mode_t mode) { struct address_space *mapping; /* * Avoid burning cycles with pages that are yet under __free_pages(), * or just got freed under us. * * In case we 'win' a race for a movable page being freed under us and * raise its refcount preventing __free_pages() from doing its job * the put_page() at the end of this block will take care of * release this page, thus avoiding a nasty leakage. */ if (unlikely(!get_page_unless_zero(page))) goto out; /* * Check PageMovable before holding a PG_lock because page's owner * assumes anybody doesn't touch PG_lock of newly allocated page * so unconditionally grapping the lock ruins page's owner side. */ if (unlikely(!__PageMovable(page))) goto out_putpage; /* * As movable pages are not isolated from LRU lists, concurrent * compaction threads can race against page migration functions * as well as race against the releasing a page. * * In order to avoid having an already isolated movable page * being (wrongly) re-isolated while it is under migration, * or to avoid attempting to isolate pages being released, * lets be sure we have the page lock * before proceeding with the movable page isolation steps. */ if (unlikely(!trylock_page(page))) goto out_putpage; if (!PageMovable(page) || PageIsolated(page)) goto out_no_isolated; mapping = page_mapping(page); VM_BUG_ON_PAGE(!mapping, page); if (!mapping->a_ops->isolate_page(page, mode)) goto out_no_isolated; /* Driver shouldn't use PG_isolated bit of page->flags */ WARN_ON_ONCE(PageIsolated(page)); __SetPageIsolated(page); unlock_page(page); return 0; out_no_isolated: unlock_page(page); out_putpage: put_page(page); out: return -EBUSY; } /* It should be called on page which is PG_movable */ void putback_movable_page(struct page *page) { struct address_space *mapping; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageMovable(page), page); VM_BUG_ON_PAGE(!PageIsolated(page), page); mapping = page_mapping(page); mapping->a_ops->putback_page(page); __ClearPageIsolated(page); } /* * Put previously isolated pages back onto the appropriate lists * from where they were once taken off for compaction/migration. * * This function shall be used whenever the isolated pageset has been * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() * and isolate_huge_page(). */ void putback_movable_pages(struct list_head *l) { struct page *page; struct page *page2; list_for_each_entry_safe(page, page2, l, lru) { if (unlikely(PageHuge(page))) { putback_active_hugepage(page); continue; } list_del(&page->lru); /* * We isolated non-lru movable page so here we can use * __PageMovable because LRU page's mapping cannot have * PAGE_MAPPING_MOVABLE. */ if (unlikely(__PageMovable(page))) { VM_BUG_ON_PAGE(!PageIsolated(page), page); lock_page(page); if (PageMovable(page)) putback_movable_page(page); else __ClearPageIsolated(page); unlock_page(page); put_page(page); } else { dec_node_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page)); putback_lru_page(page); } } } /* * Restore a potential migration pte to a working pte entry */ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, unsigned long addr, void *old) { struct page_vma_mapped_walk pvmw = { .page = old, .vma = vma, .address = addr, .flags = PVMW_SYNC | PVMW_MIGRATION, }; struct page *new; pte_t pte; swp_entry_t entry; VM_BUG_ON_PAGE(PageTail(page), page); while (page_vma_mapped_walk(&pvmw)) { if (PageKsm(page)) new = page; else new = page - pvmw.page->index + linear_page_index(vma, pvmw.address); get_page(new); pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot))); if (pte_swp_soft_dirty(*pvmw.pte)) pte = pte_mksoft_dirty(pte); /* * Recheck VMA as permissions can change since migration started */ entry = pte_to_swp_entry(*pvmw.pte); if (is_write_migration_entry(entry)) pte = maybe_mkwrite(pte, vma); flush_dcache_page(new); #ifdef CONFIG_HUGETLB_PAGE if (PageHuge(new)) { pte = pte_mkhuge(pte); pte = arch_make_huge_pte(pte, vma, new, 0); set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); if (PageAnon(new)) hugepage_add_anon_rmap(new, vma, pvmw.address); else page_dup_rmap(new, true); } else #endif { set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); if (PageAnon(new)) page_add_anon_rmap(new, vma, pvmw.address, false); else page_add_file_rmap(new, false); } if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new)) mlock_vma_page(new); /* No need to invalidate - it was non-present before */ update_mmu_cache(vma, pvmw.address, pvmw.pte); } return true; } /* * Get rid of all migration entries and replace them by * references to the indicated page. */ void remove_migration_ptes(struct page *old, struct page *new, bool locked) { struct rmap_walk_control rwc = { .rmap_one = remove_migration_pte, .arg = old, }; if (locked) rmap_walk_locked(new, &rwc); else rmap_walk(new, &rwc); } /* * Something used the pte of a page under migration. We need to * get to the page and wait until migration is finished. * When we return from this function the fault will be retried. */ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, spinlock_t *ptl) { pte_t pte; swp_entry_t entry; struct page *page; spin_lock(ptl); pte = *ptep; if (!is_swap_pte(pte)) goto out; entry = pte_to_swp_entry(pte); if (!is_migration_entry(entry)) goto out; page = migration_entry_to_page(entry); /* * Once radix-tree replacement of page migration started, page_count * *must* be zero. And, we don't want to call wait_on_page_locked() * against a page without get_page(). * So, we use get_page_unless_zero(), here. Even failed, page fault * will occur again. */ if (!get_page_unless_zero(page)) goto out; pte_unmap_unlock(ptep, ptl); wait_on_page_locked(page); put_page(page); return; out: pte_unmap_unlock(ptep, ptl); } void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, unsigned long address) { spinlock_t *ptl = pte_lockptr(mm, pmd); pte_t *ptep = pte_offset_map(pmd, address); __migration_entry_wait(mm, ptep, ptl); } void migration_entry_wait_huge(struct vm_area_struct *vma, struct mm_struct *mm, pte_t *pte) { spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte); __migration_entry_wait(mm, pte, ptl); } #ifdef CONFIG_BLOCK /* Returns true if all buffers are successfully locked */ static bool buffer_migrate_lock_buffers(struct buffer_head *head, enum migrate_mode mode) { struct buffer_head *bh = head; /* Simple case, sync compaction */ if (mode != MIGRATE_ASYNC) { do { get_bh(bh); lock_buffer(bh); bh = bh->b_this_page; } while (bh != head); return true; } /* async case, we cannot block on lock_buffer so use trylock_buffer */ do { get_bh(bh); if (!trylock_buffer(bh)) { /* * We failed to lock the buffer and cannot stall in * async migration. Release the taken locks */ struct buffer_head *failed_bh = bh; put_bh(failed_bh); bh = head; while (bh != failed_bh) { unlock_buffer(bh); put_bh(bh); bh = bh->b_this_page; } return false; } bh = bh->b_this_page; } while (bh != head); return true; } #else static inline bool buffer_migrate_lock_buffers(struct buffer_head *head, enum migrate_mode mode) { return true; } #endif /* CONFIG_BLOCK */ /* * Replace the page in the mapping. * * The number of remaining references must be: * 1 for anonymous pages without a mapping * 2 for pages with a mapping * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. */ int migrate_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page, struct buffer_head *head, enum migrate_mode mode, int extra_count) { struct zone *oldzone, *newzone; int dirty; int expected_count = 1 + extra_count; void **pslot; if (!mapping) { /* Anonymous page without mapping */ if (page_count(page) != expected_count) return -EAGAIN; /* No turning back from here */ newpage->index = page->index; newpage->mapping = page->mapping; if (PageSwapBacked(page)) __SetPageSwapBacked(newpage); return MIGRATEPAGE_SUCCESS; } oldzone = page_zone(page); newzone = page_zone(newpage); spin_lock_irq(&mapping->tree_lock); pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page)); expected_count += 1 + page_has_private(page); if (page_count(page) != expected_count || radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } if (!page_ref_freeze(page, expected_count)) { spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } /* * In the async migration case of moving a page with buffers, lock the * buffers using trylock before the mapping is moved. If the mapping * was moved, we later failed to lock the buffers and could not move * the mapping back due to an elevated page count, we would have to * block waiting on other references to be dropped. */ if (mode == MIGRATE_ASYNC && head && !buffer_migrate_lock_buffers(head, mode)) { page_ref_unfreeze(page, expected_count); spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } /* * Now we know that no one else is looking at the page: * no turning back from here. */ newpage->index = page->index; newpage->mapping = page->mapping; get_page(newpage); /* add cache reference */ if (PageSwapBacked(page)) { __SetPageSwapBacked(newpage); if (PageSwapCache(page)) { SetPageSwapCache(newpage); set_page_private(newpage, page_private(page)); } } else { VM_BUG_ON_PAGE(PageSwapCache(page), page); } /* Move dirty while page refs frozen and newpage not yet exposed */ dirty = PageDirty(page); if (dirty) { ClearPageDirty(page); SetPageDirty(newpage); } radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); /* * Drop cache reference from old page by unfreezing * to one less reference. * We know this isn't the last reference. */ page_ref_unfreeze(page, expected_count - 1); spin_unlock(&mapping->tree_lock); /* Leave irq disabled to prevent preemption while updating stats */ /* * If moved to a different zone then also account * the page for that zone. Other VM counters will be * taken care of when we establish references to the * new page and drop references to the old page. * * Note that anonymous pages are accounted for * via NR_FILE_PAGES and NR_ANON_MAPPED if they * are mapped to swap space. */ if (newzone != oldzone) { __dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES); __inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES); if (PageSwapBacked(page) && !PageSwapCache(page)) { __dec_node_state(oldzone->zone_pgdat, NR_SHMEM); __inc_node_state(newzone->zone_pgdat, NR_SHMEM);