// SPDX-License-Identifier: GPL-2.0/* * Device Memory Migration functionality. * * Originally written by Jérôme Glisse. */#include <linux/export.h>#include <linux/memremap.h>#include <linux/migrate.h>#include <linux/mm.h>#include <linux/mm_inline.h>#include <linux/mmu_notifier.h>#include <linux/oom.h>#include <linux/pagewalk.h>#include <linux/rmap.h>#include <linux/leafops.h>#include <linux/pgalloc.h>#include <asm/tlbflush.h>#include "internal.h" static int migrate_vma_collect_skip(unsigned long start, unsigned long end, struct mm_walk *walk){ struct migrate_vma *migrate = walk->private; unsigned long addr; for (addr = start; addr < end; addr += PAGE_SIZE) { migrate->dst[migrate->npages] = 0; migrate->src[migrate->npages++] = 0; } return 0;} static int migrate_vma_collect_hole(unsigned long start, unsigned long end, __always_unused int depth, struct mm_walk *walk){ struct migrate_vma *migrate = walk->private; unsigned long addr; /* Only allow populating anonymous memory. */ if (!vma_is_anonymous(walk->vma)) return migrate_vma_collect_skip(start, end, walk); if (thp_migration_supported() && (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) && (IS_ALIGNED(start, HPAGE_PMD_SIZE) && IS_ALIGNED(end, HPAGE_PMD_SIZE))) { migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE | MIGRATE_PFN_COMPOUND; migrate->dst[migrate->npages] = 0; migrate->npages++; migrate->cpages++; /* * Collect the remaining entries as holes, in case we * need to split later */ return migrate_vma_collect_skip(start + PAGE_SIZE, end, walk); } for (addr = start; addr < end; addr += PAGE_SIZE) { migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE; migrate->dst[migrate->npages] = 0; migrate->npages++; migrate->cpages++; } return 0;} /** * migrate_vma_split_folio() - Helper function to split a THP folio * @folio: the folio to split * @fault_page: struct page associated with the fault if any * * If @folio is not the folio containing @fault_page, the caller must hold a * reference on @folio. The helper consumes that reference. * * Returns 0 on success */static int migrate_vma_split_folio(struct folio *folio, struct page *fault_page){ int ret; struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL; struct folio *new_fault_folio = NULL; if (folio != fault_folio) folio_lock(folio); ret = split_folio(folio); if (ret) { if (folio != fault_folio) { folio_unlock(folio); folio_put(folio); } return ret; } new_fault_folio = fault_page ? page_folio(fault_page) : NULL; /* * Ensure the lock is held on the correct * folio after the split */ if (!new_fault_folio) { folio_unlock(folio); folio_put(folio); } else if (folio != new_fault_folio) { if (new_fault_folio != fault_folio) { folio_get(new_fault_folio); folio_lock(new_fault_folio); } folio_unlock(folio); folio_put(folio); } return 0;} /** migrate_vma_collect_huge_pmd - collect THP pages without splitting the * folio for device private pages. * @pmdp: pointer to pmd entry * @start: start address of the range for migration * @end: end address of the range for migration * @walk: mm_walk callback structure * @fault_folio: folio associated with the fault if any * * Collect the huge pmd entry at @pmdp for migration and set the * MIGRATE_PFN_COMPOUND flag in the migrate src entry to indicate that * migration will occur at HPAGE_PMD granularity */static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, unsigned long end, struct mm_walk *walk, struct folio *fault_folio){ struct mm_struct *mm = walk->mm; struct folio *folio; struct migrate_vma *migrate = walk->private; spinlock_t *ptl; int ret; unsigned long write = 0; ptl = pmd_lock(mm, pmdp); if (pmd_none(*pmdp)) { spin_unlock(ptl); return migrate_vma_collect_hole(start, end, -1, walk); } if (pmd_trans_huge(*pmdp)) { if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } folio = pmd_folio(*pmdp); if (is_huge_zero_folio(folio)) { spin_unlock(ptl); return migrate_vma_collect_hole(start, end, -1, walk); } if (pmd_write(*pmdp)) write = MIGRATE_PFN_WRITE; } else if (!pmd_present(*pmdp)) { const softleaf_t entry = softleaf_from_pmd(*pmdp); if (!softleaf_is_device_private(entry) || !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } folio = softleaf_to_folio(entry); if (folio->pgmap->owner != migrate->pgmap_owner) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } if (softleaf_is_device_private_write(entry)) write = MIGRATE_PFN_WRITE; } else { spin_unlock(ptl); return -EAGAIN; } folio_get(folio); if (folio != fault_folio && unlikely(!folio_trylock(folio))) { spin_unlock(ptl); folio_put(folio); return migrate_vma_collect_skip(start, end, walk); } if (thp_migration_supported() && (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) && (IS_ALIGNED(start, HPAGE_PMD_SIZE) && IS_ALIGNED(end, HPAGE_PMD_SIZE))) { struct page_vma_mapped_walk pvmw = { .ptl = ptl, .address = start, .pmd = pmdp, .vma = walk->vma, }; unsigned long pfn = page_to_pfn(folio_page(folio, 0)); migrate->src[migrate->npages] = migrate_pfn(pfn) | write | MIGRATE_PFN_MIGRATE | MIGRATE_PFN_COMPOUND; migrate->dst[migrate->npages++] = 0; migrate->cpages++; ret = set_pmd_migration_entry(&pvmw, folio_page(folio, 0)); if (ret) { migrate->npages--; migrate->cpages--; migrate->src[migrate->npages] = 0; migrate->dst[migrate->npages] = 0; goto fallback; } migrate_vma_collect_skip(start + PAGE_SIZE, end, walk); spin_unlock(ptl); return 0; } fallback: spin_unlock(ptl); if (!folio_test_large(folio)) goto done; ret = split_folio(folio); if (fault_folio != folio) folio_unlock(folio); folio_put(folio); if (ret) return migrate_vma_collect_skip(start, end, walk); if (pmd_none(pmdp_get_lockless(pmdp))) return migrate_vma_collect_hole(start, end, -1, walk); done: return -ENOENT;} static int migrate_vma_collect_pmd(pmd_t *pmdp, unsigned long start, unsigned long end, struct mm_walk *walk){ struct migrate_vma *migrate = walk->private; struct vm_area_struct *vma = walk->vma; struct mm_struct *mm = vma->vm_mm; unsigned long addr = start, unmapped = 0; spinlock_t *ptl; struct folio *fault_folio = migrate->fault_page ? page_folio(migrate->fault_page) : NULL; pte_t *ptep; again: if (pmd_trans_huge(*pmdp) || !pmd_present(*pmdp)) { int ret = migrate_vma_collect_huge_pmd(pmdp, start, end, walk, fault_folio); if (ret == -EAGAIN) goto again; if (ret == 0) return 0; } ptep = pte_offset_map_lock(mm, pmdp, start, &ptl); if (!ptep) goto again; lazy_mmu_mode_enable(); ptep += (addr - start) / PAGE_SIZE; for (; addr < end; addr += PAGE_SIZE, ptep++) { struct dev_pagemap *pgmap; unsigned long mpfn = 0, pfn; struct folio *folio; struct page *page; softleaf_t entry; pte_t pte; pte = ptep_get(ptep); if (pte_none(pte)) { if (vma_is_anonymous(vma)) { mpfn = MIGRATE_PFN_MIGRATE; migrate->cpages++; } goto next; } if (!pte_present(pte)) { /* * Only care about unaddressable device page special * page table entry. Other special swap entries are not * migratable, and we ignore regular swapped page. */ entry = softleaf_from_pte(pte); if (!softleaf_is_device_private(entry)) goto next; page = softleaf_to_page(entry); pgmap = page_pgmap(page); if (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) || pgmap->owner != migrate->pgmap_owner) goto next; folio = page_folio(page); if (folio_test_large(folio)) { int ret; /* migrate_vma_split_folio() consumes this reference */ if (folio != fault_folio) folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, migrate->fault_page); if (ret) { if (unmapped) flush_tlb_range(walk->vma, start, end); return migrate_vma_collect_skip(addr, end, walk); } goto again; } mpfn = migrate_pfn(page_to_pfn(page)) | MIGRATE_PFN_MIGRATE; if (softleaf_is_device_private_write(entry)) mpfn |= MIGRATE_PFN_WRITE; } else { pfn = pte_pfn(pte); if (is_zero_pfn(pfn) && (migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { mpfn = MIGRATE_PFN_MIGRATE; migrate->cpages++; goto next; } page = vm_normal_page(migrate->vma, addr, pte); if (page && !is_zone_device_page(page) && !(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { goto next; } else if (page && is_device_coherent_page(page)) { pgmap = page_pgmap(page); if (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_COHERENT) || pgmap->owner != migrate->pgmap_owner) goto next; } folio = page ? page_folio(page) : NULL; if (folio && folio_test_large(folio)) { int ret; /* migrate_vma_split_folio() consumes this reference */ if (folio != fault_folio) folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, migrate->fault_page); if (ret) { if (unmapped) flush_tlb_range(walk->vma, start, end); return migrate_vma_collect_skip(addr, end, walk); } goto again; } mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE; mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0; } if (!page || !page->mapping) { mpfn = 0; goto next; } /* * By getting a reference on the folio we pin it and that blocks * any kind of migration. Side effect is that it "freezes" the * pte. * * We drop this reference after isolating the folio from the lru * for non device folio (device folio are not on the lru and thus * can't be dropped from it). */ folio = page_folio(page); folio_get(folio); /* * We rely on folio_trylock() to avoid deadlock between * concurrent migrations where each is waiting on the others * folio lock. If we can't immediately lock the folio we fail this * migration as it is only best effort anyway. * * If we can lock the folio it's safe to set up a migration entry * now. In the common case where the folio is mapped once in a * single process setting up the migration entry now is an * optimisation to avoid walking the rmap later with * try_to_migrate(). */ if (fault_folio == folio || folio_trylock(folio)) { bool anon_exclusive; pte_t swp_pte; if (pte_present(pte)) flush_cache_page(vma, addr, pte_pfn(pte)); anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); if (anon_exclusive) { pte = ptep_clear_flush(vma, addr, ptep); if (folio_try_share_anon_rmap_pte(folio, page)) { set_pte_at(mm, addr, ptep, pte); if (fault_folio != folio) folio_unlock(folio); folio_put(folio); mpfn = 0; goto next; } } else { pte = ptep_get_and_clear(mm, addr, ptep); } migrate->cpages++; /* Set the dirty flag on the folio now the pte is gone. */ if (pte_present(pte) && pte_dirty(pte)) folio_mark_dirty(folio); /* Setup special migration page table entry */ if (mpfn & MIGRATE_PFN_WRITE) entry = make_writable_migration_entry( page_to_pfn(page)); else if (anon_exclusive) entry = make_readable_exclusive_migration_entry( page_to_pfn(page)); else entry = make_readable_migration_entry( page_to_pfn(page)); if (pte_present(pte)) { if (pte_young(pte)) entry = make_migration_entry_young(entry); if (pte_dirty(pte)) entry = make_migration_entry_dirty(entry); } swp_pte = swp_entry_to_pte(entry); if (pte_present(pte)) { if (pte_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); if (pte_uffd(pte)) swp_pte = pte_swp_mkuffd(swp_pte); } else { if (pte_swp_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); if (pte_swp_uffd(pte)) swp_pte = pte_swp_mkuffd(swp_pte); } set_pte_at(mm, addr, ptep, swp_pte); /* * This is like regular unmap: we remove the rmap and * drop the folio refcount. The folio won't be freed, as * we took a reference just above. */ folio_remove_rmap_pte(folio, page, vma); folio_put(folio); if (pte_present(pte)) unmapped++; } else { folio_put(folio); mpfn = 0; } next: migrate->dst[migrate->npages] = 0; migrate->src[migrate->npages++] = mpfn; } /* Only flush the TLB if we actually modified any entries */ if (unmapped) flush_tlb_range(walk->vma, start, end); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep - 1, ptl); return 0;} static const struct mm_walk_ops migrate_vma_walk_ops = { .pmd_entry = migrate_vma_collect_pmd,Lines 1–500 of 1530. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Device Memory Migration functionality. * * Originally written by Jérôme Glisse. */ #include <linux/export.h> #include <linux/memremap.h> #include <linux/migrate.h> #include <linux/mm.h> #include <linux/mm_inline.h> #include <linux/mmu_notifier.h> #include <linux/oom.h> #include <linux/pagewalk.h> #include <linux/rmap.h> #include <linux/leafops.h> #include <linux/pgalloc.h> #include <asm/tlbflush.h> #include "internal.h" static int migrate_vma_collect_skip(unsigned long start, unsigned long end, struct mm_walk *walk) { struct migrate_vma *migrate = walk->private; unsigned long addr; for (addr = start; addr < end; addr += PAGE_SIZE) { migrate->dst[migrate->npages] = 0; migrate->src[migrate->npages++] = 0; } return 0; } static int migrate_vma_collect_hole(unsigned long start, unsigned long end, __always_unused int depth, struct mm_walk *walk) { struct migrate_vma *migrate = walk->private; unsigned long addr; /* Only allow populating anonymous memory. */ if (!vma_is_anonymous(walk->vma)) return migrate_vma_collect_skip(start, end, walk); if (thp_migration_supported() && (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) && (IS_ALIGNED(start, HPAGE_PMD_SIZE) && IS_ALIGNED(end, HPAGE_PMD_SIZE))) { migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE | MIGRATE_PFN_COMPOUND; migrate->dst[migrate->npages] = 0; migrate->npages++; migrate->cpages++; /* * Collect the remaining entries as holes, in case we * need to split later */ return migrate_vma_collect_skip(start + PAGE_SIZE, end, walk); } for (addr = start; addr < end; addr += PAGE_SIZE) { migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE; migrate->dst[migrate->npages] = 0; migrate->npages++; migrate->cpages++; } return 0; } /** * migrate_vma_split_folio() - Helper function to split a THP folio * @folio: the folio to split * @fault_page: struct page associated with the fault if any * * If @folio is not the folio containing @fault_page, the caller must hold a * reference on @folio. The helper consumes that reference. * * Returns 0 on success */ static int migrate_vma_split_folio(struct folio *folio, struct page *fault_page) { int ret; struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL; struct folio *new_fault_folio = NULL; if (folio != fault_folio) folio_lock(folio); ret = split_folio(folio); if (ret) { if (folio != fault_folio) { folio_unlock(folio); folio_put(folio); } return ret; } new_fault_folio = fault_page ? page_folio(fault_page) : NULL; /* * Ensure the lock is held on the correct * folio after the split */ if (!new_fault_folio) { folio_unlock(folio); folio_put(folio); } else if (folio != new_fault_folio) { if (new_fault_folio != fault_folio) { folio_get(new_fault_folio); folio_lock(new_fault_folio); } folio_unlock(folio); folio_put(folio); } return 0; } /** migrate_vma_collect_huge_pmd - collect THP pages without splitting the * folio for device private pages. * @pmdp: pointer to pmd entry * @start: start address of the range for migration * @end: end address of the range for migration * @walk: mm_walk callback structure * @fault_folio: folio associated with the fault if any * * Collect the huge pmd entry at @pmdp for migration and set the * MIGRATE_PFN_COMPOUND flag in the migrate src entry to indicate that * migration will occur at HPAGE_PMD granularity */ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, unsigned long end, struct mm_walk *walk, struct folio *fault_folio) { struct mm_struct *mm = walk->mm; struct folio *folio; struct migrate_vma *migrate = walk->private; spinlock_t *ptl; int ret; unsigned long write = 0; ptl = pmd_lock(mm, pmdp); if (pmd_none(*pmdp)) { spin_unlock(ptl); return migrate_vma_collect_hole(start, end, -1, walk); } if (pmd_trans_huge(*pmdp)) { if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } folio = pmd_folio(*pmdp); if (is_huge_zero_folio(folio)) { spin_unlock(ptl); return migrate_vma_collect_hole(start, end, -1, walk); } if (pmd_write(*pmdp)) write = MIGRATE_PFN_WRITE; } else if (!pmd_present(*pmdp)) { const softleaf_t entry = softleaf_from_pmd(*pmdp); if (!softleaf_is_device_private(entry) || !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } folio = softleaf_to_folio(entry); if (folio->pgmap->owner != migrate->pgmap_owner) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } if (softleaf_is_device_private_write(entry)) write = MIGRATE_PFN_WRITE; } else { spin_unlock(ptl); return -EAGAIN; } folio_get(folio); if (folio != fault_folio && unlikely(!folio_trylock(folio))) { spin_unlock(ptl); folio_put(folio); return migrate_vma_collect_skip(start, end, walk); } if (thp_migration_supported() && (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) && (IS_ALIGNED(start, HPAGE_PMD_SIZE) && IS_ALIGNED(end, HPAGE_PMD_SIZE))) { struct page_vma_mapped_walk pvmw = { .ptl = ptl, .address = start, .pmd = pmdp, .vma = walk->vma, }; unsigned long pfn = page_to_pfn(folio_page(folio, 0)); migrate->src[migrate->npages] = migrate_pfn(pfn) | write | MIGRATE_PFN_MIGRATE | MIGRATE_PFN_COMPOUND; migrate->dst[migrate->npages++] = 0; migrate->cpages++; ret = set_pmd_migration_entry(&pvmw, folio_page(folio, 0)); if (ret) { migrate->npages--; migrate->cpages--; migrate->src[migrate->npages] = 0; migrate->dst[migrate->npages] = 0; goto fallback; } migrate_vma_collect_skip(start + PAGE_SIZE, end, walk); spin_unlock(ptl); return 0; } fallback: spin_unlock(ptl); if (!folio_test_large(folio)) goto done; ret = split_folio(folio); if (fault_folio != folio) folio_unlock(folio); folio_put(folio); if (ret) return migrate_vma_collect_skip(start, end, walk); if (pmd_none(pmdp_get_lockless(pmdp))) return migrate_vma_collect_hole(start, end, -1, walk); done: return -ENOENT; } static int migrate_vma_collect_pmd(pmd_t *pmdp, unsigned long start, unsigned long end, struct mm_walk *walk) { struct migrate_vma *migrate = walk->private; struct vm_area_struct *vma = walk->vma; struct mm_struct *mm = vma->vm_mm; unsigned long addr = start, unmapped = 0; spinlock_t *ptl; struct folio *fault_folio = migrate->fault_page ? page_folio(migrate->fault_page) : NULL; pte_t *ptep; again: if (pmd_trans_huge(*pmdp) || !pmd_present(*pmdp)) { int ret = migrate_vma_collect_huge_pmd(pmdp, start, end, walk, fault_folio); if (ret == -EAGAIN) goto again; if (ret == 0) return 0; } ptep = pte_offset_map_lock(mm, pmdp, start, &ptl); if (!ptep) goto again; lazy_mmu_mode_enable(); ptep += (addr - start) / PAGE_SIZE; for (; addr < end; addr += PAGE_SIZE, ptep++) { struct dev_pagemap *pgmap; unsigned long mpfn = 0, pfn; struct folio *folio; struct page *page; softleaf_t entry; pte_t pte; pte = ptep_get(ptep); if (pte_none(pte)) { if (vma_is_anonymous(vma)) { mpfn = MIGRATE_PFN_MIGRATE; migrate->cpages++; } goto next; } if (!pte_present(pte)) { /* * Only care about unaddressable device page special * page table entry. Other special swap entries are not * migratable, and we ignore regular swapped page. */ entry = softleaf_from_pte(pte); if (!softleaf_is_device_private(entry)) goto next; page = softleaf_to_page(entry); pgmap = page_pgmap(page); if (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) || pgmap->owner != migrate->pgmap_owner) goto next; folio = page_folio(page); if (folio_test_large(folio)) { int ret; /* migrate_vma_split_folio() consumes this reference */ if (folio != fault_folio) folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, migrate->fault_page); if (ret) { if (unmapped) flush_tlb_range(walk->vma, start, end); return migrate_vma_collect_skip(addr, end, walk); } goto again; } mpfn = migrate_pfn(page_to_pfn(page)) | MIGRATE_PFN_MIGRATE; if (softleaf_is_device_private_write(entry)) mpfn |= MIGRATE_PFN_WRITE; } else { pfn = pte_pfn(pte); if (is_zero_pfn(pfn) && (migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { mpfn = MIGRATE_PFN_MIGRATE; migrate->cpages++; goto next; } page = vm_normal_page(migrate->vma, addr, pte); if (page && !is_zone_device_page(page) && !(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { goto next; } else if (page && is_device_coherent_page(page)) { pgmap = page_pgmap(page); if (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_COHERENT) || pgmap->owner != migrate->pgmap_owner) goto next; } folio = page ? page_folio(page) : NULL; if (folio && folio_test_large(folio)) { int ret; /* migrate_vma_split_folio() consumes this reference */ if (folio != fault_folio) folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, migrate->fault_page); if (ret) { if (unmapped) flush_tlb_range(walk->vma, start, end); return migrate_vma_collect_skip(addr, end, walk); } goto again; } mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE; mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0; } if (!page || !page->mapping) { mpfn = 0; goto next; } /* * By getting a reference on the folio we pin it and that blocks * any kind of migration. Side effect is that it "freezes" the * pte. * * We drop this reference after isolating the folio from the lru * for non device folio (device folio are not on the lru and thus * can't be dropped from it). */ folio = page_folio(page); folio_get(folio); /* * We rely on folio_trylock() to avoid deadlock between * concurrent migrations where each is waiting on the others * folio lock. If we can't immediately lock the folio we fail this * migration as it is only best effort anyway. * * If we can lock the folio it's safe to set up a migration entry * now. In the common case where the folio is mapped once in a * single process setting up the migration entry now is an * optimisation to avoid walking the rmap later with * try_to_migrate(). */ if (fault_folio == folio || folio_trylock(folio)) { bool anon_exclusive; pte_t swp_pte; if (pte_present(pte)) flush_cache_page(vma, addr, pte_pfn(pte)); anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); if (anon_exclusive) { pte = ptep_clear_flush(vma, addr, ptep); if (folio_try_share_anon_rmap_pte(folio, page)) { set_pte_at(mm, addr, ptep, pte); if (fault_folio != folio) folio_unlock(folio); folio_put(folio); mpfn = 0; goto next; } } else { pte = ptep_get_and_clear(mm, addr, ptep); } migrate->cpages++; /* Set the dirty flag on the folio now the pte is gone. */ if (pte_present(pte) && pte_dirty(pte)) folio_mark_dirty(folio); /* Setup special migration page table entry */ if (mpfn & MIGRATE_PFN_WRITE) entry = make_writable_migration_entry( page_to_pfn(page)); else if (anon_exclusive) entry = make_readable_exclusive_migration_entry( page_to_pfn(page)); else entry = make_readable_migration_entry( page_to_pfn(page)); if (pte_present(pte)) { if (pte_young(pte)) entry = make_migration_entry_young(entry); if (pte_dirty(pte)) entry = make_migration_entry_dirty(entry); } swp_pte = swp_entry_to_pte(entry); if (pte_present(pte)) { if (pte_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); if (pte_uffd(pte)) swp_pte = pte_swp_mkuffd(swp_pte); } else { if (pte_swp_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); if (pte_swp_uffd(pte)) swp_pte = pte_swp_mkuffd(swp_pte); } set_pte_at(mm, addr, ptep, swp_pte); /* * This is like regular unmap: we remove the rmap and * drop the folio refcount. The folio won't be freed, as * we took a reference just above. */ folio_remove_rmap_pte(folio, page, vma); folio_put(folio); if (pte_present(pte)) unmapped++; } else { folio_put(folio); mpfn = 0; } next: migrate->dst[migrate->npages] = 0; migrate->src[migrate->npages++] = mpfn; } /* Only flush the TLB if we actually modified any entries */ if (unmapped) flush_tlb_range(walk->vma, start, end); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep - 1, ptl); return 0; } static const struct mm_walk_ops migrate_vma_walk_ops = { .pmd_entry = migrate_vma_collect_pmd,