// SPDX-License-Identifier: GPL-2.0-only/* * mm/userfaultfd.c * * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org> * Copyright (C) 2008-2009 Red Hat, Inc. * Copyright (C) 2015 Red Hat, Inc. * * Some part derived from fs/eventfd.c (anon inode setup) and * mm/ksm.c (mm hashing). */ #include <linux/mm.h>#include <linux/sched/signal.h>#include <linux/pagemap.h>#include <linux/rmap.h>#include <linux/swap.h>#include <linux/leafops.h>#include <linux/userfaultfd_k.h>#include <linux/mmu_notifier.h>#include <linux/hugetlb.h>#include <linux/list.h>#include <linux/sched/mm.h>#include <linux/mm_inline.h>#include <linux/poll.h>#include <linux/slab.h>#include <linux/seq_file.h>#include <linux/bug.h>#include <linux/anon_inodes.h>#include <linux/syscalls.h>#include <linux/miscdevice.h>#include <linux/uio.h>#include <linux/file.h>#include <linux/cleanup.h>#include <asm/tlbflush.h>#include <asm/tlb.h>#include "internal.h"#include "swap.h" struct mfill_state { struct userfaultfd_ctx *ctx; unsigned long src_start; unsigned long dst_start; unsigned long len; uffd_flags_t flags; struct vm_area_struct *vma; unsigned long src_addr; unsigned long dst_addr; pmd_t *pmd;}; static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags){ /* anonymous memory does not support MINOR mode */ if (vm_flags & VM_UFFD_MINOR) return false; return true;} static struct folio *anon_alloc_folio(struct vm_area_struct *vma, unsigned long addr){ struct folio *folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr); if (!folio) return NULL; if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL)) { folio_put(folio); return NULL; } return folio;} static const struct vm_uffd_ops anon_uffd_ops = { .can_userfault = anon_can_userfault, .alloc_folio = anon_alloc_folio,}; static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma){ if (vma_is_anonymous(vma)) return &anon_uffd_ops; return vma->vm_ops->uffd_ops;} static __always_inlinebool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end){ /* Make sure that the dst range is fully within dst_vma. */ if (dst_end > dst_vma->vm_end) return false; /* * Check the vma is registered in uffd, this is required to * enforce the VM_MAYWRITE check done at uffd registration * time. */ if (!dst_vma->vm_userfaultfd_ctx.ctx) return false; return true;} static __always_inlinestruct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, unsigned long addr){ struct vm_area_struct *vma; mmap_assert_locked(mm); vma = vma_lookup(mm, addr); if (!vma) vma = ERR_PTR(-ENOENT); else if (!(vma->vm_flags & VM_SHARED) && unlikely(anon_vma_prepare(vma))) vma = ERR_PTR(-ENOMEM); return vma;} #ifdef CONFIG_PER_VMA_LOCK/* * uffd_lock_vma() - Lookup and lock vma corresponding to @address. * @mm: mm to search vma in. * @address: address that the vma should contain. * * Should be called without holding mmap_lock. * * Return: A locked vma containing @address, -ENOENT if no vma is found, or * -ENOMEM if anon_vma couldn't be allocated. */static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm, unsigned long address){ struct vm_area_struct *vma; vma = lock_vma_under_rcu(mm, address); if (vma) { /* * We know we're going to need to use anon_vma, so check * that early. */ if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma)) vma_end_read(vma); else return vma; } mmap_read_lock(mm); vma = find_vma_and_prepare_anon(mm, address); if (!IS_ERR(vma)) { bool locked = vma_start_read_locked(vma); if (!locked) vma = ERR_PTR(-EAGAIN); } mmap_read_unlock(mm); return vma;} static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long len){ struct vm_area_struct *dst_vma; dst_vma = uffd_lock_vma(dst_mm, dst_start); if (IS_ERR(dst_vma) || validate_dst_vma(dst_vma, dst_start + len)) return dst_vma; vma_end_read(dst_vma); return ERR_PTR(-ENOENT);} static void uffd_mfill_unlock(struct vm_area_struct *vma){ vma_end_read(vma);} #else static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long len){ struct vm_area_struct *dst_vma; mmap_read_lock(dst_mm); dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start); if (IS_ERR(dst_vma)) goto out_unlock; if (validate_dst_vma(dst_vma, dst_start + len)) return dst_vma; dst_vma = ERR_PTR(-ENOENT);out_unlock: mmap_read_unlock(dst_mm); return dst_vma;} static void uffd_mfill_unlock(struct vm_area_struct *vma){ mmap_read_unlock(vma->vm_mm);}#endif static void mfill_put_vma(struct mfill_state *state){ if (!state->vma) return; up_read(&state->ctx->map_changing_lock); uffd_mfill_unlock(state->vma); state->vma = NULL;} static int mfill_get_vma(struct mfill_state *state){ struct userfaultfd_ctx *ctx = state->ctx; uffd_flags_t flags = state->flags; struct vm_area_struct *dst_vma; const struct vm_uffd_ops *ops; int err; /* * Make sure the vma is not shared, that the dst range is * both valid and fully within a single existing vma. */ dst_vma = uffd_mfill_lock(ctx->mm, state->dst_start, state->len); if (IS_ERR(dst_vma)) return PTR_ERR(dst_vma); /* * If memory mappings are changing because of non-cooperative * operation (e.g. mremap) running in parallel, bail out and * request the user to retry later */ down_read(&ctx->map_changing_lock); state->vma = dst_vma; err = -EAGAIN; if (atomic_read(&ctx->mmap_changing)) goto out_unlock; err = -EINVAL; /* * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but * it will overwrite vm_ops, so vma_is_anonymous must return false. */ if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) && dst_vma->vm_flags & VM_SHARED)) goto out_unlock; /* * validate 'mode' now that we know the dst_vma: don't allow * a wrprotect copy if the userfaultfd didn't register as WP. */ if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) goto out_unlock; if (is_vm_hugetlb_page(dst_vma)) return 0; ops = vma_uffd_ops(dst_vma); if (!ops) goto out_unlock; if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE) && !ops->get_folio_noalloc) goto out_unlock; return 0; out_unlock: mfill_put_vma(state); return err;} static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address){ pgd_t *pgd; p4d_t *p4d; pud_t *pud; pgd = pgd_offset(mm, address); p4d = p4d_alloc(mm, pgd, address); if (!p4d) return NULL; pud = pud_alloc(mm, p4d, address); if (!pud) return NULL; /* * Note that we didn't run this because the pmd was * missing, the *pmd may be already established and in * turn it may also be a trans_huge_pmd. */ return pmd_alloc(mm, pud, address);} static int mfill_establish_pmd(struct mfill_state *state){ struct mm_struct *dst_mm = state->ctx->mm; pmd_t *dst_pmd, dst_pmdval; dst_pmd = mm_alloc_pmd(dst_mm, state->dst_addr); if (unlikely(!dst_pmd)) return -ENOMEM; dst_pmdval = pmdp_get_lockless(dst_pmd); if (unlikely(pmd_none(dst_pmdval)) && unlikely(__pte_alloc(dst_mm, dst_pmd))) return -ENOMEM; dst_pmdval = pmdp_get_lockless(dst_pmd); /* * If the dst_pmd is THP don't override it and just be strict. * (This includes the case where the PMD used to be THP and * changed back to none after __pte_alloc().) */ if (unlikely(!pmd_present(dst_pmdval) || pmd_leaf(dst_pmdval))) return -EEXIST; if (unlikely(pmd_bad(dst_pmdval))) return -EFAULT; state->pmd = dst_pmd; return 0;} /* Check if dst_addr is outside of file's size. Must be called with ptl held. */static bool mfill_file_over_size(struct vm_area_struct *dst_vma, unsigned long dst_addr){ struct inode *inode; pgoff_t offset, max_off; if (!dst_vma->vm_file) return false; inode = dst_vma->vm_file->f_inode; offset = linear_page_index(dst_vma, dst_addr); max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); return offset >= max_off;} /* * Install PTEs, to map dst_addr (within dst_vma) to page. * * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem * and anon, and for both shared and private VMAs. */static int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, uffd_flags_t flags){ int ret; struct mm_struct *dst_mm = dst_vma->vm_mm; pte_t _dst_pte, *dst_pte; bool writable = dst_vma->vm_flags & VM_WRITE; bool vm_shared = dst_vma->vm_flags & VM_SHARED; spinlock_t *ptl; struct folio *folio = page_folio(page); bool page_in_cache = folio_mapping(folio); pte_t dst_ptep; _dst_pte = mk_pte(page, dst_vma->vm_page_prot); _dst_pte = pte_mkdirty(_dst_pte); if (page_in_cache && !vm_shared) writable = false; if (writable) _dst_pte = pte_mkwrite(_dst_pte, dst_vma); if (flags & MFILL_ATOMIC_WP) _dst_pte = pte_mkuffd(_dst_pte); ret = -EAGAIN; dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); if (!dst_pte) goto out; if (mfill_file_over_size(dst_vma, dst_addr)) { ret = -EFAULT; goto out_unlock; } ret = -EEXIST; dst_ptep = ptep_get(dst_pte); /* * We are allowed to overwrite a UFFD pte marker: consider when both * MISSING|WP registered, we firstly wr-protect a none pte which has no * page cache page backing it, then access the page. */ if (!pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep)) goto out_unlock; if (page_in_cache) { folio_add_file_rmap_pte(folio, page, dst_vma); } else { folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, dst_vma); } /* * Must happen after rmap, as mm_counter() checks mapping (via * PageAnon()), which is set by __page_set_anon_rmap(). */ inc_mm_counter(dst_mm, mm_counter(folio)); set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); if (page_in_cache) folio_unlock(folio); /* No need to invalidate - it was non-present before */ update_mmu_cache(dst_vma, dst_addr, dst_pte); ret = 0;out_unlock: pte_unmap_unlock(dst_pte, ptl);out: return ret;} static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr){ void *kaddr; int ret; kaddr = kmap_local_folio(folio, 0); /* * The read mmap_lock is held here. Despite the * mmap_lock being read recursive a deadlock is still * possible if a writer has taken a lock. For example: * * process A thread 1 takes read lock on own mmap_lock * process A thread 2 calls mmap, blocks taking write lock * process B thread 1 takes page fault, read lock on own mmap lock * process B thread 2 calls mmap, blocks taking write lock * process A thread 1 blocks taking read lock on process B * process B thread 1 blocks taking read lock on process A * * Disable page faults to prevent potential deadlock * and retry the copy outside the mmap_lock. */ pagefault_disable(); ret = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); pagefault_enable(); kunmap_local(kaddr); if (ret) return -EFAULT; flush_dcache_folio(folio); return ret;} #define MFILL_RETRY_STATE_VMA_FLAGS \ append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT) /* * VMA state saved before dropping the locks in mfill_copy_folio_retry(). * Used to detect VMA replacement or incompatible changes after reacquiring the * locks. */struct mfill_retry_state { const struct vm_uffd_ops *ops; struct file *file; vma_flags_t flags; pgoff_t pgoff;}; static void mfill_retry_state_save(struct mfill_retry_state *s, struct vm_area_struct *vma){ s->flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS); s->ops = vma_uffd_ops(vma); s->pgoff = vma_start_pgoff(vma); if (vma->vm_file) s->file = get_file(vma->vm_file);} static bool mfill_retry_state_changed(struct mfill_retry_state *state, struct vm_area_struct *vma){ vma_flags_t flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS); /* Have any UFFD flags (missing, WP, minor) changed? */ if (!vma_flags_same_pair(&state->flags, &flags)) return true; /* VMA type or effective uffd_ops changed while the lock was dropped */Lines 1–500 of 4888. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * mm/userfaultfd.c * * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org> * Copyright (C) 2008-2009 Red Hat, Inc. * Copyright (C) 2015 Red Hat, Inc. * * Some part derived from fs/eventfd.c (anon inode setup) and * mm/ksm.c (mm hashing). */ #include <linux/mm.h> #include <linux/sched/signal.h> #include <linux/pagemap.h> #include <linux/rmap.h> #include <linux/swap.h> #include <linux/leafops.h> #include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> #include <linux/list.h> #include <linux/sched/mm.h> #include <linux/mm_inline.h> #include <linux/poll.h> #include <linux/slab.h> #include <linux/seq_file.h> #include <linux/bug.h> #include <linux/anon_inodes.h> #include <linux/syscalls.h> #include <linux/miscdevice.h> #include <linux/uio.h> #include <linux/file.h> #include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h" #include "swap.h" struct mfill_state { struct userfaultfd_ctx *ctx; unsigned long src_start; unsigned long dst_start; unsigned long len; uffd_flags_t flags; struct vm_area_struct *vma; unsigned long src_addr; unsigned long dst_addr; pmd_t *pmd; }; static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) { /* anonymous memory does not support MINOR mode */ if (vm_flags & VM_UFFD_MINOR) return false; return true; } static struct folio *anon_alloc_folio(struct vm_area_struct *vma, unsigned long addr) { struct folio *folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr); if (!folio) return NULL; if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL)) { folio_put(folio); return NULL; } return folio; } static const struct vm_uffd_ops anon_uffd_ops = { .can_userfault = anon_can_userfault, .alloc_folio = anon_alloc_folio, }; static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) { if (vma_is_anonymous(vma)) return &anon_uffd_ops; return vma->vm_ops->uffd_ops; } static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) { /* Make sure that the dst range is fully within dst_vma. */ if (dst_end > dst_vma->vm_end) return false; /* * Check the vma is registered in uffd, this is required to * enforce the VM_MAYWRITE check done at uffd registration * time. */ if (!dst_vma->vm_userfaultfd_ctx.ctx) return false; return true; } static __always_inline struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, unsigned long addr) { struct vm_area_struct *vma; mmap_assert_locked(mm); vma = vma_lookup(mm, addr); if (!vma) vma = ERR_PTR(-ENOENT); else if (!(vma->vm_flags & VM_SHARED) && unlikely(anon_vma_prepare(vma))) vma = ERR_PTR(-ENOMEM); return vma; } #ifdef CONFIG_PER_VMA_LOCK /* * uffd_lock_vma() - Lookup and lock vma corresponding to @address. * @mm: mm to search vma in. * @address: address that the vma should contain. * * Should be called without holding mmap_lock. * * Return: A locked vma containing @address, -ENOENT if no vma is found, or * -ENOMEM if anon_vma couldn't be allocated. */ static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm, unsigned long address) { struct vm_area_struct *vma; vma = lock_vma_under_rcu(mm, address); if (vma) { /* * We know we're going to need to use anon_vma, so check * that early. */ if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma)) vma_end_read(vma); else return vma; } mmap_read_lock(mm); vma = find_vma_and_prepare_anon(mm, address); if (!IS_ERR(vma)) { bool locked = vma_start_read_locked(vma); if (!locked) vma = ERR_PTR(-EAGAIN); } mmap_read_unlock(mm); return vma; } static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long len) { struct vm_area_struct *dst_vma; dst_vma = uffd_lock_vma(dst_mm, dst_start); if (IS_ERR(dst_vma) || validate_dst_vma(dst_vma, dst_start + len)) return dst_vma; vma_end_read(dst_vma); return ERR_PTR(-ENOENT); } static void uffd_mfill_unlock(struct vm_area_struct *vma) { vma_end_read(vma); } #else static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long len) { struct vm_area_struct *dst_vma; mmap_read_lock(dst_mm); dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start); if (IS_ERR(dst_vma)) goto out_unlock; if (validate_dst_vma(dst_vma, dst_start + len)) return dst_vma; dst_vma = ERR_PTR(-ENOENT); out_unlock: mmap_read_unlock(dst_mm); return dst_vma; } static void uffd_mfill_unlock(struct vm_area_struct *vma) { mmap_read_unlock(vma->vm_mm); } #endif static void mfill_put_vma(struct mfill_state *state) { if (!state->vma) return; up_read(&state->ctx->map_changing_lock); uffd_mfill_unlock(state->vma); state->vma = NULL; } static int mfill_get_vma(struct mfill_state *state) { struct userfaultfd_ctx *ctx = state->ctx; uffd_flags_t flags = state->flags; struct vm_area_struct *dst_vma; const struct vm_uffd_ops *ops; int err; /* * Make sure the vma is not shared, that the dst range is * both valid and fully within a single existing vma. */ dst_vma = uffd_mfill_lock(ctx->mm, state->dst_start, state->len); if (IS_ERR(dst_vma)) return PTR_ERR(dst_vma); /* * If memory mappings are changing because of non-cooperative * operation (e.g. mremap) running in parallel, bail out and * request the user to retry later */ down_read(&ctx->map_changing_lock); state->vma = dst_vma; err = -EAGAIN; if (atomic_read(&ctx->mmap_changing)) goto out_unlock; err = -EINVAL; /* * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but * it will overwrite vm_ops, so vma_is_anonymous must return false. */ if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) && dst_vma->vm_flags & VM_SHARED)) goto out_unlock; /* * validate 'mode' now that we know the dst_vma: don't allow * a wrprotect copy if the userfaultfd didn't register as WP. */ if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) goto out_unlock; if (is_vm_hugetlb_page(dst_vma)) return 0; ops = vma_uffd_ops(dst_vma); if (!ops) goto out_unlock; if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE) && !ops->get_folio_noalloc) goto out_unlock; return 0; out_unlock: mfill_put_vma(state); return err; } static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) { pgd_t *pgd; p4d_t *p4d; pud_t *pud; pgd = pgd_offset(mm, address); p4d = p4d_alloc(mm, pgd, address); if (!p4d) return NULL; pud = pud_alloc(mm, p4d, address); if (!pud) return NULL; /* * Note that we didn't run this because the pmd was * missing, the *pmd may be already established and in * turn it may also be a trans_huge_pmd. */ return pmd_alloc(mm, pud, address); } static int mfill_establish_pmd(struct mfill_state *state) { struct mm_struct *dst_mm = state->ctx->mm; pmd_t *dst_pmd, dst_pmdval; dst_pmd = mm_alloc_pmd(dst_mm, state->dst_addr); if (unlikely(!dst_pmd)) return -ENOMEM; dst_pmdval = pmdp_get_lockless(dst_pmd); if (unlikely(pmd_none(dst_pmdval)) && unlikely(__pte_alloc(dst_mm, dst_pmd))) return -ENOMEM; dst_pmdval = pmdp_get_lockless(dst_pmd); /* * If the dst_pmd is THP don't override it and just be strict. * (This includes the case where the PMD used to be THP and * changed back to none after __pte_alloc().) */ if (unlikely(!pmd_present(dst_pmdval) || pmd_leaf(dst_pmdval))) return -EEXIST; if (unlikely(pmd_bad(dst_pmdval))) return -EFAULT; state->pmd = dst_pmd; return 0; } /* Check if dst_addr is outside of file's size. Must be called with ptl held. */ static bool mfill_file_over_size(struct vm_area_struct *dst_vma, unsigned long dst_addr) { struct inode *inode; pgoff_t offset, max_off; if (!dst_vma->vm_file) return false; inode = dst_vma->vm_file->f_inode; offset = linear_page_index(dst_vma, dst_addr); max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); return offset >= max_off; } /* * Install PTEs, to map dst_addr (within dst_vma) to page. * * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem * and anon, and for both shared and private VMAs. */ static int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, uffd_flags_t flags) { int ret; struct mm_struct *dst_mm = dst_vma->vm_mm; pte_t _dst_pte, *dst_pte; bool writable = dst_vma->vm_flags & VM_WRITE; bool vm_shared = dst_vma->vm_flags & VM_SHARED; spinlock_t *ptl; struct folio *folio = page_folio(page); bool page_in_cache = folio_mapping(folio); pte_t dst_ptep; _dst_pte = mk_pte(page, dst_vma->vm_page_prot); _dst_pte = pte_mkdirty(_dst_pte); if (page_in_cache && !vm_shared) writable = false; if (writable) _dst_pte = pte_mkwrite(_dst_pte, dst_vma); if (flags & MFILL_ATOMIC_WP) _dst_pte = pte_mkuffd(_dst_pte); ret = -EAGAIN; dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); if (!dst_pte) goto out; if (mfill_file_over_size(dst_vma, dst_addr)) { ret = -EFAULT; goto out_unlock; } ret = -EEXIST; dst_ptep = ptep_get(dst_pte); /* * We are allowed to overwrite a UFFD pte marker: consider when both * MISSING|WP registered, we firstly wr-protect a none pte which has no * page cache page backing it, then access the page. */ if (!pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep)) goto out_unlock; if (page_in_cache) { folio_add_file_rmap_pte(folio, page, dst_vma); } else { folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, dst_vma); } /* * Must happen after rmap, as mm_counter() checks mapping (via * PageAnon()), which is set by __page_set_anon_rmap(). */ inc_mm_counter(dst_mm, mm_counter(folio)); set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); if (page_in_cache) folio_unlock(folio); /* No need to invalidate - it was non-present before */ update_mmu_cache(dst_vma, dst_addr, dst_pte); ret = 0; out_unlock: pte_unmap_unlock(dst_pte, ptl); out: return ret; } static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) { void *kaddr; int ret; kaddr = kmap_local_folio(folio, 0); /* * The read mmap_lock is held here. Despite the * mmap_lock being read recursive a deadlock is still * possible if a writer has taken a lock. For example: * * process A thread 1 takes read lock on own mmap_lock * process A thread 2 calls mmap, blocks taking write lock * process B thread 1 takes page fault, read lock on own mmap lock * process B thread 2 calls mmap, blocks taking write lock * process A thread 1 blocks taking read lock on process B * process B thread 1 blocks taking read lock on process A * * Disable page faults to prevent potential deadlock * and retry the copy outside the mmap_lock. */ pagefault_disable(); ret = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); pagefault_enable(); kunmap_local(kaddr); if (ret) return -EFAULT; flush_dcache_folio(folio); return ret; } #define MFILL_RETRY_STATE_VMA_FLAGS \ append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT) /* * VMA state saved before dropping the locks in mfill_copy_folio_retry(). * Used to detect VMA replacement or incompatible changes after reacquiring the * locks. */ struct mfill_retry_state { const struct vm_uffd_ops *ops; struct file *file; vma_flags_t flags; pgoff_t pgoff; }; static void mfill_retry_state_save(struct mfill_retry_state *s, struct vm_area_struct *vma) { s->flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS); s->ops = vma_uffd_ops(vma); s->pgoff = vma_start_pgoff(vma); if (vma->vm_file) s->file = get_file(vma->vm_file); } static bool mfill_retry_state_changed(struct mfill_retry_state *state, struct vm_area_struct *vma) { vma_flags_t flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS); /* Have any UFFD flags (missing, WP, minor) changed? */ if (!vma_flags_same_pair(&state->flags, &flags)) return true; /* VMA type or effective uffd_ops changed while the lock was dropped */