History

mm/hugetlb: only drop uffd-wp special pte if required

As with shmem uffd-wp special ptes, only drop the uffd-wp special swap pte if unmapping an entire vma or synchronized such that faults can not race with the unmap operation. This requires passing zap_flags all the way to the lowest level hugetlb unmap routine: __unmap_hugepage_range. In general, unmap calls originated in hugetlbfs code will pass the ZAP_FLAG_DROP_MARKER flag as synchronization is in place to prevent faults. The exception is hole punch which will first unmap without any synchronization. Later when hole punch actually removes the page from the file, it will check to see if there was a subsequent fault and if so take the hugetlb fault mutex while unmapping again. This second unmap will pass in ZAP_FLAG_DROP_MARKER. The justification of "whether to apply ZAP_FLAG_DROP_MARKER flag when unmap a hugetlb range" is (IMHO): we should never reach a state when a page fault could errornously fault in a page-cache page that was wr-protected to be writable, even in an extremely short period. That could happen if e.g. we pass ZAP_FLAG_DROP_MARKER when hugetlbfs_punch_hole() calls hugetlb_vmdelete_list(), because if a page faults after that call and before remove_inode_hugepages() is executed, the page cache can be mapped writable again in the small racy window, that can cause unexpected data overwritten. [peterx@redhat.com: fix sparse warning] Link: https://lkml.kernel.org/r/Ylcdw8I1L5iAoWhb@xz-m1.local [akpm@linux-foundation.org: move zap_flags_t from mm.h to mm_types.h to fix build issues] Link: https://lkml.kernel.org/r/20220405014915.14873-1-peterx@redhat.com Signed-off-by: Peter Xu <peterx@redhat.com> Reviewed-by: Mike Kravetz <mike.kravetz@oracle.com> Cc: Alistair Popple <apopple@nvidia.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Axel Rasmussen <axelrasmussen@google.com> Cc: David Hildenbrand <david@redhat.com> Cc: Hugh Dickins <hughd@google.com> Cc: Jerome Glisse <jglisse@redhat.com> Cc: "Kirill A . Shutemov" <kirill@shutemov.name> Cc: Matthew Wilcox <willy@infradead.org> Cc: Mike Rapoport <rppt@linux.vnet.ibm.com> Cc: Nadav Amit <nadav.amit@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Peter Xu committed 4 years ago

Commit 05e90bd1 parent 60dfaad6 files · +45 −20Browse files

Unified · Split

6 files
  1. fs/hugetlbfs/inode.c +9 −6
  2. include/linux/hugetlb.h +5 −3
  3. include/linux/mm.h +0 −2
  4. include/linux/mm_types.h +2 −0
  5. mm/hugetlb.c +25 −8
  6. mm/memory.c +4 −1
fs/hugetlbfs/inode.c+9−6
@@ -405,7 +405,8 @@ static void remove_huge_page(struct page *page) · 404 unmodified lines
405}405}
406 406
407static void407static void
408hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end)408hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
409 zap_flags_t zap_flags)
409{410{
410 struct vm_area_struct *vma;411 struct vm_area_struct *vma;
411 412
@@ -439,7 +440,7 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end) · 27 unmodified lines
439 }440 }
440 441
441 unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end,442 unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end,
442 NULL);443 NULL, zap_flags);
443 }444 }
444}445}
445 446
@@ -517,7 +518,8 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart, · 71 unmodified lines
517 mutex_lock(&hugetlb_fault_mutex_table[hash]);518 mutex_lock(&hugetlb_fault_mutex_table[hash]);
518 hugetlb_vmdelete_list(&mapping->i_mmap,519 hugetlb_vmdelete_list(&mapping->i_mmap,
519 index * pages_per_huge_page(h),520 index * pages_per_huge_page(h),
520 (index + 1) * pages_per_huge_page(h));521 (index + 1) * pages_per_huge_page(h),
522 ZAP_FLAG_DROP_MARKER);
521 i_mmap_unlock_write(mapping);523 i_mmap_unlock_write(mapping);
522 }524 }
523 525
@@ -583,7 +585,8 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) · 59 unmodified lines
583 i_mmap_lock_write(mapping);585 i_mmap_lock_write(mapping);
584 i_size_write(inode, offset);586 i_size_write(inode, offset);
585 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))587 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
586 hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0);588 hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0,
589 ZAP_FLAG_DROP_MARKER);
587 i_mmap_unlock_write(mapping);590 i_mmap_unlock_write(mapping);
588 remove_inode_hugepages(inode, offset, LLONG_MAX);591 remove_inode_hugepages(inode, offset, LLONG_MAX);
589}592}
@@ -616,8 +619,8 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) · 26 unmodified lines
616 i_mmap_lock_write(mapping);619 i_mmap_lock_write(mapping);
617 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))620 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
618 hugetlb_vmdelete_list(&mapping->i_mmap,621 hugetlb_vmdelete_list(&mapping->i_mmap,
619 hole_start >> PAGE_SHIFT,622 hole_start >> PAGE_SHIFT,
620 hole_end >> PAGE_SHIFT);623 hole_end >> PAGE_SHIFT, 0);
621 i_mmap_unlock_write(mapping);624 i_mmap_unlock_write(mapping);
622 remove_inode_hugepages(inode, hole_start, hole_end);625 remove_inode_hugepages(inode, hole_start, hole_end);
623 inode_unlock(inode);626 inode_unlock(inode);
include/linux/hugetlb.h+5−3
@@ -143,11 +143,12 @@ long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, · 142 unmodified lines
143 unsigned long *, unsigned long *, long, unsigned int,143 unsigned long *, unsigned long *, long, unsigned int,
144 int *);144 int *);
145void unmap_hugepage_range(struct vm_area_struct *,145void unmap_hugepage_range(struct vm_area_struct *,
146 unsigned long, unsigned long, struct page *);146 unsigned long, unsigned long, struct page *,
147 zap_flags_t);
147void __unmap_hugepage_range_final(struct mmu_gather *tlb,148void __unmap_hugepage_range_final(struct mmu_gather *tlb,
148 struct vm_area_struct *vma,149 struct vm_area_struct *vma,
149 unsigned long start, unsigned long end,150 unsigned long start, unsigned long end,
150 struct page *ref_page);151 struct page *ref_page, zap_flags_t zap_flags);
151void hugetlb_report_meminfo(struct seq_file *);152void hugetlb_report_meminfo(struct seq_file *);
152int hugetlb_report_node_meminfo(char *buf, int len, int nid);153int hugetlb_report_node_meminfo(char *buf, int len, int nid);
153void hugetlb_show_meminfo(void);154void hugetlb_show_meminfo(void);
@@ -406,7 +407,8 @@ static inline unsigned long hugetlb_change_protection( · 252 unmodified lines
406 407
407static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,408static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
408 struct vm_area_struct *vma, unsigned long start,409 struct vm_area_struct *vma, unsigned long start,
409 unsigned long end, struct page *ref_page)410 unsigned long end, struct page *ref_page,
411 zap_flags_t zap_flags)
410{412{
411 BUG();413 BUG();
412}414}
include/linux/mm.h+0−2
@@ -3428,8 +3428,6 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start, · 3427 unmodified lines
3428}3428}
3429#endif3429#endif
3430 3430
3431typedef unsigned int __bitwise zap_flags_t;
3432
3433/*3431/*
3434 * Whether to drop the pte markers, for example, the uffd-wp information for3432 * Whether to drop the pte markers, for example, the uffd-wp information for
3435 * file-backed memory. This should only be specified when we will completely3433 * file-backed memory. This should only be specified when we will completely
include/linux/mm_types.h+2−0
@@ -863,4 +863,6 @@ enum fault_flag { · 862 unmodified lines
863 FAULT_FLAG_ORIG_PTE_VALID = 1 << 11,863 FAULT_FLAG_ORIG_PTE_VALID = 1 << 11,
864};864};
865 865
866typedef unsigned int __bitwise zap_flags_t;
867
866#endif /* _LINUX_MM_TYPES_H */868#endif /* _LINUX_MM_TYPES_H */
mm/hugetlb.c+25−8
@@ -4973,7 +4973,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, · 4972 unmodified lines
4973 4973
4974static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,4974static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
4975 unsigned long start, unsigned long end,4975 unsigned long start, unsigned long end,
4976 struct page *ref_page)4976 struct page *ref_page, zap_flags_t zap_flags)
4977{4977{
4978 struct mm_struct *mm = vma->vm_mm;4978 struct mm_struct *mm = vma->vm_mm;
4979 unsigned long address;4979 unsigned long address;
@@ -5029,7 +5029,18 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct · 49 unmodified lines
5029 * unmapped and its refcount is dropped, so just clear pte here.5029 * unmapped and its refcount is dropped, so just clear pte here.
5030 */5030 */
5031 if (unlikely(!pte_present(pte))) {5031 if (unlikely(!pte_present(pte))) {
5032 huge_pte_clear(mm, address, ptep, sz);5032 /*
5033 * If the pte was wr-protected by uffd-wp in any of the
5034 * swap forms, meanwhile the caller does not want to
5035 * drop the uffd-wp bit in this zap, then replace the
5036 * pte with a marker.
5037 */
5038 if (pte_swp_uffd_wp_any(pte) &&
5039 !(zap_flags & ZAP_FLAG_DROP_MARKER))
5040 set_huge_pte_at(mm, address, ptep,
5041 make_pte_marker(PTE_MARKER_UFFD_WP));
5042 else
5043 huge_pte_clear(mm, address, ptep, sz);
5033 spin_unlock(ptl);5044 spin_unlock(ptl);
5034 continue;5045 continue;
5035 }5046 }
@@ -5057,7 +5068,11 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct · 21 unmodified lines
5057 tlb_remove_huge_tlb_entry(h, tlb, ptep, address);5068 tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
5058 if (huge_pte_dirty(pte))5069 if (huge_pte_dirty(pte))
5059 set_page_dirty(page);5070 set_page_dirty(page);
5060 5071 /* Leave a uffd-wp pte marker if needed */
5072 if (huge_pte_uffd_wp(pte) &&
5073 !(zap_flags & ZAP_FLAG_DROP_MARKER))
5074 set_huge_pte_at(mm, address, ptep,
5075 make_pte_marker(PTE_MARKER_UFFD_WP));
5061 hugetlb_count_sub(pages_per_huge_page(h), mm);5076 hugetlb_count_sub(pages_per_huge_page(h), mm);
5062 page_remove_rmap(page, vma, true);5077 page_remove_rmap(page, vma, true);
5063 5078
@@ -5091,9 +5106,10 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct · 27 unmodified lines
5091 5106
5092void __unmap_hugepage_range_final(struct mmu_gather *tlb,5107void __unmap_hugepage_range_final(struct mmu_gather *tlb,
5093 struct vm_area_struct *vma, unsigned long start,5108 struct vm_area_struct *vma, unsigned long start,
5094 unsigned long end, struct page *ref_page)5109 unsigned long end, struct page *ref_page,
5110 zap_flags_t zap_flags)
5095{5111{
5096 __unmap_hugepage_range(tlb, vma, start, end, ref_page);5112 __unmap_hugepage_range(tlb, vma, start, end, ref_page, zap_flags);
5097 5113
5098 /*5114 /*
5099 * Clear this flag so that x86's huge_pmd_share page_table_shareable5115 * Clear this flag so that x86's huge_pmd_share page_table_shareable
@@ -5109,12 +5125,13 @@ void __unmap_hugepage_range_final(struct mmu_gather *tlb, · 9 unmodified lines
5109}5125}
5110 5126
5111void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,5127void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
5112 unsigned long end, struct page *ref_page)5128 unsigned long end, struct page *ref_page,
5129 zap_flags_t zap_flags)
5113{5130{
5114 struct mmu_gather tlb;5131 struct mmu_gather tlb;
5115 5132
5116 tlb_gather_mmu(&tlb, vma->vm_mm);5133 tlb_gather_mmu(&tlb, vma->vm_mm);
5117 __unmap_hugepage_range(&tlb, vma, start, end, ref_page);5134 __unmap_hugepage_range(&tlb, vma, start, end, ref_page, zap_flags);
5118 tlb_finish_mmu(&tlb);5135 tlb_finish_mmu(&tlb);
5119}5136}
5120 5137
@@ -5169,7 +5186,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, · 48 unmodified lines
5169 */5186 */
5170 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))5187 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
5171 unmap_hugepage_range(iter_vma, address,5188 unmap_hugepage_range(iter_vma, address,
5172 address + huge_page_size(h), page);5189 address + huge_page_size(h), page, 0);
5173 }5190 }
5174 i_mmap_unlock_write(mapping);5191 i_mmap_unlock_write(mapping);
5175}5192}
mm/memory.c+4−1
@@ -1675,8 +1675,11 @@ static void unmap_single_vma(struct mmu_gather *tlb, · 1674 unmodified lines
1675 * safe to do nothing in this case.1675 * safe to do nothing in this case.
1676 */1676 */
1677 if (vma->vm_file) {1677 if (vma->vm_file) {
1678 zap_flags_t zap_flags = details ?
1679 details->zap_flags : 0;
1678 i_mmap_lock_write(vma->vm_file->f_mapping);1680 i_mmap_lock_write(vma->vm_file->f_mapping);
1679 __unmap_hugepage_range_final(tlb, vma, start, end, NULL);1681 __unmap_hugepage_range_final(tlb, vma, start, end,
1682 NULL, zap_flags);
1680 i_mmap_unlock_write(vma->vm_file->f_mapping);1683 i_mmap_unlock_write(vma->vm_file->f_mapping);
1681 }1684 }
1682 } else1685 } else