History

Merge branch 'simplify-do_wp_page'

Merge emailed patches from Peter Xu: "This is a small series that I picked up from Linus's suggestion to simplify cow handling (and also make it more strict) by checking against page refcounts rather than mapcounts. This makes uffd-wp work again (verified by running upmapsort)" Note: this is horrendously bad timing, and making this kind of fundamental vm change after -rc3 is not at all how things should work. The saving grace is that it really is a a nice simplification: 8 files changed, 29 insertions(+), 120 deletions(-) The reason for the bad timing is that it turns out that commit 17839856fd58 ("gup: document and work around 'COW can break either way' issue" broke not just UFFD functionality (as Peter noticed), but Mikulas Patocka also reports that it caused issues for strace when running in a DAX environment with ext4 on a persistent memory setup. And we can't just revert that commit without re-introducing the original issue that is a potential security hole, so making COW stricter (and in the process much simpler) is a step to then undoing the forced COW that broke other uses. Link: https://lore.kernel.org/lkml/alpine.LRH.2.02.2009031328040.6929@file01.intranet.prod.int.rdu2.redhat.com/ * emailed patches from Peter Xu <peterx@redhat.com>: mm: Add PGREUSE counter mm/gup: Remove enfornced COW mechanism mm/ksm: Remove reuse_ksm_page() mm: do_wp_page() simplification

Linus Torvalds committed 6 years ago

Commit b25d1dc2 parents cfc905f 798a6b88 files · +29 −120Browse files

Unified · Split

8 files
  1. drivers/gpu/drm/i915/gem/i915_gem_userptr.c +0 −8
  2. include/linux/ksm.h +0 −7
  3. include/linux/vm_event_item.h +1 −0
  4. mm/gup.c +5 −35
  5. mm/huge_memory.c +4 −3
  6. mm/ksm.c +0 −25
  7. mm/memory.c +18 −42
  8. mm/vmstat.c +1 −0
drivers/gpu/drm/i915/gem/i915_gem_userptr.c+0−8
@@ -596,14 +596,6 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) · 595 unmodified lines
596 GFP_KERNEL |596 GFP_KERNEL |
597 __GFP_NORETRY |597 __GFP_NORETRY |
598 __GFP_NOWARN);598 __GFP_NOWARN);
599 /*
600 * Using __get_user_pages_fast() with a read-only
601 * access is questionable. A read-only page may be
602 * COW-broken, and then this might end up giving
603 * the wrong side of the COW..
604 *
605 * We may or may not care.
606 */
607 if (pvec) {599 if (pvec) {
608 /* defer to worker if malloc fails */600 /* defer to worker if malloc fails */
609 if (!i915_gem_object_is_readonly(obj))601 if (!i915_gem_object_is_readonly(obj))
include/linux/ksm.h+0−7
@@ -53,8 +53,6 @@ struct page *ksm_might_need_to_copy(struct page *page, · 52 unmodified lines
53 53
54void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);54void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);
55void ksm_migrate_page(struct page *newpage, struct page *oldpage);55void ksm_migrate_page(struct page *newpage, struct page *oldpage);
56bool reuse_ksm_page(struct page *page,
57 struct vm_area_struct *vma, unsigned long address);
58 56
59#else /* !CONFIG_KSM */57#else /* !CONFIG_KSM */
60 58
@@ -88,11 +86,6 @@ static inline void rmap_walk_ksm(struct page *page, · 27 unmodified lines
88static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)86static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)
89{87{
90}88}
91static inline bool reuse_ksm_page(struct page *page,
92 struct vm_area_struct *vma, unsigned long address)
93{
94 return false;
95}
96#endif /* CONFIG_MMU */89#endif /* CONFIG_MMU */
97#endif /* !CONFIG_KSM */90#endif /* !CONFIG_KSM */
98 91
include/linux/vm_event_item.h+1−0
@@ -30,6 +30,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, · 29 unmodified lines
30 PGFAULT, PGMAJFAULT,30 PGFAULT, PGMAJFAULT,
31 PGLAZYFREED,31 PGLAZYFREED,
32 PGREFILL,32 PGREFILL,
33 PGREUSE,
33 PGSTEAL_KSWAPD,34 PGSTEAL_KSWAPD,
34 PGSTEAL_DIRECT,35 PGSTEAL_DIRECT,
35 PGSCAN_KSWAPD,36 PGSCAN_KSWAPD,
mm/gup.c+5−35
@@ -381,22 +381,13 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address, · 380 unmodified lines
381}381}
382 382
383/*383/*
384 * FOLL_FORCE or a forced COW break can write even to unwritable pte's,384 * FOLL_FORCE can write to even unwritable pte's, but only
385 * but only after we've gone through a COW cycle and they are dirty.385 * after we've gone through a COW cycle and they are dirty.
386 */386 */
387static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)387static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
388{388{
389 return pte_write(pte) || ((flags & FOLL_COW) && pte_dirty(pte));389 return pte_write(pte) ||
390}390 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
391
392/*
393 * A (separate) COW fault might break the page the other way and
394 * get_user_pages() would return the page from what is now the wrong
395 * VM. So we need to force a COW break at GUP time even for reads.
396 */
397static inline bool should_force_cow_break(struct vm_area_struct *vma, unsigned int flags)
398{
399 return is_cow_mapping(vma->vm_flags) && (flags & (FOLL_GET | FOLL_PIN));
400}391}
401 392
402static struct page *follow_page_pte(struct vm_area_struct *vma,393static struct page *follow_page_pte(struct vm_area_struct *vma,
@@ -1067,11 +1058,9 @@ static long __get_user_pages(struct mm_struct *mm, · 664 unmodified lines
1067 goto out;1058 goto out;
1068 }1059 }
1069 if (is_vm_hugetlb_page(vma)) {1060 if (is_vm_hugetlb_page(vma)) {
1070 if (should_force_cow_break(vma, foll_flags))
1071 foll_flags |= FOLL_WRITE;
1072 i = follow_hugetlb_page(mm, vma, pages, vmas,1061 i = follow_hugetlb_page(mm, vma, pages, vmas,
1073 &start, &nr_pages, i,1062 &start, &nr_pages, i,
1074 foll_flags, locked);1063 gup_flags, locked);
1075 if (locked && *locked == 0) {1064 if (locked && *locked == 0) {
1076 /*1065 /*
1077 * We've got a VM_FAULT_RETRY1066 * We've got a VM_FAULT_RETRY
@@ -1085,10 +1074,6 @@ static long __get_user_pages(struct mm_struct *mm, · 7 unmodified lines
1085 continue;1074 continue;
1086 }1075 }
1087 }1076 }
1088
1089 if (should_force_cow_break(vma, foll_flags))
1090 foll_flags |= FOLL_WRITE;
1091
1092retry:1077retry:
1093 /*1078 /*
1094 * If we have a pending SIGKILL, don't keep faulting pages and1079 * If we have a pending SIGKILL, don't keep faulting pages and
@@ -2689,19 +2674,6 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, · 1594 unmodified lines
2689 return -EFAULT;2674 return -EFAULT;
2690 2675
2691 /*2676 /*
2692 * The FAST_GUP case requires FOLL_WRITE even for pure reads,
2693 * because get_user_pages() may need to cause an early COW in
2694 * order to avoid confusing the normal COW routines. So only
2695 * targets that are already writable are safe to do by just
2696 * looking at the page tables.
2697 *
2698 * NOTE! With FOLL_FAST_ONLY we allow read-only gup_fast() here,
2699 * because there is no slow path to fall back on. But you'd
2700 * better be careful about possible COW pages - you'll get _a_
2701 * COW page, but not necessarily the one you intended to get
2702 * depending on what COW event happens after this. COW may break
2703 * the page copy in a random direction.
2704 *
2705 * Disable interrupts. The nested form is used, in order to allow2677 * Disable interrupts. The nested form is used, in order to allow
2706 * full, general purpose use of this routine.2678 * full, general purpose use of this routine.
2707 *2679 *
@@ -2714,8 +2686,6 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, · 6 unmodified lines
2714 */2686 */
2715 if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) && gup_fast_permitted(start, end)) {2687 if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) && gup_fast_permitted(start, end)) {
2716 unsigned long fast_flags = gup_flags;2688 unsigned long fast_flags = gup_flags;
2717 if (!(gup_flags & FOLL_FAST_ONLY))
2718 fast_flags |= FOLL_WRITE;
2719 2689
2720 local_irq_save(flags);2690 local_irq_save(flags);
2721 gup_pgd_range(addr, end, fast_flags, pages, &nr_pinned);2691 gup_pgd_range(addr, end, fast_flags, pages, &nr_pinned);
mm/huge_memory.c+4−3
@@ -1291,12 +1291,13 @@ fallback: · 1290 unmodified lines
1291}1291}
1292 1292
1293/*1293/*
1294 * FOLL_FORCE or a forced COW break can write even to unwritable pmd's,1294 * FOLL_FORCE can write to even unwritable pmd's, but only
1295 * but only after we've gone through a COW cycle and they are dirty.1295 * after we've gone through a COW cycle and they are dirty.
1296 */1296 */
1297static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)1297static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)
1298{1298{
1299 return pmd_write(pmd) || ((flags & FOLL_COW) && pmd_dirty(pmd));1299 return pmd_write(pmd) ||
1300 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));
1300}1301}
1301 1302
1302struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,1303struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
mm/ksm.c+0−25
@@ -2661,31 +2661,6 @@ again: · 2660 unmodified lines
2661 goto again;2661 goto again;
2662}2662}
2663 2663
2664bool reuse_ksm_page(struct page *page,
2665 struct vm_area_struct *vma,
2666 unsigned long address)
2667{
2668#ifdef CONFIG_DEBUG_VM
2669 if (WARN_ON(is_zero_pfn(page_to_pfn(page))) ||
2670 WARN_ON(!page_mapped(page)) ||
2671 WARN_ON(!PageLocked(page))) {
2672 dump_page(page, "reuse_ksm_page");
2673 return false;
2674 }
2675#endif
2676
2677 if (PageSwapCache(page) || !page_stable_node(page))
2678 return false;
2679 /* Prohibit parallel get_ksm_page() */
2680 if (!page_ref_freeze(page, 1))
2681 return false;
2682
2683 page_move_anon_rmap(page, vma);
2684 page->index = linear_page_index(vma, address);
2685 page_ref_unfreeze(page, 1);
2686
2687 return true;
2688}
2689#ifdef CONFIG_MIGRATION2664#ifdef CONFIG_MIGRATION
2690void ksm_migrate_page(struct page *newpage, struct page *oldpage)2665void ksm_migrate_page(struct page *newpage, struct page *oldpage)
2691{2666{
mm/memory.c+18−42
@@ -2622,6 +2622,7 @@ static inline void wp_page_reuse(struct vm_fault *vmf) · 2621 unmodified lines
2622 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))2622 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
2623 update_mmu_cache(vma, vmf->address, vmf->pte);2623 update_mmu_cache(vma, vmf->address, vmf->pte);
2624 pte_unmap_unlock(vmf->pte, vmf->ptl);2624 pte_unmap_unlock(vmf->pte, vmf->ptl);
2625 count_vm_event(PGREUSE);
2625}2626}
2626 2627
2627/*2628/*
@@ -2927,50 +2928,25 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) · 299 unmodified lines
2927 * not dirty accountable.2928 * not dirty accountable.
2928 */2929 */
2929 if (PageAnon(vmf->page)) {2930 if (PageAnon(vmf->page)) {
2930 int total_map_swapcount;2931 struct page *page = vmf->page;
2931 if (PageKsm(vmf->page) && (PageSwapCache(vmf->page) ||2932
2932 page_count(vmf->page) != 1))2933 /* PageKsm() doesn't necessarily raise the page refcount */
2934 if (PageKsm(page) || page_count(page) != 1)
2935 goto copy;
2936 if (!trylock_page(page))
2937 goto copy;
2938 if (PageKsm(page) || page_mapcount(page) != 1 || page_count(page) != 1) {
2939 unlock_page(page);
2933 goto copy;2940 goto copy;
2934 if (!trylock_page(vmf->page)) {
2935 get_page(vmf->page);
2936 pte_unmap_unlock(vmf->pte, vmf->ptl);
2937 lock_page(vmf->page);
2938 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
2939 vmf->address, &vmf->ptl);
2940 if (!pte_same(*vmf->pte, vmf->orig_pte)) {
2941 update_mmu_tlb(vma, vmf->address, vmf->pte);
2942 unlock_page(vmf->page);
2943 pte_unmap_unlock(vmf->pte, vmf->ptl);
2944 put_page(vmf->page);
2945 return 0;
2946 }
2947 put_page(vmf->page);
2948 }
2949 if (PageKsm(vmf->page)) {
2950 bool reused = reuse_ksm_page(vmf->page, vmf->vma,
2951 vmf->address);
2952 unlock_page(vmf->page);
2953 if (!reused)
2954 goto copy;
2955 wp_page_reuse(vmf);
2956 return VM_FAULT_WRITE;
2957 }
2958 if (reuse_swap_page(vmf->page, &total_map_swapcount)) {
2959 if (total_map_swapcount == 1) {
2960 /*
2961 * The page is all ours. Move it to
2962 * our anon_vma so the rmap code will
2963 * not search our parent or siblings.
2964 * Protected against the rmap code by
2965 * the page lock.
2966 */
2967 page_move_anon_rmap(vmf->page, vma);
2968 }
2969 unlock_page(vmf->page);
2970 wp_page_reuse(vmf);
2971 return VM_FAULT_WRITE;
2972 }2941 }
2973 unlock_page(vmf->page);2942 /*
2943 * Ok, we've got the only map reference, and the only
2944 * page count reference, and the page is locked,
2945 * it's dark out, and we're wearing sunglasses. Hit it.
2946 */
2947 wp_page_reuse(vmf);
2948 unlock_page(page);
2949 return VM_FAULT_WRITE;
2974 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==2950 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
2975 (VM_WRITE|VM_SHARED))) {2951 (VM_WRITE|VM_SHARED))) {
2976 return wp_page_shared(vmf);2952 return wp_page_shared(vmf);
mm/vmstat.c+1−0
@@ -1241,6 +1241,7 @@ const char * const vmstat_text[] = { · 1240 unmodified lines
1241 "pglazyfreed",1241 "pglazyfreed",
1242 1242
1243 "pgrefill",1243 "pgrefill",
1244 "pgreuse",
1244 "pgsteal_kswapd",1245 "pgsteal_kswapd",
1245 "pgsteal_direct",1246 "pgsteal_direct",
1246 "pgscan_kswapd",1247 "pgscan_kswapd",