History

Merge branch 'simplify-do_wp_page'

Merge emailed patches from Peter Xu: "This is a small series that I picked up from Linus's suggestion to simplify cow handling (and also make it more strict) by checking against page refcounts rather than mapcounts. This makes uffd-wp work again (verified by running upmapsort)" Note: this is horrendously bad timing, and making this kind of fundamental vm change after -rc3 is not at all how things should work. The saving grace is that it really is a a nice simplification: 8 files changed, 29 insertions(+), 120 deletions(-) The reason for the bad timing is that it turns out that commit 17839856fd58 ("gup: document and work around 'COW can break either way' issue" broke not just UFFD functionality (as Peter noticed), but Mikulas Patocka also reports that it caused issues for strace when running in a DAX environment with ext4 on a persistent memory setup. And we can't just revert that commit without re-introducing the original issue that is a potential security hole, so making COW stricter (and in the process much simpler) is a step to then undoing the forced COW that broke other uses. Link: https://lore.kernel.org/lkml/alpine.LRH.2.02.2009031328040.6929@file01.intranet.prod.int.rdu2.redhat.com/ * emailed patches from Peter Xu <peterx@redhat.com>: mm: Add PGREUSE counter mm/gup: Remove enfornced COW mechanism mm/ksm: Remove reuse_ksm_page() mm: do_wp_page() simplification

Linus Torvalds committed 6 years ago

Commit b25d1dc2 parents cfc905f 798a6b88 files · +29 −120Browse files

Unified · Split

8 files
  1. drivers/gpu/drm/i915/gem/i915_gem_userptr.c +0 −8
  2. include/linux/ksm.h +0 −7
  3. include/linux/vm_event_item.h +1 −0
  4. mm/gup.c +5 −35
  5. mm/huge_memory.c +4 −3
  6. mm/ksm.c +0 −25
  7. mm/memory.c +18 −42
  8. mm/vmstat.c +1 −0
drivers/gpu/drm/i915/gem/i915_gem_userptr.c+0−8
@@ -596,14 +596,6 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) · 595 unmodified lines
596596 GFP_KERNEL |
597597 __GFP_NORETRY |
598598 __GFP_NOWARN);
599− /*
600− * Using __get_user_pages_fast() with a read-only
601− * access is questionable. A read-only page may be
602− * COW-broken, and then this might end up giving
603− * the wrong side of the COW..
604− *
605− * We may or may not care.
606− */
607599 if (pvec) {
608600 /* defer to worker if malloc fails */
609601 if (!i915_gem_object_is_readonly(obj))
include/linux/ksm.h+0−7
@@ -53,8 +53,6 @@ struct page *ksm_might_need_to_copy(struct page *page, · 52 unmodified lines
5353
5454 void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);
5555 void ksm_migrate_page(struct page *newpage, struct page *oldpage);
56−bool reuse_ksm_page(struct page *page,
57− struct vm_area_struct *vma, unsigned long address);
5856
5957 #else /* !CONFIG_KSM */
6058
@@ -88,11 +86,6 @@ static inline void rmap_walk_ksm(struct page *page, · 27 unmodified lines
8886 static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)
8987 {
9088 }
91−static inline bool reuse_ksm_page(struct page *page,
92− struct vm_area_struct *vma, unsigned long address)
93−{
94− return false;
95−}
9689 #endif /* CONFIG_MMU */
9790 #endif /* !CONFIG_KSM */
9891
include/linux/vm_event_item.h+1−0
@@ -30,6 +30,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, · 29 unmodified lines
3030 PGFAULT, PGMAJFAULT,
3131 PGLAZYFREED,
3232 PGREFILL,
33+ PGREUSE,
3334 PGSTEAL_KSWAPD,
3435 PGSTEAL_DIRECT,
3536 PGSCAN_KSWAPD,
mm/gup.c+5−35
@@ -381,22 +381,13 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address, · 380 unmodified lines
381381 }
382382
383383 /*
384− * FOLL_FORCE or a forced COW break can write even to unwritable pte's,
385− * but only after we've gone through a COW cycle and they are dirty.
384+ * FOLL_FORCE can write to even unwritable pte's, but only
385+ * after we've gone through a COW cycle and they are dirty.
386386 */
387387 static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
388388 {
389− return pte_write(pte) || ((flags & FOLL_COW) && pte_dirty(pte));
390−}
391−
392−/*
393− * A (separate) COW fault might break the page the other way and
394− * get_user_pages() would return the page from what is now the wrong
395− * VM. So we need to force a COW break at GUP time even for reads.
396− */
397−static inline bool should_force_cow_break(struct vm_area_struct *vma, unsigned int flags)
398−{
399− return is_cow_mapping(vma->vm_flags) && (flags & (FOLL_GET | FOLL_PIN));
389+ return pte_write(pte) ||
390+ ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
400391 }
401392
402393 static struct page *follow_page_pte(struct vm_area_struct *vma,
@@ -1067,11 +1058,9 @@ static long __get_user_pages(struct mm_struct *mm, · 664 unmodified lines
10671058 goto out;
10681059 }
10691060 if (is_vm_hugetlb_page(vma)) {
1070− if (should_force_cow_break(vma, foll_flags))
1071− foll_flags |= FOLL_WRITE;
10721061 i = follow_hugetlb_page(mm, vma, pages, vmas,
10731062 &start, &nr_pages, i,
1074− foll_flags, locked);
1063+ gup_flags, locked);
10751064 if (locked && *locked == 0) {
10761065 /*
10771066 * We've got a VM_FAULT_RETRY
@@ -1085,10 +1074,6 @@ static long __get_user_pages(struct mm_struct *mm, · 7 unmodified lines
10851074 continue;
10861075 }
10871076 }
1088−
1089− if (should_force_cow_break(vma, foll_flags))
1090− foll_flags |= FOLL_WRITE;
1091−
10921077 retry:
10931078 /*
10941079 * If we have a pending SIGKILL, don't keep faulting pages and
@@ -2689,19 +2674,6 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, · 1594 unmodified lines
26892674 return -EFAULT;
26902675
26912676 /*
2692− * The FAST_GUP case requires FOLL_WRITE even for pure reads,
2693− * because get_user_pages() may need to cause an early COW in
2694− * order to avoid confusing the normal COW routines. So only
2695− * targets that are already writable are safe to do by just
2696− * looking at the page tables.
2697− *
2698− * NOTE! With FOLL_FAST_ONLY we allow read-only gup_fast() here,
2699− * because there is no slow path to fall back on. But you'd
2700− * better be careful about possible COW pages - you'll get _a_
2701− * COW page, but not necessarily the one you intended to get
2702− * depending on what COW event happens after this. COW may break
2703− * the page copy in a random direction.
2704− *
27052677 * Disable interrupts. The nested form is used, in order to allow
27062678 * full, general purpose use of this routine.
27072679 *
@@ -2714,8 +2686,6 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, · 6 unmodified lines
27142686 */
27152687 if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) && gup_fast_permitted(start, end)) {
27162688 unsigned long fast_flags = gup_flags;
2717− if (!(gup_flags & FOLL_FAST_ONLY))
2718− fast_flags |= FOLL_WRITE;
27192689
27202690 local_irq_save(flags);
27212691 gup_pgd_range(addr, end, fast_flags, pages, &nr_pinned);
mm/huge_memory.c+4−3
@@ -1291,12 +1291,13 @@ fallback: · 1290 unmodified lines
12911291 }
12921292
12931293 /*
1294− * FOLL_FORCE or a forced COW break can write even to unwritable pmd's,
1295− * but only after we've gone through a COW cycle and they are dirty.
1294+ * FOLL_FORCE can write to even unwritable pmd's, but only
1295+ * after we've gone through a COW cycle and they are dirty.
12961296 */
12971297 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)
12981298 {
1299− return pmd_write(pmd) || ((flags & FOLL_COW) && pmd_dirty(pmd));
1299+ return pmd_write(pmd) ||
1300+ ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));
13001301 }
13011302
13021303 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
mm/ksm.c+0−25
@@ -2661,31 +2661,6 @@ again: · 2660 unmodified lines
26612661 goto again;
26622662 }
26632663
2664−bool reuse_ksm_page(struct page *page,
2665− struct vm_area_struct *vma,
2666− unsigned long address)
2667−{
2668−#ifdef CONFIG_DEBUG_VM
2669− if (WARN_ON(is_zero_pfn(page_to_pfn(page))) ||
2670− WARN_ON(!page_mapped(page)) ||
2671− WARN_ON(!PageLocked(page))) {
2672− dump_page(page, "reuse_ksm_page");
2673− return false;
2674− }
2675−#endif
2676−
2677− if (PageSwapCache(page) || !page_stable_node(page))
2678− return false;
2679− /* Prohibit parallel get_ksm_page() */
2680− if (!page_ref_freeze(page, 1))
2681− return false;
2682−
2683− page_move_anon_rmap(page, vma);
2684− page->index = linear_page_index(vma, address);
2685− page_ref_unfreeze(page, 1);
2686−
2687− return true;
2688−}
26892664 #ifdef CONFIG_MIGRATION
26902665 void ksm_migrate_page(struct page *newpage, struct page *oldpage)
26912666 {
mm/memory.c+18−42
@@ -2622,6 +2622,7 @@ static inline void wp_page_reuse(struct vm_fault *vmf) · 2621 unmodified lines
26222622 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
26232623 update_mmu_cache(vma, vmf->address, vmf->pte);
26242624 pte_unmap_unlock(vmf->pte, vmf->ptl);
2625+ count_vm_event(PGREUSE);
26252626 }
26262627
26272628 /*
@@ -2927,50 +2928,25 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) · 299 unmodified lines
29272928 * not dirty accountable.
29282929 */
29292930 if (PageAnon(vmf->page)) {
2930− int total_map_swapcount;
2931− if (PageKsm(vmf->page) && (PageSwapCache(vmf->page) ||
2932− page_count(vmf->page) != 1))
2931+ struct page *page = vmf->page;
2932+
2933+ /* PageKsm() doesn't necessarily raise the page refcount */
2934+ if (PageKsm(page) || page_count(page) != 1)
2935+ goto copy;
2936+ if (!trylock_page(page))
2937+ goto copy;
2938+ if (PageKsm(page) || page_mapcount(page) != 1 || page_count(page) != 1) {
2939+ unlock_page(page);
29332940 goto copy;
2934− if (!trylock_page(vmf->page)) {
2935− get_page(vmf->page);
2936− pte_unmap_unlock(vmf->pte, vmf->ptl);
2937− lock_page(vmf->page);
2938− vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
2939− vmf->address, &vmf->ptl);
2940− if (!pte_same(*vmf->pte, vmf->orig_pte)) {
2941− update_mmu_tlb(vma, vmf->address, vmf->pte);
2942− unlock_page(vmf->page);
2943− pte_unmap_unlock(vmf->pte, vmf->ptl);
2944− put_page(vmf->page);
2945− return 0;
2946− }
2947− put_page(vmf->page);
2948− }
2949− if (PageKsm(vmf->page)) {
2950− bool reused = reuse_ksm_page(vmf->page, vmf->vma,
2951− vmf->address);
2952− unlock_page(vmf->page);
2953− if (!reused)
2954− goto copy;
2955− wp_page_reuse(vmf);
2956− return VM_FAULT_WRITE;
2957− }
2958− if (reuse_swap_page(vmf->page, &total_map_swapcount)) {
2959− if (total_map_swapcount == 1) {
2960− /*
2961− * The page is all ours. Move it to
2962− * our anon_vma so the rmap code will
2963− * not search our parent or siblings.
2964− * Protected against the rmap code by
2965− * the page lock.
2966− */
2967− page_move_anon_rmap(vmf->page, vma);
2968− }
2969− unlock_page(vmf->page);
2970− wp_page_reuse(vmf);
2971− return VM_FAULT_WRITE;
29722941 }
2973− unlock_page(vmf->page);
2942+ /*
2943+ * Ok, we've got the only map reference, and the only
2944+ * page count reference, and the page is locked,
2945+ * it's dark out, and we're wearing sunglasses. Hit it.
2946+ */
2947+ wp_page_reuse(vmf);
2948+ unlock_page(page);
2949+ return VM_FAULT_WRITE;
29742950 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
29752951 (VM_WRITE|VM_SHARED))) {
29762952 return wp_page_shared(vmf);
mm/vmstat.c+1−0
@@ -1241,6 +1241,7 @@ const char * const vmstat_text[] = { · 1240 unmodified lines
12411241 "pglazyfreed",
12421242
12431243 "pgrefill",
1244+ "pgreuse",
12441245 "pgsteal_kswapd",
12451246 "pgsteal_direct",
12461247 "pgscan_kswapd",