History

mm: memcontrol: convert anon and file-thp to new mem_cgroup_charge() API

With the page->mapping requirement gone from memcg, we can charge anon and file-thp pages in one single step, right after they're allocated. This removes two out of three API calls - especially the tricky commit step that needed to happen at just the right time between when the page is "set up" and when it's "published" - somewhat vague and fluid concepts that varied by page type. All we need is a freshly allocated page and a memcg context to charge. v2: prevent double charges on pre-allocated hugepages in khugepaged [hannes@cmpxchg.org: Fix crash - *hpage could be ERR_PTR instead of NULL] Link: http://lkml.kernel.org/r/20200512215813.GA487759@cmpxchg.org Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Reviewed-by: Joonsoo Kim <iamjoonsoo.kim@lge.com> Cc: Alex Shi <alex.shi@linux.alibaba.com> Cc: Hugh Dickins <hughd@google.com> Cc: "Kirill A. Shutemov" <kirill@shutemov.name> Cc: Michal Hocko <mhocko@suse.com> Cc: Roman Gushchin <guro@fb.com> Cc: Shakeel Butt <shakeelb@google.com> Cc: Balbir Singh <bsingharora@gmail.com> Cc: Qian Cai <cai@lca.pw> Link: http://lkml.kernel.org/r/20200508183105.225460-13-hannes@cmpxchg.org Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Johannes Weiner committed 6 years ago

Commit 9d82c691 parent 468c3989 files · +31 −82Browse files

Unified · Split

9 files
  1. include/linux/mm.h +1 −3
  2. kernel/events/uprobes.c +3 −8
  3. mm/filemap.c +1 −1
  4. mm/huge_memory.c +3 −6
  5. mm/khugepaged.c +10 −25
  6. mm/memory.c +10 −26
  7. mm/migrate.c +1 −4
  8. mm/swapfile.c +1 −5
  9. mm/userfaultfd.c +1 −4
include/linux/mm.h+1−3
@@ -501,7 +501,6 @@ struct vm_fault { · 500 unmodified lines
501501 pte_t orig_pte; /* Value of PTE at the time of fault */
502502
503503 struct page *cow_page; /* Page handler may use for COW fault */
504− struct mem_cgroup *memcg; /* Cgroup cow_page belongs to */
505504 struct page *page; /* ->fault handlers should return a
506505 * page here, unless VM_FAULT_NOPAGE
507506 * is set (which is also implied by
@@ -946,8 +945,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) · 438 unmodified lines
946945 return pte;
947946 }
948947
949−vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
950− struct page *page);
948+vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page);
951949 vm_fault_t finish_fault(struct vm_fault *vmf);
952950 vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf);
953951 #endif
kernel/events/uprobes.c+3−8
@@ -162,14 +162,13 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 161 unmodified lines
162162 };
163163 int err;
164164 struct mmu_notifier_range range;
165− struct mem_cgroup *memcg;
166165
167166 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
168167 addr + PAGE_SIZE);
169168
170169 if (new_page) {
171− err = mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL,
172− &memcg);
170+ err = mem_cgroup_charge(new_page, vma->vm_mm, GFP_KERNEL,
171+ false);
173172 if (err)
174173 return err;
175174 }
@@ -179,16 +178,12 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 3 unmodified lines
179178
180179 mmu_notifier_invalidate_range_start(&range);
181180 err = -EAGAIN;
182− if (!page_vma_mapped_walk(&pvmw)) {
183− if (new_page)
184− mem_cgroup_cancel_charge(new_page, memcg);
181+ if (!page_vma_mapped_walk(&pvmw))
185182 goto unlock;
186− }
187183 VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
188184
189185 if (new_page) {
190186 get_page(new_page);
191− mem_cgroup_commit_charge(new_page, memcg, false);
192187 page_add_new_anon_rmap(new_page, vma, addr, false);
193188 lru_cache_add_active_or_unevictable(new_page, vma);
194189 } else
mm/filemap.c+1−1
@@ -2633,7 +2633,7 @@ void filemap_map_pages(struct vm_fault *vmf, · 2632 unmodified lines
26332633 if (vmf->pte)
26342634 vmf->pte += xas.xa_index - last_pgoff;
26352635 last_pgoff = xas.xa_index;
2636− if (alloc_set_pte(vmf, NULL, page))
2636+ if (alloc_set_pte(vmf, page))
26372637 goto unlock;
26382638 unlock_page(page);
26392639 goto next;
mm/huge_memory.c+3−6
@@ -587,19 +587,19 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, · 586 unmodified lines
587587 struct page *page, gfp_t gfp)
588588 {
589589 struct vm_area_struct *vma = vmf->vma;
590− struct mem_cgroup *memcg;
591590 pgtable_t pgtable;
592591 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
593592 vm_fault_t ret = 0;
594593
595594 VM_BUG_ON_PAGE(!PageCompound(page), page);
596595
597− if (mem_cgroup_try_charge_delay(page, vma->vm_mm, gfp, &memcg)) {
596+ if (mem_cgroup_charge(page, vma->vm_mm, gfp, false)) {
598597 put_page(page);
599598 count_vm_event(THP_FAULT_FALLBACK);
600599 count_vm_event(THP_FAULT_FALLBACK_CHARGE);
601600 return VM_FAULT_FALLBACK;
602601 }
602+ cgroup_throttle_swaprate(page, gfp);
603603
604604 pgtable = pte_alloc_one(vma->vm_mm);
605605 if (unlikely(!pgtable)) {
@@ -630,7 +630,6 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, · 24 unmodified lines
630630 vm_fault_t ret2;
631631
632632 spin_unlock(vmf->ptl);
633− mem_cgroup_cancel_charge(page, memcg);
634633 put_page(page);
635634 pte_free(vma->vm_mm, pgtable);
636635 ret2 = handle_userfault(vmf, VM_UFFD_MISSING);
@@ -640,7 +639,6 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, · 3 unmodified lines
640639
641640 entry = mk_huge_pmd(page, vma->vm_page_prot);
642641 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
643− mem_cgroup_commit_charge(page, memcg, false);
644642 page_add_new_anon_rmap(page, vma, haddr, true);
645643 lru_cache_add_active_or_unevictable(page, vma);
646644 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
@@ -649,7 +647,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, · 2 unmodified lines
649647 mm_inc_nr_ptes(vma->vm_mm);
650648 spin_unlock(vmf->ptl);
651649 count_vm_event(THP_FAULT_ALLOC);
652− count_memcg_events(memcg, THP_FAULT_ALLOC, 1);
650+ count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC);
653651 }
654652
655653 return 0;
@@ -658,7 +656,6 @@ unlock_release: · 2 unmodified lines
658656 release:
659657 if (pgtable)
660658 pte_free(vma->vm_mm, pgtable);
661− mem_cgroup_cancel_charge(page, memcg);
662659 put_page(page);
663660 return ret;
664661
mm/khugepaged.c+10−25
@@ -1037,7 +1037,6 @@ static void collapse_huge_page(struct mm_struct *mm, · 1036 unmodified lines
10371037 struct page *new_page;
10381038 spinlock_t *pmd_ptl, *pte_ptl;
10391039 int isolated = 0, result = 0;
1040− struct mem_cgroup *memcg;
10411040 struct vm_area_struct *vma;
10421041 struct mmu_notifier_range range;
10431042 gfp_t gfp;
@@ -1060,15 +1059,15 @@ static void collapse_huge_page(struct mm_struct *mm, · 16 unmodified lines
10601059 goto out_nolock;
10611060 }
10621061
1063− if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg))) {
1062+ if (unlikely(mem_cgroup_charge(new_page, mm, gfp, false))) {
10641063 result = SCAN_CGROUP_CHARGE_FAIL;
10651064 goto out_nolock;
10661065 }
1066+ count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC);
10671067
10681068 down_read(&mm->mmap_sem);
10691069 result = hugepage_vma_revalidate(mm, address, &vma);
10701070 if (result) {
1071− mem_cgroup_cancel_charge(new_page, memcg);
10721071 up_read(&mm->mmap_sem);
10731072 goto out_nolock;
10741073 }
@@ -1076,7 +1075,6 @@ static void collapse_huge_page(struct mm_struct *mm, · 1 unmodified line
10761075 pmd = mm_find_pmd(mm, address);
10771076 if (!pmd) {
10781077 result = SCAN_PMD_NULL;
1079− mem_cgroup_cancel_charge(new_page, memcg);
10801078 up_read(&mm->mmap_sem);
10811079 goto out_nolock;
10821080 }
@@ -1088,7 +1086,6 @@ static void collapse_huge_page(struct mm_struct *mm, · 5 unmodified lines
10881086 */
10891087 if (unmapped && !__collapse_huge_page_swapin(mm, vma, address,
10901088 pmd, referenced)) {
1091− mem_cgroup_cancel_charge(new_page, memcg);
10921089 up_read(&mm->mmap_sem);
10931090 goto out_nolock;
10941091 }
@@ -1175,9 +1172,7 @@ static void collapse_huge_page(struct mm_struct *mm, · 80 unmodified lines
11751172
11761173 spin_lock(pmd_ptl);
11771174 BUG_ON(!pmd_none(*pmd));
1178− mem_cgroup_commit_charge(new_page, memcg, false);
11791175 page_add_new_anon_rmap(new_page, vma, address, true);
1180− count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);
11811176 lru_cache_add_active_or_unevictable(new_page, vma);
11821177 pgtable_trans_huge_deposit(mm, pmd, pgtable);
11831178 set_pmd_at(mm, address, pmd, _pmd);
@@ -1191,10 +1186,11 @@ static void collapse_huge_page(struct mm_struct *mm, · 7 unmodified lines
11911186 out_up_write:
11921187 up_write(&mm->mmap_sem);
11931188 out_nolock:
1189+ if (!IS_ERR_OR_NULL(*hpage))
1190+ mem_cgroup_uncharge(*hpage);
11941191 trace_mm_collapse_huge_page(mm, isolated, result);
11951192 return;
11961193 out:
1197− mem_cgroup_cancel_charge(new_page, memcg);
11981194 goto out_up_write;
11991195 }
12001196
@@ -1618,7 +1614,6 @@ static void collapse_file(struct mm_struct *mm, · 417 unmodified lines
16181614 struct address_space *mapping = file->f_mapping;
16191615 gfp_t gfp;
16201616 struct page *new_page;
1621− struct mem_cgroup *memcg;
16221617 pgoff_t index, end = start + HPAGE_PMD_NR;
16231618 LIST_HEAD(pagelist);
16241619 XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
@@ -1637,10 +1632,11 @@ static void collapse_file(struct mm_struct *mm, · 12 unmodified lines
16371632 goto out;
16381633 }
16391634
1640− if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg))) {
1635+ if (unlikely(mem_cgroup_charge(new_page, mm, gfp, false))) {
16411636 result = SCAN_CGROUP_CHARGE_FAIL;
16421637 goto out;
16431638 }
1639+ count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC);
16441640
16451641 /* This will be less messy when we use multi-index entries */
16461642 do {
@@ -1650,7 +1646,6 @@ static void collapse_file(struct mm_struct *mm, · 3 unmodified lines
16501646 break;
16511647 xas_unlock_irq(&xas);
16521648 if (!xas_nomem(&xas, GFP_KERNEL)) {
1653− mem_cgroup_cancel_charge(new_page, memcg);
16541649 result = SCAN_FAIL;
16551650 goto out;
16561651 }
@@ -1844,18 +1839,9 @@ out_unlock: · 187 unmodified lines
18441839 }
18451840
18461841 if (nr_none) {
1847− struct lruvec *lruvec;
1848− /*
1849− * XXX: We have started try_charge and pinned the
1850− * memcg, but the page isn't committed yet so we
1851− * cannot use mod_lruvec_page_state(). This hackery
1852− * will be cleaned up when remove the page->mapping
1853− * dependency from memcg and fully charge above.
1854− */
1855− lruvec = mem_cgroup_lruvec(memcg, page_pgdat(new_page));
1856− __mod_lruvec_state(lruvec, NR_FILE_PAGES, nr_none);
1842+ __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none);
18571843 if (is_shmem)
1858− __mod_lruvec_state(lruvec, NR_SHMEM, nr_none);
1844+ __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none);
18591845 }
18601846
18611847 xa_locked:
@@ -1893,7 +1879,6 @@ xa_unlocked: · 31 unmodified lines
18931879
18941880 SetPageUptodate(new_page);
18951881 page_ref_add(new_page, HPAGE_PMD_NR - 1);
1896− mem_cgroup_commit_charge(new_page, memcg, false);
18971882
18981883 if (is_shmem) {
18991884 set_page_dirty(new_page);
@@ -1901,7 +1886,6 @@ xa_unlocked: · 1 unmodified line
19011886 } else {
19021887 lru_cache_add_file(new_page);
19031888 }
1904− count_memcg_events(memcg, THP_COLLAPSE_ALLOC, 1);
19051889
19061890 /*
19071891 * Remove pte page tables, so we can re-fault the page as huge.
@@ -1948,13 +1932,14 @@ xa_unlocked: · 40 unmodified lines
19481932 VM_BUG_ON(nr_none);
19491933 xas_unlock_irq(&xas);
19501934
1951− mem_cgroup_cancel_charge(new_page, memcg);
19521935 new_page->mapping = NULL;
19531936 }
19541937
19551938 unlock_page(new_page);
19561939 out:
19571940 VM_BUG_ON(!list_empty(&pagelist));
1941+ if (!IS_ERR_OR_NULL(*hpage))
1942+ mem_cgroup_uncharge(*hpage);
19581943 /* TODO: tracepoints */
19591944 }
19601945
mm/memory.c+10−26
@@ -2645,7 +2645,6 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) · 2644 unmodified lines
26452645 struct page *new_page = NULL;
26462646 pte_t entry;
26472647 int page_copied = 0;
2648− struct mem_cgroup *memcg;
26492648 struct mmu_notifier_range range;
26502649
26512650 if (unlikely(anon_vma_prepare(vma)))
@@ -2676,8 +2675,9 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) · 24 unmodified lines
26762675 }
26772676 }
26782677
2679− if (mem_cgroup_try_charge_delay(new_page, mm, GFP_KERNEL, &memcg))
2678+ if (mem_cgroup_charge(new_page, mm, GFP_KERNEL, false))
26802679 goto oom_free_new;
2680+ cgroup_throttle_swaprate(new_page, GFP_KERNEL);
26812681
26822682 __SetPageUptodate(new_page);
26832683
@@ -2710,7 +2710,6 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) · 26 unmodified lines
27102710 * thread doing COW.
27112711 */
27122712 ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
2713− mem_cgroup_commit_charge(new_page, memcg, false);
27142713 page_add_new_anon_rmap(new_page, vma, vmf->address, false);
27152714 lru_cache_add_active_or_unevictable(new_page, vma);
27162715 /*
@@ -2749,8 +2748,6 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) · 32 unmodified lines
27492748 /* Free the old page.. */
27502749 new_page = old_page;
27512750 page_copied = 1;
2752− } else {
2753− mem_cgroup_cancel_charge(new_page, memcg);
27542751 }
27552752
27562753 if (new_page)
@@ -3088,7 +3085,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) · 331 unmodified lines
30883085 {
30893086 struct vm_area_struct *vma = vmf->vma;
30903087 struct page *page = NULL, *swapcache;
3091− struct mem_cgroup *memcg;
30923088 swp_entry_t entry;
30933089 pte_t pte;
30943090 int locked;
@@ -3193,10 +3189,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) · 98 unmodified lines
31933189 goto out_page;
31943190 }
31953191
3196− if (mem_cgroup_try_charge_delay(page, vma->vm_mm, GFP_KERNEL, &memcg)) {
3192+ if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL, true)) {
31973193 ret = VM_FAULT_OOM;
31983194 goto out_page;
31993195 }
3196+ cgroup_throttle_swaprate(page, GFP_KERNEL);
32003197
32013198 /*
32023199 * Back out if somebody else already faulted in this pte.
@@ -3243,11 +3240,9 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) · 40 unmodified lines
32433240
32443241 /* ksm created a completely new copy */
32453242 if (unlikely(page != swapcache && swapcache)) {
3246− mem_cgroup_commit_charge(page, memcg, false);
32473243 page_add_new_anon_rmap(page, vma, vmf->address, false);
32483244 lru_cache_add_active_or_unevictable(page, vma);
32493245 } else {
3250− mem_cgroup_commit_charge(page, memcg, true);
32513246 do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
32523247 activate_page(page);
32533248 }
@@ -3284,7 +3279,6 @@ unlock: · 30 unmodified lines
32843279 out:
32853280 return ret;
32863281 out_nomap:
3287− mem_cgroup_cancel_charge(page, memcg);
32883282 pte_unmap_unlock(vmf->pte, vmf->ptl);
32893283 out_page:
32903284 unlock_page(page);
@@ -3305,7 +3299,6 @@ out_release: · 14 unmodified lines
33053299 static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
33063300 {
33073301 struct vm_area_struct *vma = vmf->vma;
3308− struct mem_cgroup *memcg;
33093302 struct page *page;
33103303 vm_fault_t ret = 0;
33113304 pte_t entry;
@@ -3358,8 +3351,9 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) · 46 unmodified lines
33583351 if (!page)
33593352 goto oom;
33603353
3361− if (mem_cgroup_try_charge_delay(page, vma->vm_mm, GFP_KERNEL, &memcg))
3354+ if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL, false))
33623355 goto oom_free_page;
3356+ cgroup_throttle_swaprate(page, GFP_KERNEL);
33633357
33643358 /*
33653359 * The memory barrier inside __SetPageUptodate makes sure that
@@ -3384,13 +3378,11 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) · 18 unmodified lines
33843378 /* Deliver the page fault to userland, check inside PT lock */
33853379 if (userfaultfd_missing(vma)) {
33863380 pte_unmap_unlock(vmf->pte, vmf->ptl);
3387− mem_cgroup_cancel_charge(page, memcg);
33883381 put_page(page);
33893382 return handle_userfault(vmf, VM_UFFD_MISSING);
33903383 }
33913384
33923385 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
3393− mem_cgroup_commit_charge(page, memcg, false);
33943386 page_add_new_anon_rmap(page, vma, vmf->address, false);
33953387 lru_cache_add_active_or_unevictable(page, vma);
33963388 setpte:
@@ -3402,7 +3394,6 @@ unlock: · 5 unmodified lines
34023394 pte_unmap_unlock(vmf->pte, vmf->ptl);
34033395 return ret;
34043396 release:
3405− mem_cgroup_cancel_charge(page, memcg);
34063397 put_page(page);
34073398 goto unlock;
34083399 oom_free_page:
@@ -3607,7 +3598,6 @@ static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) · 198 unmodified lines
36073598 * mapping. If needed, the fucntion allocates page table or use pre-allocated.
36083599 *
36093600 * @vmf: fault environment
3610− * @memcg: memcg to charge page (only for private mappings)
36113601 * @page: page to map
36123602 *
36133603 * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on
@@ -3618,8 +3608,7 @@ static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) · 4 unmodified lines
36183608 *
36193609 * Return: %0 on success, %VM_FAULT_ code in case of error.
36203610 */
3621−vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
3622− struct page *page)
3611+vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page)
36233612 {
36243613 struct vm_area_struct *vma = vmf->vma;
36253614 bool write = vmf->flags & FAULT_FLAG_WRITE;
@@ -3627,9 +3616,6 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg, · 1 unmodified line
36273616 vm_fault_t ret;
36283617
36293618 if (pmd_none(*vmf->pmd) && PageTransCompound(page)) {
3630− /* THP on COW? */
3631− VM_BUG_ON_PAGE(memcg, page);
3632−
36333619 ret = do_set_pmd(vmf, page);
36343620 if (ret != VM_FAULT_FALLBACK)
36353621 return ret;
@@ -3652,7 +3638,6 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg, · 16 unmodified lines
36523638 /* copy-on-write page */
36533639 if (write && !(vma->vm_flags & VM_SHARED)) {
36543640 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
3655− mem_cgroup_commit_charge(page, memcg, false);
36563641 page_add_new_anon_rmap(page, vma, vmf->address, false);
36573642 lru_cache_add_active_or_unevictable(page, vma);
36583643 } else {
@@ -3702,7 +3687,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf) · 43 unmodified lines
37023687 if (!(vmf->vma->vm_flags & VM_SHARED))
37033688 ret = check_stable_address_space(vmf->vma->vm_mm);
37043689 if (!ret)
3705− ret = alloc_set_pte(vmf, vmf->memcg, page);
3690+ ret = alloc_set_pte(vmf, page);
37063691 if (vmf->pte)
37073692 pte_unmap_unlock(vmf->pte, vmf->ptl);
37083693 return ret;
@@ -3862,11 +3847,11 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) · 153 unmodified lines
38623847 if (!vmf->cow_page)
38633848 return VM_FAULT_OOM;
38643849
3865− if (mem_cgroup_try_charge_delay(vmf->cow_page, vma->vm_mm,
3866− GFP_KERNEL, &vmf->memcg)) {
3850+ if (mem_cgroup_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL, false)) {
38673851 put_page(vmf->cow_page);
38683852 return VM_FAULT_OOM;
38693853 }
3854+ cgroup_throttle_swaprate(vmf->cow_page, GFP_KERNEL);
38703855
38713856 ret = __do_fault(vmf);
38723857 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
@@ -3884,7 +3869,6 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) · 11 unmodified lines
38843869 goto uncharge_out;
38853870 return ret;
38863871 uncharge_out:
3887− mem_cgroup_cancel_charge(vmf->cow_page, vmf->memcg);
38883872 put_page(vmf->cow_page);
38893873 return ret;
38903874 }
mm/migrate.c+1−4
@@ -2740,7 +2740,6 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, · 2739 unmodified lines
27402740 {
27412741 struct vm_area_struct *vma = migrate->vma;
27422742 struct mm_struct *mm = vma->vm_mm;
2743− struct mem_cgroup *memcg;
27442743 bool flush = false;
27452744 spinlock_t *ptl;
27462745 pte_t entry;
@@ -2787,7 +2786,7 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, · 40 unmodified lines
27872786
27882787 if (unlikely(anon_vma_prepare(vma)))
27892788 goto abort;
2790− if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg))
2789+ if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL, false))
27912790 goto abort;
27922791
27932792 /*
@@ -2832,7 +2831,6 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, · 38 unmodified lines
28322831 goto unlock_abort;
28332832
28342833 inc_mm_counter(mm, MM_ANONPAGES);
2835− mem_cgroup_commit_charge(page, memcg, false);
28362834 page_add_new_anon_rmap(page, vma, addr, false);
28372835 if (!is_zone_device_page(page))
28382836 lru_cache_add_active_or_unevictable(page, vma);
@@ -2855,7 +2853,6 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, · 16 unmodified lines
28552853
28562854 unlock_abort:
28572855 pte_unmap_unlock(ptep, ptl);
2858− mem_cgroup_cancel_charge(page, memcg);
28592856 abort:
28602857 *src &= ~MIGRATE_PFN_MIGRATE;
28612858 }
mm/swapfile.c+1−5
@@ -1892,7 +1892,6 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, · 1891 unmodified lines
18921892 unsigned long addr, swp_entry_t entry, struct page *page)
18931893 {
18941894 struct page *swapcache;
1895− struct mem_cgroup *memcg;
18961895 spinlock_t *ptl;
18971896 pte_t *pte;
18981897 int ret = 1;
@@ -1902,14 +1901,13 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, · 3 unmodified lines
19021901 if (unlikely(!page))
19031902 return -ENOMEM;
19041903
1905− if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg)) {
1904+ if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL, true)) {
19061905 ret = -ENOMEM;
19071906 goto out_nolock;
19081907 }
19091908
19101909 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
19111910 if (unlikely(!pte_same_as_swp(*pte, swp_entry_to_pte(entry)))) {
1912− mem_cgroup_cancel_charge(page, memcg);
19131911 ret = 0;
19141912 goto out;
19151913 }
@@ -1920,10 +1918,8 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, · 4 unmodified lines
19201918 set_pte_at(vma->vm_mm, addr, pte,
19211919 pte_mkold(mk_pte(page, vma->vm_page_prot)));
19221920 if (page == swapcache) {
1923− mem_cgroup_commit_charge(page, memcg, true);
19241921 page_add_anon_rmap(page, vma, addr, false);
19251922 } else { /* ksm created a completely new copy */
1926− mem_cgroup_commit_charge(page, memcg, false);
19271923 page_add_new_anon_rmap(page, vma, addr, false);
19281924 lru_cache_add_active_or_unevictable(page, vma);
19291925 }
mm/userfaultfd.c+1−4
@@ -56,7 +56,6 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, · 55 unmodified lines
5656 struct page **pagep,
5757 bool wp_copy)
5858 {
59− struct mem_cgroup *memcg;
6059 pte_t _dst_pte, *dst_pte;
6160 spinlock_t *ptl;
6261 void *page_kaddr;
@@ -97,7 +96,7 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, · 34 unmodified lines
9796 __SetPageUptodate(page);
9897
9998 ret = -ENOMEM;
100− if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg))
99+ if (mem_cgroup_charge(page, dst_mm, GFP_KERNEL, false))
101100 goto out_release;
102101
103102 _dst_pte = pte_mkdirty(mk_pte(page, dst_vma->vm_page_prot));
@@ -123,7 +122,6 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, · 19 unmodified lines
123122 goto out_release_uncharge_unlock;
124123
125124 inc_mm_counter(dst_mm, MM_ANONPAGES);
126− mem_cgroup_commit_charge(page, memcg, false);
127125 page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
128126 lru_cache_add_active_or_unevictable(page, dst_vma);
129127
@@ -138,7 +136,6 @@ out: · 8 unmodified lines
138136 return ret;
139137 out_release_uncharge_unlock:
140138 pte_unmap_unlock(dst_pte, ptl);
141− mem_cgroup_cancel_charge(page, memcg);
142139 out_release:
143140 put_page(page);
144141 goto out;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t