mm: memcontrol: convert page cache to a new mem_cgroup_charge() API

The try/commit/cancel protocol that memcg uses dates back to when pages used to be uncharged upon removal from the page cache, and thus couldn't be committed before the insertion had succeeded. Nowadays, pages are uncharged when they are physically freed; it doesn't matter whether the insertion was successful or not. For the page cache, the transaction dance has become unnecessary. Introduce a mem_cgroup_charge() function that simply charges a newly allocated page to a cgroup and sets up page->mem_cgroup in one single step. If the insertion fails, the caller doesn't have to do anything but free/put the page. Then switch the page cache over to this new API. Subsequent patches will also convert anon pages, but it needs a bit more prep work. Right now, memcg depends on page->mapping being already set up at the time of charging, so that it can maintain its own MEMCG_CACHE and MEMCG_RSS counters. For anon, page->mapping is set under the same pte lock under which the page is publishd, so a single charge point that can block doesn't work there just yet. The following prep patches will replace the private memcg counters with the generic vmstat counters, thus removing the page->mapping dependency, then complete the transition to the new single-point charge API and delete the old transactional scheme. v2: leave shmem swapcache when charging fails to avoid double IO (Joonsoo) v3: rebase on preceeding shmem simplification patch

Johannes Weiner committed 6 years ago

Commit 3fea5a41 parent 6caa6a04 files · +77 −59Browse files

Unified · Split

4 files

include/linux/memcontrol.h +10 −0
mm/filemap.c +10 −14
mm/memcontrol.c +27 −2
mm/shmem.c +30 −43

include/linux/memcontrol.h+10−0

 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
 			      bool lrucare);
 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg);
+int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask,
+		      bool lrucare);
 void mem_cgroup_uncharge(struct page *page);
 void mem_cgroup_uncharge_list(struct list_head *page_list);
 {
 }
+static inline int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+				    gfp_t gfp_mask, bool lrucare)
+{
+	return 0;
+}
 static inline void mem_cgroup_uncharge(struct page *page)
 {
 }

mm/filemap.c+10−14

 {
 	XA_STATE(xas, &mapping->i_pages, offset);
 	int huge = PageHuge(page);
-	struct mem_cgroup *memcg;
 	int error;
 	void *old;
 	VM_BUG_ON_PAGE(PageSwapBacked(page), page);
 	mapping_set_update(&xas, mapping);
-	if (!huge) {
-		error = mem_cgroup_try_charge(page, current->mm,
-					      gfp_mask, &memcg);
-		if (error)
-			return error;
-	}
 	get_page(page);
 	page->mapping = mapping;
 	page->index = offset;
+	if (!huge) {
+		error = mem_cgroup_charge(page, current->mm, gfp_mask, false);
+		if (error)
+			goto error;
+	}
 	do {
 		xas_lock_irq(&xas);
 		old = xas_load(&xas);
 		xas_unlock_irq(&xas);
 	} while (xas_nomem(&xas, gfp_mask & GFP_RECLAIM_MASK));
-	if (xas_error(&xas))
+	if (xas_error(&xas)) {
+		error = xas_error(&xas);
 		goto error;
+	}
-	if (!huge)
-		mem_cgroup_commit_charge(page, memcg, false);
 	trace_mm_filemap_add_to_page_cache(page);
 	return 0;
 error:
 	page->mapping = NULL;
 	/* Leave page->index set: truncation relies upon it */
-	if (!huge)
-		mem_cgroup_cancel_charge(page, memcg);
 	put_page(page);
-	return xas_error(&xas);
+	return error;
 }
 ALLOW_ERROR_INJECTION(__add_to_page_cache_locked, ERRNO);

mm/memcontrol.c+27−2

@@ -6637,6 +6637,33 @@ void mem_cgroup_cancel_charge(struct page page, struct mem_cgroup memcg) · 6636 unmodified lines
6637	cancel_charge(memcg, nr_pages);	6637	cancel_charge(memcg, nr_pages);
6638	}	6638	}
6639		6639
		6640	/**
		6641	* mem_cgroup_charge - charge a newly allocated page to a cgroup
		6642	* @page: page to charge
		6643	* @mm: mm context of the victim
		6644	* @gfp_mask: reclaim mode
		6645	* @lrucare: page might be on the LRU already
		6646	*
		6647	* Try to charge @page to the memcg that @mm belongs to, reclaiming
		6648	* pages according to @gfp_mask if necessary.
		6649	*
		6650	* Returns 0 on success. Otherwise, an error code is returned.
		6651	*/
		6652	int mem_cgroup_charge(struct page page, struct mm_struct mm, gfp_t gfp_mask,
		6653	bool lrucare)
		6654	{
		6655	struct mem_cgroup *memcg;
		6656	int ret;
		6657
		6658	VM_BUG_ON_PAGE(!page->mapping, page);
		6659
		6660	ret = mem_cgroup_try_charge(page, mm, gfp_mask, &memcg);
		6661	if (ret)
		6662	return ret;
		6663	mem_cgroup_commit_charge(page, memcg, lrucare);
		6664	return 0;
		6665	}
		6666
6640	struct uncharge_gather {	6667	struct uncharge_gather {
6641	struct mem_cgroup *memcg;	6668	struct mem_cgroup *memcg;
6642	unsigned long pgpgout;	6669	unsigned long pgpgout;
@@ -6684,8 +6711,6 @@ static void uncharge_batch(const struct uncharge_gather *ug) · 41 unmodified lines
6684	static void uncharge_page(struct page page, struct uncharge_gather ug)	6711	static void uncharge_page(struct page page, struct uncharge_gather ug)
6685	{	6712	{
6686	VM_BUG_ON_PAGE(PageLRU(page), page);	6713	VM_BUG_ON_PAGE(PageLRU(page), page);
6687	VM_BUG_ON_PAGE(page_count(page) && !is_zone_device_page(page) &&
6688	!PageHWPoison(page) , page);
6689		6714
6690	if (!page->mem_cgroup)	6715	if (!page->mem_cgroup)
6691	return;	6716	return;

mm/shmem.c+30−43

  */
 static int shmem_add_to_page_cache(struct page *page,
 				   struct address_space *mapping,
-				   pgoff_t index, void *expected, gfp_t gfp)
+				   pgoff_t index, void *expected, gfp_t gfp,
+				   struct mm_struct *charge_mm)
 {
 	XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
 	unsigned long i = 0;
 	unsigned long nr = compound_nr(page);
+	int error;
 	VM_BUG_ON_PAGE(PageTail(page), page);
 	VM_BUG_ON_PAGE(index != round_down(index, nr), page);
 	page->mapping = mapping;
 	page->index = index;
+	error = mem_cgroup_charge(page, charge_mm, gfp, PageSwapCache(page));
+	if (error) {
+		if (!PageSwapCache(page) && PageTransHuge(page)) {
+			count_vm_event(THP_FILE_FALLBACK);
+			count_vm_event(THP_FILE_FALLBACK_CHARGE);
+		}
+		goto error;
+	}
+	cgroup_throttle_swaprate(page, gfp);
 	do {
 		void *entry;
 		xas_lock_irq(&xas);
 	} while (xas_nomem(&xas, gfp));
 	if (xas_error(&xas)) {
-		page->mapping = NULL;
+		error = xas_error(&xas);
-		page_ref_sub(page, nr);
+		goto error;
-		return xas_error(&xas);
 	}
 	return 0;
+error:
+	page->mapping = NULL;
+	page_ref_sub(page, nr);
+	return error;
 }
 /*
 	struct address_space *mapping = inode->i_mapping;
 	struct shmem_inode_info *info = SHMEM_I(inode);
 	struct mm_struct *charge_mm = vma ? vma->vm_mm : current->mm;
-	struct mem_cgroup *memcg;
 	struct page *page;
 	swp_entry_t swap;
 	int error;
 			goto failed;
 	}
-	error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg);
-	if (error)
-		goto failed;
 	error = shmem_add_to_page_cache(page, mapping, index,
-					swp_to_radix_entry(swap), gfp);
+					swp_to_radix_entry(swap), gfp,
-	if (error) {
+					charge_mm);
-		mem_cgroup_cancel_charge(page, memcg);
+	if (error)
 		goto failed;
-	}
-	mem_cgroup_commit_charge(page, memcg, true);
 	spin_lock_irq(&info->lock);
 	info->swapped--;
 	struct shmem_inode_info *info = SHMEM_I(inode);
 	struct shmem_sb_info *sbinfo;
 	struct mm_struct *charge_mm;
-	struct mem_cgroup *memcg;
 	struct page *page;
 	enum sgp_type sgp_huge = sgp;
 	pgoff_t hindex = index;
 	if (sgp == SGP_WRITE)
 		__SetPageReferenced(page);
-	error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg);
-	if (error) {
-		if (PageTransHuge(page)) {
-			count_vm_event(THP_FILE_FALLBACK);
-			count_vm_event(THP_FILE_FALLBACK_CHARGE);
-		}
-		goto unacct;
-	}
 	error = shmem_add_to_page_cache(page, mapping, hindex,
-					NULL, gfp & GFP_RECLAIM_MASK);
+					NULL, gfp & GFP_RECLAIM_MASK,
-	if (error) {
+					charge_mm);
-		mem_cgroup_cancel_charge(page, memcg);
+	if (error)
 		goto unacct;
-	}
-	mem_cgroup_commit_charge(page, memcg, false);
 	lru_cache_add_anon(page);
 	spin_lock_irq(&info->lock);
 	struct address_space *mapping = inode->i_mapping;
 	gfp_t gfp = mapping_gfp_mask(mapping);
 	pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
-	struct mem_cgroup *memcg;
 	spinlock_t *ptl;
 	void *page_kaddr;
 	struct page *page;
 	if (unlikely(offset >= max_off))
 		goto out_release;
-	ret = mem_cgroup_try_charge_delay(page, dst_mm, gfp, &memcg);
-	if (ret)
-		goto out_release;
 	ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
-						gfp & GFP_RECLAIM_MASK);
+				      gfp & GFP_RECLAIM_MASK, dst_mm);
 	if (ret)
-		goto out_release_uncharge;
+		goto out_release;
-	mem_cgroup_commit_charge(page, memcg, false);
 	_dst_pte = mk_pte(page, dst_vma->vm_page_prot);
 	if (dst_vma->vm_flags & VM_WRITE)
 	ret = -EFAULT;
 	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
 	if (unlikely(offset >= max_off))
-		goto out_release_uncharge_unlock;
+		goto out_release_unlock;
 	ret = -EEXIST;
 	if (!pte_none(*dst_pte))
-		goto out_release_uncharge_unlock;
+		goto out_release_unlock;
 	lru_cache_add_anon(page);
 	ret = 0;
 out:
 	return ret;
-out_release_uncharge_unlock:
+out_release_unlock:
 	pte_unmap_unlock(dst_pte, ptl);
 	ClearPageDirty(page);
 	delete_from_page_cache(page);
-out_release_uncharge:
-	mem_cgroup_cancel_charge(page, memcg);
 out_release:
 	unlock_page(page);
 	put_page(page);