mm: vmscan: determine anon/file pressure balance at the reclaim root

We split the LRU lists into anon and file, and we rebalance the scan pressure between them when one of them begins thrashing: if the file cache experiences workingset refaults, we increase the pressure on anonymous pages; if the workload is stalled on swapins, we increase the pressure on the file cache instead. With cgroups and their nested LRU lists, we currently don't do this correctly. While recursive cgroup reclaim establishes a relative LRU order among the pages of all involved cgroups, LRU pressure balancing is done on an individual cgroup LRU level. As a result, when one cgroup is thrashing on the filesystem cache while a sibling may have cold anonymous pages, pressure doesn't get equalized between them. This patch moves LRU balancing decision to the root of reclaim - the same level where the LRU order is established. It does this by tracking LRU cost recursively, so that every level of the cgroup tree knows the aggregate LRU cost of all memory within its domain. When the page scanner calculates the scan balance for any given individual cgroup's LRU list, it uses the values from the ancestor cgroup that initiated the reclaim cycle. If one sibling is then thrashing on the cache, it will tip the pressure balance inside its ancestors, and the next hierarchical reclaim iteration will go more after the anon pages in the tree.

Johannes Weiner committed 6 years ago

Commit 7cf111b1 parent 314b57f3 files · +57 −29Browse files

Unified · Split

3 files

include/linux/memcontrol.h +13 −0
mm/swap.c +27 −5
mm/vmscan.c +17 −24

include/linux/memcontrol.h+13−0

@@ -1242,6 +1242,19 @@ static inline void dec_lruvec_page_state(struct page *page, · 1241 unmodified lines
1242	mod_lruvec_page_state(page, idx, -1);	1242	mod_lruvec_page_state(page, idx, -1);
1243	}	1243	}
1244		1244
		1245	static inline struct lruvec parent_lruvec(struct lruvec lruvec)
		1246	{
		1247	struct mem_cgroup *memcg;
		1248
		1249	memcg = lruvec_memcg(lruvec);
		1250	if (!memcg)
		1251	return NULL;
		1252	memcg = parent_mem_cgroup(memcg);
		1253	if (!memcg)
		1254	return NULL;
		1255	return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));
		1256	}
		1257
1245	#ifdef CONFIG_CGROUP_WRITEBACK	1258	#ifdef CONFIG_CGROUP_WRITEBACK
1246		1259
1247	struct wb_domain mem_cgroup_wb_domain(struct bdi_writeback wb);	1260	struct wb_domain mem_cgroup_wb_domain(struct bdi_writeback wb);

mm/swap.c+27−5

@@ -282,11 +282,33 @@ void lru_note_cost(struct page *page) · 281 unmodified lines
282	{	282	{
283	struct lruvec *lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));	283	struct lruvec *lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
284		284
285	/* Record new data point */	285	do {
286	if (page_is_file_lru(page))	286	unsigned long lrusize;
287	lruvec->file_cost++;	287
288	else	288	/* Record cost event */
289	lruvec->anon_cost++;	289	if (page_is_file_lru(page))
		290	lruvec->file_cost++;
		291	else
		292	lruvec->anon_cost++;
		293
		294	/*
		295	* Decay previous events
		296	*
		297	* Because workloads change over time (and to avoid
		298	* overflow) we keep these statistics as a floating
		299	* average, which ends up weighing recent refaults
		300	* more than old ones.
		301	*/
		302	lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
		303	lruvec_page_state(lruvec, NR_ACTIVE_ANON) +
		304	lruvec_page_state(lruvec, NR_INACTIVE_FILE) +
		305	lruvec_page_state(lruvec, NR_ACTIVE_FILE);
		306
		307	if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) {
		308	lruvec->file_cost /= 2;
		309	lruvec->anon_cost /= 2;
		310	}
		311	} while ((lruvec = parent_lruvec(lruvec)));
290	}	312	}
291		313
292	static void __activate_page(struct page page, struct lruvec lruvec,	314	static void __activate_page(struct page page, struct lruvec lruvec,

mm/vmscan.c+17−24

 	 */
 	struct mem_cgroup *target_mem_cgroup;
+	/*
+	 * Scan pressure balancing between anon and file LRUs
+	 */
+	unsigned long	anon_cost;
+	unsigned long	file_cost;
 	/* Can active pages be deactivated as part of reclaim? */
 #define DEACTIVATE_ANON 1
 #define DEACTIVATE_FILE 2
 	int swappiness = mem_cgroup_swappiness(memcg);
 	u64 fraction[2];
 	u64 denominator = 0;	/* gcc */
-	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
 	unsigned long anon_prio, file_prio;
 	enum scan_balance scan_balance;
-	unsigned long anon, file;
 	unsigned long totalcost;
 	unsigned long ap, fp;
 	enum lru_list lru;
 	}
 	scan_balance = SCAN_FRACT;
 	/*
 	 * Calculate the pressure balance between anon and file pages.
 	 *
 	anon_prio = swappiness;
 	file_prio = 200 - anon_prio;
-	/*
+	totalcost = sc->anon_cost + sc->file_cost;
-	 * Because workloads change over time (and to avoid overflow)
-	 * we keep these statistics as a floating average, which ends
-	 * up weighing recent refaults more than old ones.
-	 */
-	anon  = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +
-		lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, MAX_NR_ZONES);
-	file  = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) +
-		lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);
-	spin_lock_irq(&pgdat->lru_lock);
-	totalcost = lruvec->anon_cost + lruvec->file_cost;
-	if (unlikely(totalcost > (anon + file) / 4)) {
-		lruvec->anon_cost /= 2;
-		lruvec->file_cost /= 2;
-		totalcost /= 2;
-	}
 	ap = anon_prio * (totalcost + 1);
-	ap /= lruvec->anon_cost + 1;
+	ap /= sc->anon_cost + 1;
 	fp = file_prio * (totalcost + 1);
-	fp /= lruvec->file_cost + 1;
+	fp /= sc->file_cost + 1;
-	spin_unlock_irq(&pgdat->lru_lock);
 	fraction[0] = ap;
 	fraction[1] = fp;
 	nr_reclaimed = sc->nr_reclaimed;
 	nr_scanned = sc->nr_scanned;
+	/*
+	 * Determine the scan balance between anon and file LRUs.
+	 */
+	spin_lock_irq(&pgdat->lru_lock);
+	sc->anon_cost = target_lruvec->anon_cost;
+	sc->file_cost = target_lruvec->file_cost;
+	spin_unlock_irq(&pgdat->lru_lock);
 	/*
 	 * Target desirable inactive:active list ratios for the anon
 	 * and file LRU lists.