History

mm: base LRU balancing on an explicit cost model

Currently, scan pressure between the anon and file LRU lists is balanced based on a mixture of reclaim efficiency and a somewhat vague notion of "value" of having certain pages in memory over others. That concept of value is problematic, because it has caused us to count any event that remotely makes one LRU list more or less preferrable for reclaim, even when these events are not directly comparable and impose very different costs on the system. One example is referenced file pages that we still deactivate and referenced anonymous pages that we actually rotate back to the head of the list. There is also conceptual overlap with the LRU algorithm itself. By rotating recently used pages instead of reclaiming them, the algorithm already biases the applied scan pressure based on page value. Thus, when rebalancing scan pressure due to rotations, we should think of reclaim cost, and leave assessing the page value to the LRU algorithm. Lastly, considering both value-increasing as well as value-decreasing events can sometimes cause the same type of event to be counted twice, i.e. how rotating a page increases the LRU value, while reclaiming it succesfully decreases the value. In itself this will balance out fine, but it quietly skews the impact of events that are only recorded once. The abstract metric of "value", the murky relationship with the LRU algorithm, and accounting both negative and positive events make the current pressure balancing model hard to reason about and modify. This patch switches to a balancing model of accounting the concrete, actually observed cost of reclaiming one LRU over another. For now, that cost includes pages that are scanned but rotated back to the list head. Subsequent patches will add consideration for IO caused by refaulting of recently evicted pages. Replace struct zone_reclaim_stat with two cost counters in the lruvec, and make everything that affects cost go through a new lru_note_cost() function.

Johannes Weiner committed 6 years ago

Commit 1431d4d1 parent a4fe1635 files · +45 −59Browse files

Unified · Split

5 files
  1. include/linux/mmzone.h +7 −14
  2. include/linux/swap.h +2 −0
  3. mm/memcontrol.c +6 −12
  4. mm/swap.c +8 −11
  5. mm/vmscan.c +22 −22
include/linux/mmzone.h+7−14
@@ -242,19 +242,6 @@ static inline bool is_active_lru(enum lru_list lru) · 241 unmodified lines
242 return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);242 return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
243}243}
244 244
245struct zone_reclaim_stat {
246 /*
247 * The pageout code in vmscan.c keeps track of how many of the
248 * mem/swap backed and file backed pages are referenced.
249 * The higher the rotated/scanned ratio, the more valuable
250 * that cache is.
251 *
252 * The anon LRU stats live in [0], file LRU stats in [1]
253 */
254 unsigned long recent_rotated[2];
255 unsigned long recent_scanned[2];
256};
257
258enum lruvec_flags {245enum lruvec_flags {
259 LRUVEC_CONGESTED, /* lruvec has many dirty pages246 LRUVEC_CONGESTED, /* lruvec has many dirty pages
260 * backed by a congested BDI247 * backed by a congested BDI
@@ -263,7 +250,13 @@ enum lruvec_flags { · 2 unmodified lines
263 250
264struct lruvec {251struct lruvec {
265 struct list_head lists[NR_LRU_LISTS];252 struct list_head lists[NR_LRU_LISTS];
266 struct zone_reclaim_stat reclaim_stat;253 /*
254 * These track the cost of reclaiming one LRU - file or anon -
255 * over the other. As the observed cost of reclaiming one LRU
256 * increases, the reclaim scan balance tips toward the other.
257 */
258 unsigned long anon_cost;
259 unsigned long file_cost;
267 /* Evictions & activations on the inactive file list */260 /* Evictions & activations on the inactive file list */
268 atomic_long_t inactive_age;261 atomic_long_t inactive_age;
269 /* Refaults at the time of last reclaim cycle */262 /* Refaults at the time of last reclaim cycle */
include/linux/swap.h+2−0
@@ -334,6 +334,8 @@ extern unsigned long nr_free_pagecache_pages(void); · 333 unmodified lines
334 334
335 335
336/* linux/mm/swap.c */336/* linux/mm/swap.c */
337extern void lru_note_cost(struct lruvec *lruvec, bool file,
338 unsigned int nr_pages);
337extern void lru_cache_add(struct page *);339extern void lru_cache_add(struct page *);
338extern void lru_add_page_tail(struct page *page, struct page *page_tail,340extern void lru_add_page_tail(struct page *page, struct page *page_tail,
339 struct lruvec *lruvec, struct list_head *head);341 struct lruvec *lruvec, struct list_head *head);
mm/memcontrol.c+6−12
@@ -3853,23 +3853,17 @@ static int memcg_stat_show(struct seq_file *m, void *v) · 3852 unmodified lines
3853 {3853 {
3854 pg_data_t *pgdat;3854 pg_data_t *pgdat;
3855 struct mem_cgroup_per_node *mz;3855 struct mem_cgroup_per_node *mz;
3856 struct zone_reclaim_stat *rstat;3856 unsigned long anon_cost = 0;
3857 unsigned long recent_rotated[2] = {0, 0};3857 unsigned long file_cost = 0;
3858 unsigned long recent_scanned[2] = {0, 0};
3859 3858
3860 for_each_online_pgdat(pgdat) {3859 for_each_online_pgdat(pgdat) {
3861 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);3860 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3862 rstat = &mz->lruvec.reclaim_stat;
3863 3861
3864 recent_rotated[0] += rstat->recent_rotated[0];3862 anon_cost += mz->lruvec.anon_cost;
3865 recent_rotated[1] += rstat->recent_rotated[1];3863 file_cost += mz->lruvec.file_cost;
3866 recent_scanned[0] += rstat->recent_scanned[0];
3867 recent_scanned[1] += rstat->recent_scanned[1];
3868 }3864 }
3869 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);3865 seq_printf(m, "anon_cost %lu\n", anon_cost);
3870 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);3866 seq_printf(m, "file_cost %lu\n", file_cost);
3871 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3872 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3873 }3867 }
3874#endif3868#endif
3875 3869
mm/swap.c+8−11
@@ -278,15 +278,12 @@ void rotate_reclaimable_page(struct page *page) · 277 unmodified lines
278 }278 }
279}279}
280 280
281static void update_page_reclaim_stat(struct lruvec *lruvec,281void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages)
282 int file, int rotated,
283 unsigned int nr_pages)
284{282{
285 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;283 if (file)
286 284 lruvec->file_cost += nr_pages;
287 reclaim_stat->recent_scanned[file] += nr_pages;285 else
288 if (rotated)286 lruvec->anon_cost += nr_pages;
289 reclaim_stat->recent_rotated[file] += nr_pages;
290}287}
291 288
292static void __activate_page(struct page *page, struct lruvec *lruvec,289static void __activate_page(struct page *page, struct lruvec *lruvec,
@@ -541,7 +538,7 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, · 248 unmodified lines
541 538
542 if (active)539 if (active)
543 __count_vm_event(PGDEACTIVATE);540 __count_vm_event(PGDEACTIVATE);
544 update_page_reclaim_stat(lruvec, file, 0, hpage_nr_pages(page));541 lru_note_cost(lruvec, !file, hpage_nr_pages(page));
545}542}
546 543
547static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,544static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
@@ -557,7 +554,7 @@ static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, · 9 unmodified lines
557 add_page_to_lru_list(page, lruvec, lru);554 add_page_to_lru_list(page, lruvec, lru);
558 555
559 __count_vm_events(PGDEACTIVATE, hpage_nr_pages(page));556 __count_vm_events(PGDEACTIVATE, hpage_nr_pages(page));
560 update_page_reclaim_stat(lruvec, file, 0, hpage_nr_pages(page));557 lru_note_cost(lruvec, !file, hpage_nr_pages(page));
561 }558 }
562}559}
563 560
@@ -582,7 +579,7 @@ static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, · 18 unmodified lines
582 579
583 __count_vm_events(PGLAZYFREE, hpage_nr_pages(page));580 __count_vm_events(PGLAZYFREE, hpage_nr_pages(page));
584 count_memcg_page_event(page, PGLAZYFREE);581 count_memcg_page_event(page, PGLAZYFREE);
585 update_page_reclaim_stat(lruvec, 1, 0, hpage_nr_pages(page));582 lru_note_cost(lruvec, 0, hpage_nr_pages(page));
586 }583 }
587}584}
588 585
mm/vmscan.c+22−22
@@ -1916,7 +1916,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 1915 unmodified lines
1916 bool file = is_file_lru(lru);1916 bool file = is_file_lru(lru);
1917 enum vm_event_item item;1917 enum vm_event_item item;
1918 struct pglist_data *pgdat = lruvec_pgdat(lruvec);1918 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1919 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1920 bool stalled = false;1919 bool stalled = false;
1921 1920
1922 while (unlikely(too_many_isolated(pgdat, file, sc))) {1921 while (unlikely(too_many_isolated(pgdat, file, sc))) {
@@ -1940,7 +1939,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 17 unmodified lines
1940 &nr_scanned, sc, lru);1939 &nr_scanned, sc, lru);
1941 1940
1942 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);1941 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
1943 reclaim_stat->recent_scanned[file] += nr_taken;
1944 item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;1942 item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
1945 if (!cgroup_reclaim(sc))1943 if (!cgroup_reclaim(sc))
1946 __count_vm_events(item, nr_scanned);1944 __count_vm_events(item, nr_scanned);
@@ -1960,8 +1958,12 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 13 unmodified lines
1960 move_pages_to_lru(lruvec, &page_list);1958 move_pages_to_lru(lruvec, &page_list);
1961 1959
1962 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);1960 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
1963 reclaim_stat->recent_rotated[0] += stat.nr_activate[0];1961 /*
1964 reclaim_stat->recent_rotated[1] += stat.nr_activate[1];1962 * Rotating pages costs CPU without actually
1963 * progressing toward the reclaim goal.
1964 */
1965 lru_note_cost(lruvec, 0, stat.nr_activate[0]);
1966 lru_note_cost(lruvec, 1, stat.nr_activate[1]);
1965 item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;1967 item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
1966 if (!cgroup_reclaim(sc))1968 if (!cgroup_reclaim(sc))
1967 __count_vm_events(item, nr_reclaimed);1969 __count_vm_events(item, nr_reclaimed);
@@ -2013,7 +2015,6 @@ static void shrink_active_list(unsigned long nr_to_scan, · 45 unmodified lines
2013 LIST_HEAD(l_active);2015 LIST_HEAD(l_active);
2014 LIST_HEAD(l_inactive);2016 LIST_HEAD(l_inactive);
2015 struct page *page;2017 struct page *page;
2016 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2017 unsigned nr_deactivate, nr_activate;2018 unsigned nr_deactivate, nr_activate;
2018 unsigned nr_rotated = 0;2019 unsigned nr_rotated = 0;
2019 int file = is_file_lru(lru);2020 int file = is_file_lru(lru);
@@ -2027,7 +2028,6 @@ static void shrink_active_list(unsigned long nr_to_scan, · 7 unmodified lines
2027 &nr_scanned, sc, lru);2028 &nr_scanned, sc, lru);
2028 2029
2029 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);2030 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2030 reclaim_stat->recent_scanned[file] += nr_taken;
2031 2031
2032 __count_vm_events(PGREFILL, nr_scanned);2032 __count_vm_events(PGREFILL, nr_scanned);
2033 __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);2033 __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
@@ -2085,7 +2085,7 @@ static void shrink_active_list(unsigned long nr_to_scan, · 51 unmodified lines
2085 * helps balance scan pressure between file and anonymous pages in2085 * helps balance scan pressure between file and anonymous pages in
2086 * get_scan_count.2086 * get_scan_count.
2087 */2087 */
2088 reclaim_stat->recent_rotated[file] += nr_rotated;2088 lru_note_cost(lruvec, file, nr_rotated);
2089 2089
2090 nr_activate = move_pages_to_lru(lruvec, &l_active);2090 nr_activate = move_pages_to_lru(lruvec, &l_active);
2091 nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);2091 nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);
@@ -2242,13 +2242,13 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, · 150 unmodified lines
2242{2242{
2243 struct mem_cgroup *memcg = lruvec_memcg(lruvec);2243 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
2244 int swappiness = mem_cgroup_swappiness(memcg);2244 int swappiness = mem_cgroup_swappiness(memcg);
2245 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2246 u64 fraction[2];2245 u64 fraction[2];
2247 u64 denominator = 0; /* gcc */2246 u64 denominator = 0; /* gcc */
2248 struct pglist_data *pgdat = lruvec_pgdat(lruvec);2247 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2249 unsigned long anon_prio, file_prio;2248 unsigned long anon_prio, file_prio;
2250 enum scan_balance scan_balance;2249 enum scan_balance scan_balance;
2251 unsigned long anon, file;2250 unsigned long anon, file;
2251 unsigned long totalcost;
2252 unsigned long ap, fp;2252 unsigned long ap, fp;
2253 enum lru_list lru;2253 enum lru_list lru;
2254 2254
@@ -2324,26 +2324,26 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, · 69 unmodified lines
2324 lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);2324 lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);
2325 2325
2326 spin_lock_irq(&pgdat->lru_lock);2326 spin_lock_irq(&pgdat->lru_lock);
2327 if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {2327 totalcost = lruvec->anon_cost + lruvec->file_cost;
2328 reclaim_stat->recent_scanned[0] /= 2;2328 if (unlikely(totalcost > (anon + file) / 4)) {
2329 reclaim_stat->recent_rotated[0] /= 2;2329 lruvec->anon_cost /= 2;
2330 }2330 lruvec->file_cost /= 2;
2331 2331 totalcost /= 2;
2332 if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
2333 reclaim_stat->recent_scanned[1] /= 2;
2334 reclaim_stat->recent_rotated[1] /= 2;
2335 }2332 }
2336 2333
2337 /*2334 /*
2338 * The amount of pressure on anon vs file pages is inversely2335 * The amount of pressure on anon vs file pages is inversely
2339 * proportional to the fraction of recently scanned pages on2336 * proportional to the assumed cost of reclaiming each list,
2340 * each list that were recently referenced and in active use.2337 * as determined by the share of pages that are likely going
2338 * to refault or rotate on each list (recently referenced),
2339 * times the relative IO cost of bringing back a swapped out
2340 * anonymous page vs reloading a filesystem page (swappiness).
2341 */2341 */
2342 ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);2342 ap = anon_prio * (totalcost + 1);
2343 ap /= reclaim_stat->recent_rotated[0] + 1;2343 ap /= lruvec->anon_cost + 1;
2344 2344
2345 fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);2345 fp = file_prio * (totalcost + 1);
2346 fp /= reclaim_stat->recent_rotated[1] + 1;2346 fp /= lruvec->file_cost + 1;
2347 spin_unlock_irq(&pgdat->lru_lock);2347 spin_unlock_irq(&pgdat->lru_lock);
2348 2348
2349 fraction[0] = ap;2349 fraction[0] = ap;