History

mm: memcontrol: make cgroup stats and events query API explicitly local

Patch series "mm: memcontrol: memory.stat cost & correctness". The cgroup memory.stat file holds recursive statistics for the entire subtree. The current implementation does this tree walk on-demand whenever the file is read. This is giving us problems in production. 1. The cost of aggregating the statistics on-demand is high. A lot of system service cgroups are mostly idle and their stats don't change between reads, yet we always have to check them. There are also always some lazily-dying cgroups sitting around that are pinned by a handful of remaining page cache; the same applies to them. In an application that periodically monitors memory.stat in our fleet, we have seen the aggregation consume up to 5% CPU time. 2. When cgroups die and disappear from the cgroup tree, so do their accumulated vm events. The result is that the event counters at higher-level cgroups can go backwards and confuse some of our automation, let alone people looking at the graphs over time. To address both issues, this patch series changes the stat implementation to spill counts upwards when the counters change. The upward spilling is batched using the existing per-cpu cache. In a sparse file stress test with 5 level cgroup nesting, the additional cost of the flushing was negligible (a little under 1% of CPU at 100% CPU utilization, compared to the 5% of reading memory.stat during regular operation). This patch (of 4): memcg_page_state(), lruvec_page_state(), memcg_sum_events() are currently returning the state of the local memcg or lruvec, not the recursive state. In practice there is a demand for both versions, although the callers that want the recursive counts currently sum them up by hand. Per default, cgroups are considered recursive entities and generally we expect more users of the recursive counters, with the local counts being special cases. To reflect that in the name, add a _local suffix to the current implementations. The following patch will re-incarnate these functions with recursive semantics, but with an O(1) implementation. [hannes@cmpxchg.org: fix bisection hole] Link: http://lkml.kernel.org/r/20190417160347.GC23013@cmpxchg.org Link: http://lkml.kernel.org/r/20190412151507.2769-2-hannes@cmpxchg.org Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reviewed-by: Shakeel Butt <shakeelb@google.com> Reviewed-by: Roman Gushchin <guro@fb.com> Cc: Michal Hocko <mhocko@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Johannes Weiner committed 7 years ago

Commit 205b20c1 parent 6a024334 files · +36 −33Browse files

Unified · Split

4 files
  1. include/linux/memcontrol.h +8 −8
  2. mm/memcontrol.c +21 −19
  3. mm/vmscan.c +3 −3
  4. mm/workingset.c +4 −3
include/linux/memcontrol.h+8−8
@@ -554,8 +554,8 @@ void unlock_page_memcg(struct page *page); · 553 unmodified lines
554 * idx can be of type enum memcg_stat_item or node_stat_item.554 * idx can be of type enum memcg_stat_item or node_stat_item.
555 * Keep in sync with memcg_exact_page_state().555 * Keep in sync with memcg_exact_page_state().
556 */556 */
557static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,557static inline unsigned long memcg_page_state_local(struct mem_cgroup *memcg,
558 int idx)558 int idx)
559{559{
560 long x = atomic_long_read(&memcg->vmstats[idx]);560 long x = atomic_long_read(&memcg->vmstats[idx]);
561#ifdef CONFIG_SMP561#ifdef CONFIG_SMP
@@ -624,8 +624,8 @@ static inline void mod_memcg_page_state(struct page *page, · 62 unmodified lines
624 mod_memcg_state(page->mem_cgroup, idx, val);624 mod_memcg_state(page->mem_cgroup, idx, val);
625}625}
626 626
627static inline unsigned long lruvec_page_state(struct lruvec *lruvec,627static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
628 enum node_stat_item idx)628 enum node_stat_item idx)
629{629{
630 struct mem_cgroup_per_node *pn;630 struct mem_cgroup_per_node *pn;
631 long x;631 long x;
@@ -1011,8 +1011,8 @@ static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) · 379 unmodified lines
1011{1011{
1012}1012}
1013 1013
1014static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,1014static inline unsigned long memcg_page_state_local(struct mem_cgroup *memcg,
1015 int idx)1015 int idx)
1016{1016{
1017 return 0;1017 return 0;
1018}1018}
@@ -1041,8 +1041,8 @@ static inline void mod_memcg_page_state(struct page *page, · 22 unmodified lines
1041{1041{
1042}1042}
1043 1043
1044static inline unsigned long lruvec_page_state(struct lruvec *lruvec,1044static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
1045 enum node_stat_item idx)1045 enum node_stat_item idx)
1046{1046{
1047 return node_page_state(lruvec_pgdat(lruvec), idx);1047 return node_page_state(lruvec_pgdat(lruvec), idx);
1048}1048}
mm/memcontrol.c+21−19
@@ -687,8 +687,8 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) · 686 unmodified lines
687 return mz;687 return mz;
688}688}
689 689
690static unsigned long memcg_sum_events(struct mem_cgroup *memcg,690static unsigned long memcg_events_local(struct mem_cgroup *memcg,
691 int event)691 int event)
692{692{
693 return atomic_long_read(&memcg->vmevents[event]);693 return atomic_long_read(&memcg->vmevents[event]);
694}694}
@@ -1325,12 +1325,14 @@ void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) · 630 unmodified lines
1325 if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)1325 if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
1326 continue;1326 continue;
1327 pr_cont(" %s:%luKB", memcg1_stat_names[i],1327 pr_cont(" %s:%luKB", memcg1_stat_names[i],
1328 K(memcg_page_state(iter, memcg1_stats[i])));1328 K(memcg_page_state_local(iter,
1329 memcg1_stats[i])));
1329 }1330 }
1330 1331
1331 for (i = 0; i < NR_LRU_LISTS; i++)1332 for (i = 0; i < NR_LRU_LISTS; i++)
1332 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],1333 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1333 K(memcg_page_state(iter, NR_LRU_BASE + i)));1334 K(memcg_page_state_local(iter,
1335 NR_LRU_BASE + i)));
1334 1336
1335 pr_cont("\n");1337 pr_cont("\n");
1336 }1338 }
@@ -1396,13 +1398,13 @@ static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg, · 59 unmodified lines
1396{1398{
1397 struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);1399 struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
1398 1400
1399 if (lruvec_page_state(lruvec, NR_INACTIVE_FILE) ||1401 if (lruvec_page_state_local(lruvec, NR_INACTIVE_FILE) ||
1400 lruvec_page_state(lruvec, NR_ACTIVE_FILE))1402 lruvec_page_state_local(lruvec, NR_ACTIVE_FILE))
1401 return true;1403 return true;
1402 if (noswap || !total_swap_pages)1404 if (noswap || !total_swap_pages)
1403 return false;1405 return false;
1404 if (lruvec_page_state(lruvec, NR_INACTIVE_ANON) ||1406 if (lruvec_page_state_local(lruvec, NR_INACTIVE_ANON) ||
1405 lruvec_page_state(lruvec, NR_ACTIVE_ANON))1407 lruvec_page_state_local(lruvec, NR_ACTIVE_ANON))
1406 return true;1408 return true;
1407 return false;1409 return false;
1408 1410
@@ -2961,16 +2963,16 @@ static void accumulate_vmstats(struct mem_cgroup *memcg, · 1552 unmodified lines
2961 2963
2962 for_each_mem_cgroup_tree(mi, memcg) {2964 for_each_mem_cgroup_tree(mi, memcg) {
2963 for (i = 0; i < acc->vmstats_size; i++)2965 for (i = 0; i < acc->vmstats_size; i++)
2964 acc->vmstats[i] += memcg_page_state(mi,2966 acc->vmstats[i] += memcg_page_state_local(mi,
2965 acc->vmstats_array ? acc->vmstats_array[i] : i);2967 acc->vmstats_array ? acc->vmstats_array[i] : i);
2966 2968
2967 for (i = 0; i < acc->vmevents_size; i++)2969 for (i = 0; i < acc->vmevents_size; i++)
2968 acc->vmevents[i] += memcg_sum_events(mi,2970 acc->vmevents[i] += memcg_events_local(mi,
2969 acc->vmevents_array2971 acc->vmevents_array
2970 ? acc->vmevents_array[i] : i);2972 ? acc->vmevents_array[i] : i);
2971 2973
2972 for (i = 0; i < NR_LRU_LISTS; i++)2974 for (i = 0; i < NR_LRU_LISTS; i++)
2973 acc->lru_pages[i] += memcg_page_state(mi,2975 acc->lru_pages[i] += memcg_page_state_local(mi,
2974 NR_LRU_BASE + i);2976 NR_LRU_BASE + i);
2975 }2977 }
2976}2978}
@@ -2983,10 +2985,10 @@ static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) · 6 unmodified lines
2983 struct mem_cgroup *iter;2985 struct mem_cgroup *iter;
2984 2986
2985 for_each_mem_cgroup_tree(iter, memcg) {2987 for_each_mem_cgroup_tree(iter, memcg) {
2986 val += memcg_page_state(iter, MEMCG_CACHE);2988 val += memcg_page_state_local(iter, MEMCG_CACHE);
2987 val += memcg_page_state(iter, MEMCG_RSS);2989 val += memcg_page_state_local(iter, MEMCG_RSS);
2988 if (swap)2990 if (swap)
2989 val += memcg_page_state(iter, MEMCG_SWAP);2991 val += memcg_page_state_local(iter, MEMCG_SWAP);
2990 }2992 }
2991 } else {2993 } else {
2992 if (!swap)2994 if (!swap)
@@ -3328,7 +3330,7 @@ static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, · 335 unmodified lines
3328 for_each_lru(lru) {3330 for_each_lru(lru) {
3329 if (!(BIT(lru) & lru_mask))3331 if (!(BIT(lru) & lru_mask))
3330 continue;3332 continue;
3331 nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);3333 nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
3332 }3334 }
3333 return nr;3335 return nr;
3334}3336}
@@ -3342,7 +3344,7 @@ static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, · 7 unmodified lines
3342 for_each_lru(lru) {3344 for_each_lru(lru) {
3343 if (!(BIT(lru) & lru_mask))3345 if (!(BIT(lru) & lru_mask))
3344 continue;3346 continue;
3345 nr += memcg_page_state(memcg, NR_LRU_BASE + lru);3347 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
3346 }3348 }
3347 return nr;3349 return nr;
3348}3350}
@@ -3427,17 +3429,17 @@ static int memcg_stat_show(struct seq_file *m, void *v) · 78 unmodified lines
3427 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())3429 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3428 continue;3430 continue;
3429 seq_printf(m, "%s %lu\n", memcg1_stat_names[i],3431 seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3430 memcg_page_state(memcg, memcg1_stats[i]) *3432 memcg_page_state_local(memcg, memcg1_stats[i]) *
3431 PAGE_SIZE);3433 PAGE_SIZE);
3432 }3434 }
3433 3435
3434 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)3436 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3435 seq_printf(m, "%s %lu\n", memcg1_event_names[i],3437 seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3436 memcg_sum_events(memcg, memcg1_events[i]));3438 memcg_events_local(memcg, memcg1_events[i]));
3437 3439
3438 for (i = 0; i < NR_LRU_LISTS; i++)3440 for (i = 0; i < NR_LRU_LISTS; i++)
3439 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],3441 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3440 memcg_page_state(memcg, NR_LRU_BASE + i) *3442 memcg_page_state_local(memcg, NR_LRU_BASE + i) *
3441 PAGE_SIZE);3443 PAGE_SIZE);
3442 3444
3443 /* Hierarchical information */3445 /* Hierarchical information */
mm/vmscan.c+3−3
@@ -346,7 +346,7 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone · 345 unmodified lines
346 int zid;346 int zid;
347 347
348 if (!mem_cgroup_disabled())348 if (!mem_cgroup_disabled())
349 lru_size = lruvec_page_state(lruvec, NR_LRU_BASE + lru);349 lru_size = lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
350 else350 else
351 lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);351 lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
352 352
@@ -2150,7 +2150,7 @@ static bool inactive_list_is_low(struct lruvec *lruvec, bool file, · 1797 unmodified lines
2150 * is being established. Disable active list protection to get2150 * is being established. Disable active list protection to get
2151 * rid of the stale workingset quickly.2151 * rid of the stale workingset quickly.
2152 */2152 */
2153 refaults = lruvec_page_state(lruvec, WORKINGSET_ACTIVATE);2153 refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE);
2154 if (file && actual_reclaim && lruvec->refaults != refaults) {2154 if (file && actual_reclaim && lruvec->refaults != refaults) {
2155 inactive_ratio = 0;2155 inactive_ratio = 0;
2156 } else {2156 } else {
@@ -2912,7 +2912,7 @@ static void snapshot_refaults(struct mem_cgroup *root_memcg, pg_data_t *pgdat) · 755 unmodified lines
2912 struct lruvec *lruvec;2912 struct lruvec *lruvec;
2913 2913
2914 lruvec = mem_cgroup_lruvec(pgdat, memcg);2914 lruvec = mem_cgroup_lruvec(pgdat, memcg);
2915 refaults = lruvec_page_state(lruvec, WORKINGSET_ACTIVATE);2915 refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE);
2916 lruvec->refaults = refaults;2916 lruvec->refaults = refaults;
2917 } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));2917 } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
2918}2918}
mm/workingset.c+4−3
@@ -430,9 +430,10 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 429 unmodified lines
430 430
431 lruvec = mem_cgroup_lruvec(NODE_DATA(sc->nid), sc->memcg);431 lruvec = mem_cgroup_lruvec(NODE_DATA(sc->nid), sc->memcg);
432 for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)432 for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)
433 pages += lruvec_page_state(lruvec, NR_LRU_BASE + i);433 pages += lruvec_page_state_local(lruvec,
434 pages += lruvec_page_state(lruvec, NR_SLAB_RECLAIMABLE);434 NR_LRU_BASE + i);
435 pages += lruvec_page_state(lruvec, NR_SLAB_UNRECLAIMABLE);435 pages += lruvec_page_state_local(lruvec, NR_SLAB_RECLAIMABLE);
436 pages += lruvec_page_state_local(lruvec, NR_SLAB_UNRECLAIMABLE);
436 } else437 } else
437#endif438#endif
438 pages = node_present_pages(sc->nid);439 pages = node_present_pages(sc->nid);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t