memcg_rstat_updated() uses the value of the state update to keep track of the magnitude of pending updates, so that we only do a stats flush when it's worth the work. Most values passed into memcg_rstat_updated() are in pages, however, a few of them are actually in bytes or KBs. To put this into perspective, a 512 byte slab allocation today would look the same as allocating 512 pages. This may result in premature flushes, which means unnecessary work and latency. Normalize all the state values passed into memcg_rstat_updated() to pages. Round up non-zero sub-page to 1 page, because memcg_rstat_updated() ignores 0 page updates.
Unified · Split
| @@ -763,6 +763,22 @@ unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) · 762 unmodified lines | |||
| 763 | return x; | 763 | return x; |
| 764 | } | 764 | } |
| 765 | 765 | ||
| 766 | static int memcg_page_state_unit(int item); | ||
| 767 | |||
| 768 | /* | ||
| 769 | * Normalize the value passed into memcg_rstat_updated() to be in pages. Round | ||
| 770 | * up non-zero sub-page updates to 1 page as zero page updates are ignored. | ||
| 771 | */ | ||
| 772 | static int memcg_state_val_in_pages(int idx, int val) | ||
| 773 | { | ||
| 774 | int unit = memcg_page_state_unit(idx); | ||
| 775 | |||
| 776 | if (!val || unit == PAGE_SIZE) | ||
| 777 | return val; | ||
| 778 | else | ||
| 779 | return max(val * unit / PAGE_SIZE, 1UL); | ||
| 780 | } | ||
| 781 | |||
| 766 | /** | 782 | /** |
| 767 | * __mod_memcg_state - update cgroup memory statistics | 783 | * __mod_memcg_state - update cgroup memory statistics |
| 768 | * @memcg: the memory cgroup | 784 | * @memcg: the memory cgroup |
| @@ -775,7 +791,7 @@ void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) · 6 unmodified lines | |||
| 775 | return; | 791 | return; |
| 776 | 792 | ||
| 777 | __this_cpu_add(memcg->vmstats_percpu->state[idx], val); | 793 | __this_cpu_add(memcg->vmstats_percpu->state[idx], val); |
| 778 | memcg_rstat_updated(memcg, val); | 794 | memcg_rstat_updated(memcg, memcg_state_val_in_pages(idx, val)); |
| 779 | } | 795 | } |
| 780 | 796 | ||
| 781 | /* idx can be of type enum memcg_stat_item or node_stat_item. */ | 797 | /* idx can be of type enum memcg_stat_item or node_stat_item. */ |
| @@ -826,7 +842,7 @@ void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, · 44 unmodified lines | |||
| 826 | /* Update lruvec */ | 842 | /* Update lruvec */ |
| 827 | __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val); | 843 | __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val); |
| 828 | 844 | ||
| 829 | memcg_rstat_updated(memcg, val); | 845 | memcg_rstat_updated(memcg, memcg_state_val_in_pages(idx, val)); |
| 830 | memcg_stats_unlock(); | 846 | memcg_stats_unlock(); |
| 831 | } | 847 | } |
| 832 | 848 | ||