There is a per-memcg lruvec and a NUMA node lruvec. Which one is being used is somewhat confusing right now, and it's easy to make mistakes - especially when it comes to global reclaim. How it works: when memory cgroups are enabled, we always use the root_mem_cgroup's per-node lruvecs. When memory cgroups are not compiled in or disabled at runtime, we use pgdat->lruvec. Document that in a comment. Due to the way the reclaim code is generalized, all lookups use the mem_cgroup_lruvec() helper function, and nobody should have to find the right lruvec manually right now. But to avoid future mistakes, rename the pgdat->lruvec member to pgdat->__lruvec and delete the convenience wrapper that suggests it's a commonly accessed member. While in this area, swap the mem_cgroup_lruvec() argument order. The name suggests a memcg operation, yet it takes a pgdat first and a memcg second. I have to double take every time I call this. Fix that.
Unified · Split
| @@ -385,21 +385,21 @@ mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid) · 384 unmodified lines | |||
| 385 | } | 385 | } |
| 386 | 386 | ||
| 387 | /** | 387 | /** |
| 388 | * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone | 388 | * mem_cgroup_lruvec - get the lru list vector for a memcg & node |
| 389 | * @node: node of the wanted lruvec | ||
| 390 | * @memcg: memcg of the wanted lruvec | 389 | * @memcg: memcg of the wanted lruvec |
| 391 | * | 390 | * |
| 392 | * Returns the lru list vector holding pages for a given @node or a given | 391 | * Returns the lru list vector holding pages for a given @memcg & |
| 393 | * @memcg. This can be the node lruvec, if the memory controller is disabled. | 392 | * @node combination. This can be the node lruvec, if the memory |
| 393 | * controller is disabled. | ||
| 394 | */ | 394 | */ |
| 395 | static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat, | 395 | static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, |
| 396 | struct mem_cgroup *memcg) | 396 | struct pglist_data *pgdat) |
| 397 | { | 397 | { |
| 398 | struct mem_cgroup_per_node *mz; | 398 | struct mem_cgroup_per_node *mz; |
| 399 | struct lruvec *lruvec; | 399 | struct lruvec *lruvec; |
| 400 | 400 | ||
| 401 | if (mem_cgroup_disabled()) { | 401 | if (mem_cgroup_disabled()) { |
| 402 | lruvec = node_lruvec(pgdat); | 402 | lruvec = &pgdat->__lruvec; |
| 403 | goto out; | 403 | goto out; |
| 404 | } | 404 | } |
| 405 | 405 | ||
| @@ -718,7 +718,7 @@ static inline void __mod_lruvec_page_state(struct page *page, · 312 unmodified lines | |||
| 718 | return; | 718 | return; |
| 719 | } | 719 | } |
| 720 | 720 | ||
| 721 | lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup); | 721 | lruvec = mem_cgroup_lruvec(page->mem_cgroup, pgdat); |
| 722 | __mod_lruvec_state(lruvec, idx, val); | 722 | __mod_lruvec_state(lruvec, idx, val); |
| 723 | } | 723 | } |
| 724 | 724 | ||
| @@ -889,16 +889,16 @@ static inline void mem_cgroup_migrate(struct page *old, struct page *new) · 164 unmodified lines | |||
| 889 | { | 889 | { |
| 890 | } | 890 | } |
| 891 | 891 | ||
| 892 | static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat, | 892 | static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, |
| 893 | struct mem_cgroup *memcg) | 893 | struct pglist_data *pgdat) |
| 894 | { | 894 | { |
| 895 | return node_lruvec(pgdat); | 895 | return &pgdat->__lruvec; |
| 896 | } | 896 | } |
| 897 | 897 | ||
| 898 | static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page, | 898 | static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page, |
| 899 | struct pglist_data *pgdat) | 899 | struct pglist_data *pgdat) |
| 900 | { | 900 | { |
| 901 | return &pgdat->lruvec; | 901 | return &pgdat->__lruvec; |
| 902 | } | 902 | } |
| 903 | 903 | ||
| 904 | static inline bool mm_match_cgroup(struct mm_struct *mm, | 904 | static inline bool mm_match_cgroup(struct mm_struct *mm, |
| @@ -777,7 +777,13 @@ typedef struct pglist_data { · 776 unmodified lines | |||
| 777 | #endif | 777 | #endif |
| 778 | 778 | ||
| 779 | /* Fields commonly accessed by the page reclaim scanner */ | 779 | /* Fields commonly accessed by the page reclaim scanner */ |
| 780 | struct lruvec lruvec; | 780 | |
| 781 | /* | ||
| 782 | * NOTE: THIS IS UNUSED IF MEMCG IS ENABLED. | ||
| 783 | * | ||
| 784 | * Use mem_cgroup_lruvec() to look up lruvecs. | ||
| 785 | */ | ||
| 786 | struct lruvec __lruvec; | ||
| 781 | 787 | ||
| 782 | unsigned long flags; | 788 | unsigned long flags; |
| 783 | 789 | ||
| @@ -800,11 +806,6 @@ typedef struct pglist_data { · 16 unmodified lines | |||
| 800 | #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) | 806 | #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) |
| 801 | #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid)) | 807 | #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid)) |
| 802 | 808 | ||
| 803 | static inline struct lruvec *node_lruvec(struct pglist_data *pgdat) | ||
| 804 | { | ||
| 805 | return &pgdat->lruvec; | ||
| 806 | } | ||
| 807 | |||
| 808 | static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) | 809 | static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) |
| 809 | { | 810 | { |
| 810 | return pgdat->node_start_pfn + pgdat->node_spanned_pages; | 811 | return pgdat->node_start_pfn + pgdat->node_spanned_pages; |
| @@ -842,7 +843,7 @@ static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) · 31 unmodified lines | |||
| 842 | #ifdef CONFIG_MEMCG | 843 | #ifdef CONFIG_MEMCG |
| 843 | return lruvec->pgdat; | 844 | return lruvec->pgdat; |
| 844 | #else | 845 | #else |
| 845 | return container_of(lruvec, struct pglist_data, lruvec); | 846 | return container_of(lruvec, struct pglist_data, __lruvec); |
| 846 | #endif | 847 | #endif |
| 847 | } | 848 | } |
| 848 | 849 | ||
| @@ -777,7 +777,7 @@ void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val) · 776 unmodified lines | |||
| 777 | if (!memcg || memcg == root_mem_cgroup) { | 777 | if (!memcg || memcg == root_mem_cgroup) { |
| 778 | __mod_node_page_state(pgdat, idx, val); | 778 | __mod_node_page_state(pgdat, idx, val); |
| 779 | } else { | 779 | } else { |
| 780 | lruvec = mem_cgroup_lruvec(pgdat, memcg); | 780 | lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 781 | __mod_lruvec_state(lruvec, idx, val); | 781 | __mod_lruvec_state(lruvec, idx, val); |
| 782 | } | 782 | } |
| 783 | rcu_read_unlock(); | 783 | rcu_read_unlock(); |
| @@ -1221,7 +1221,7 @@ struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgd · 437 unmodified lines | |||
| 1221 | struct lruvec *lruvec; | 1221 | struct lruvec *lruvec; |
| 1222 | 1222 | ||
| 1223 | if (mem_cgroup_disabled()) { | 1223 | if (mem_cgroup_disabled()) { |
| 1224 | lruvec = &pgdat->lruvec; | 1224 | lruvec = &pgdat->__lruvec; |
| 1225 | goto out; | 1225 | goto out; |
| 1226 | } | 1226 | } |
| 1227 | 1227 | ||
| @@ -3634,7 +3634,7 @@ static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, · 2406 unmodified lines | |||
| 3634 | static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, | 3634 | static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, |
| 3635 | int nid, unsigned int lru_mask) | 3635 | int nid, unsigned int lru_mask) |
| 3636 | { | 3636 | { |
| 3637 | struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg); | 3637 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); |
| 3638 | unsigned long nr = 0; | 3638 | unsigned long nr = 0; |
| 3639 | enum lru_list lru; | 3639 | enum lru_list lru; |
| 3640 | 3640 | ||
| @@ -5338,8 +5338,8 @@ static int mem_cgroup_move_account(struct page *page, · 1697 unmodified lines | |||
| 5338 | anon = PageAnon(page); | 5338 | anon = PageAnon(page); |
| 5339 | 5339 | ||
| 5340 | pgdat = page_pgdat(page); | 5340 | pgdat = page_pgdat(page); |
| 5341 | from_vec = mem_cgroup_lruvec(pgdat, from); | 5341 | from_vec = mem_cgroup_lruvec(from, pgdat); |
| 5342 | to_vec = mem_cgroup_lruvec(pgdat, to); | 5342 | to_vec = mem_cgroup_lruvec(to, pgdat); |
| 5343 | 5343 | ||
| 5344 | spin_lock_irqsave(&from->move_lock, flags); | 5344 | spin_lock_irqsave(&from->move_lock, flags); |
| 5345 | 5345 | ||
| @@ -6713,7 +6713,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) · 6712 unmodified lines | |||
| 6713 | 6713 | ||
| 6714 | pgdat_page_ext_init(pgdat); | 6714 | pgdat_page_ext_init(pgdat); |
| 6715 | spin_lock_init(&pgdat->lru_lock); | 6715 | spin_lock_init(&pgdat->lru_lock); |
| 6716 | lruvec_init(node_lruvec(pgdat)); | 6716 | lruvec_init(&pgdat->__lruvec); |
| 6717 | } | 6717 | } |
| 6718 | 6718 | ||
| 6719 | static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, | 6719 | static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, |
| @@ -369,7 +369,7 @@ static __always_inline int memcg_charge_slab(struct page *page, · 368 unmodified lines | |||
| 369 | if (ret) | 369 | if (ret) |
| 370 | goto out; | 370 | goto out; |
| 371 | 371 | ||
| 372 | lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg); | 372 | lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page)); |
| 373 | mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order); | 373 | mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order); |
| 374 | 374 | ||
| 375 | /* transer try_charge() page references to kmem_cache */ | 375 | /* transer try_charge() page references to kmem_cache */ |
| @@ -393,7 +393,7 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order, · 17 unmodified lines | |||
| 393 | rcu_read_lock(); | 393 | rcu_read_lock(); |
| 394 | memcg = READ_ONCE(s->memcg_params.memcg); | 394 | memcg = READ_ONCE(s->memcg_params.memcg); |
| 395 | if (likely(!mem_cgroup_is_root(memcg))) { | 395 | if (likely(!mem_cgroup_is_root(memcg))) { |
| 396 | lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg); | 396 | lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page)); |
| 397 | mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order)); | 397 | mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order)); |
| 398 | memcg_kmem_uncharge_memcg(page, order, memcg); | 398 | memcg_kmem_uncharge_memcg(page, order, memcg); |
| 399 | } else { | 399 | } else { |
| @@ -2545,7 +2545,7 @@ out: · 2544 unmodified lines | |||
| 2545 | static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg, | 2545 | static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg, |
| 2546 | struct scan_control *sc) | 2546 | struct scan_control *sc) |
| 2547 | { | 2547 | { |
| 2548 | struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg); | 2548 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 2549 | unsigned long nr[NR_LRU_LISTS]; | 2549 | unsigned long nr[NR_LRU_LISTS]; |
| 2550 | unsigned long targets[NR_LRU_LISTS]; | 2550 | unsigned long targets[NR_LRU_LISTS]; |
| 2551 | unsigned long nr_to_scan; | 2551 | unsigned long nr_to_scan; |
| @@ -3023,7 +3023,7 @@ static void snapshot_refaults(struct mem_cgroup *root_memcg, pg_data_t *pgdat) · 471 unmodified lines | |||
| 3023 | unsigned long refaults; | 3023 | unsigned long refaults; |
| 3024 | struct lruvec *lruvec; | 3024 | struct lruvec *lruvec; |
| 3025 | 3025 | ||
| 3026 | lruvec = mem_cgroup_lruvec(pgdat, memcg); | 3026 | lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 3027 | refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE); | 3027 | refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE); |
| 3028 | lruvec->refaults = refaults; | 3028 | lruvec->refaults = refaults; |
| 3029 | } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL))); | 3029 | } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL))); |
| @@ -3379,7 +3379,7 @@ static void age_active_anon(struct pglist_data *pgdat, · 349 unmodified lines | |||
| 3379 | 3379 | ||
| 3380 | memcg = mem_cgroup_iter(NULL, NULL, NULL); | 3380 | memcg = mem_cgroup_iter(NULL, NULL, NULL); |
| 3381 | do { | 3381 | do { |
| 3382 | struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg); | 3382 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 3383 | 3383 | ||
| 3384 | if (inactive_list_is_low(lruvec, false, sc, true)) | 3384 | if (inactive_list_is_low(lruvec, false, sc, true)) |
| 3385 | shrink_active_list(SWAP_CLUSTER_MAX, lruvec, | 3385 | shrink_active_list(SWAP_CLUSTER_MAX, lruvec, |
| @@ -233,7 +233,7 @@ void *workingset_eviction(struct page *page) · 232 unmodified lines | |||
| 233 | VM_BUG_ON_PAGE(page_count(page), page); | 233 | VM_BUG_ON_PAGE(page_count(page), page); |
| 234 | VM_BUG_ON_PAGE(!PageLocked(page), page); | 234 | VM_BUG_ON_PAGE(!PageLocked(page), page); |
| 235 | 235 | ||
| 236 | lruvec = mem_cgroup_lruvec(pgdat, memcg); | 236 | lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 237 | eviction = atomic_long_inc_return(&lruvec->inactive_age); | 237 | eviction = atomic_long_inc_return(&lruvec->inactive_age); |
| 238 | return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page)); | 238 | return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page)); |
| 239 | } | 239 | } |
| @@ -280,7 +280,7 @@ void workingset_refault(struct page *page, void *shadow) · 40 unmodified lines | |||
| 280 | memcg = mem_cgroup_from_id(memcgid); | 280 | memcg = mem_cgroup_from_id(memcgid); |
| 281 | if (!mem_cgroup_disabled() && !memcg) | 281 | if (!mem_cgroup_disabled() && !memcg) |
| 282 | goto out; | 282 | goto out; |
| 283 | lruvec = mem_cgroup_lruvec(pgdat, memcg); | 283 | lruvec = mem_cgroup_lruvec(memcg, pgdat); |
| 284 | refault = atomic_long_read(&lruvec->inactive_age); | 284 | refault = atomic_long_read(&lruvec->inactive_age); |
| 285 | active_file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES); | 285 | active_file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES); |
| 286 | 286 | ||
| @@ -345,7 +345,7 @@ void workingset_activation(struct page *page) · 58 unmodified lines | |||
| 345 | memcg = page_memcg_rcu(page); | 345 | memcg = page_memcg_rcu(page); |
| 346 | if (!mem_cgroup_disabled() && !memcg) | 346 | if (!mem_cgroup_disabled() && !memcg) |
| 347 | goto out; | 347 | goto out; |
| 348 | lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg); | 348 | lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page)); |
| 349 | atomic_long_inc(&lruvec->inactive_age); | 349 | atomic_long_inc(&lruvec->inactive_age); |
| 350 | out: | 350 | out: |
| 351 | rcu_read_unlock(); | 351 | rcu_read_unlock(); |
| @@ -426,7 +426,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 74 unmodified lines | |||
| 426 | struct lruvec *lruvec; | 426 | struct lruvec *lruvec; |
| 427 | int i; | 427 | int i; |
| 428 | 428 | ||
| 429 | lruvec = mem_cgroup_lruvec(NODE_DATA(sc->nid), sc->memcg); | 429 | lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid)); |
| 430 | for (pages = 0, i = 0; i < NR_LRU_LISTS; i++) | 430 | for (pages = 0, i = 0; i < NR_LRU_LISTS; i++) |
| 431 | pages += lruvec_page_state_local(lruvec, | 431 | pages += lruvec_page_state_local(lruvec, |
| 432 | NR_LRU_BASE + i); | 432 | NR_LRU_BASE + i); |