History

mm: clean up and clarify lruvec lookup procedure

There is a per-memcg lruvec and a NUMA node lruvec. Which one is being used is somewhat confusing right now, and it's easy to make mistakes - especially when it comes to global reclaim. How it works: when memory cgroups are enabled, we always use the root_mem_cgroup's per-node lruvecs. When memory cgroups are not compiled in or disabled at runtime, we use pgdat->lruvec. Document that in a comment. Due to the way the reclaim code is generalized, all lookups use the mem_cgroup_lruvec() helper function, and nobody should have to find the right lruvec manually right now. But to avoid future mistakes, rename the pgdat->lruvec member to pgdat->__lruvec and delete the convenience wrapper that suggests it's a commonly accessed member. While in this area, swap the mem_cgroup_lruvec() argument order. The name suggests a memcg operation, yet it takes a pgdat first and a memcg second. I have to double take every time I call this. Fix that.

Johannes Weiner committed 6 years ago

Commit 867e5e11 parent de3b0157 files · +35 −34Browse files

Unified · Split

7 files
  1. include/linux/memcontrol.h +12 −12
  2. include/linux/mmzone.h +8 −7
  3. mm/memcontrol.c +5 −5
  4. mm/page_alloc.c +1 −1
  5. mm/slab.h +2 −2
  6. mm/vmscan.c +3 −3
  7. mm/workingset.c +4 −4
include/linux/memcontrol.h+12−12
@@ -385,21 +385,21 @@ mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid) · 384 unmodified lines
385385 }
386386
387387 /**
388− * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
389− * @node: node of the wanted lruvec
388+ * mem_cgroup_lruvec - get the lru list vector for a memcg & node
390389 * @memcg: memcg of the wanted lruvec
391390 *
392− * Returns the lru list vector holding pages for a given @node or a given
393− * @memcg. This can be the node lruvec, if the memory controller is disabled.
391+ * Returns the lru list vector holding pages for a given @memcg &
392+ * @node combination. This can be the node lruvec, if the memory
393+ * controller is disabled.
394394 */
395−static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
396− struct mem_cgroup *memcg)
395+static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
396+ struct pglist_data *pgdat)
397397 {
398398 struct mem_cgroup_per_node *mz;
399399 struct lruvec *lruvec;
400400
401401 if (mem_cgroup_disabled()) {
402− lruvec = node_lruvec(pgdat);
402+ lruvec = &pgdat->__lruvec;
403403 goto out;
404404 }
405405
@@ -718,7 +718,7 @@ static inline void __mod_lruvec_page_state(struct page *page, · 312 unmodified lines
718718 return;
719719 }
720720
721− lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
721+ lruvec = mem_cgroup_lruvec(page->mem_cgroup, pgdat);
722722 __mod_lruvec_state(lruvec, idx, val);
723723 }
724724
@@ -889,16 +889,16 @@ static inline void mem_cgroup_migrate(struct page *old, struct page *new) · 164 unmodified lines
889889 {
890890 }
891891
892−static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
893− struct mem_cgroup *memcg)
892+static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
893+ struct pglist_data *pgdat)
894894 {
895− return node_lruvec(pgdat);
895+ return &pgdat->__lruvec;
896896 }
897897
898898 static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
899899 struct pglist_data *pgdat)
900900 {
901− return &pgdat->lruvec;
901+ return &pgdat->__lruvec;
902902 }
903903
904904 static inline bool mm_match_cgroup(struct mm_struct *mm,
include/linux/mmzone.h+8−7
@@ -777,7 +777,13 @@ typedef struct pglist_data { · 776 unmodified lines
777777 #endif
778778
779779 /* Fields commonly accessed by the page reclaim scanner */
780− struct lruvec lruvec;
780+
781+ /*
782+ * NOTE: THIS IS UNUSED IF MEMCG IS ENABLED.
783+ *
784+ * Use mem_cgroup_lruvec() to look up lruvecs.
785+ */
786+ struct lruvec __lruvec;
781787
782788 unsigned long flags;
783789
@@ -800,11 +806,6 @@ typedef struct pglist_data { · 16 unmodified lines
800806 #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
801807 #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
802808
803−static inline struct lruvec *node_lruvec(struct pglist_data *pgdat)
804−{
805− return &pgdat->lruvec;
806−}
807−
808809 static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
809810 {
810811 return pgdat->node_start_pfn + pgdat->node_spanned_pages;
@@ -842,7 +843,7 @@ static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) · 31 unmodified lines
842843 #ifdef CONFIG_MEMCG
843844 return lruvec->pgdat;
844845 #else
845− return container_of(lruvec, struct pglist_data, lruvec);
846+ return container_of(lruvec, struct pglist_data, __lruvec);
846847 #endif
847848 }
848849
mm/memcontrol.c+5−5
@@ -777,7 +777,7 @@ void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val) · 776 unmodified lines
777777 if (!memcg || memcg == root_mem_cgroup) {
778778 __mod_node_page_state(pgdat, idx, val);
779779 } else {
780− lruvec = mem_cgroup_lruvec(pgdat, memcg);
780+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
781781 __mod_lruvec_state(lruvec, idx, val);
782782 }
783783 rcu_read_unlock();
@@ -1221,7 +1221,7 @@ struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgd · 437 unmodified lines
12211221 struct lruvec *lruvec;
12221222
12231223 if (mem_cgroup_disabled()) {
1224− lruvec = &pgdat->lruvec;
1224+ lruvec = &pgdat->__lruvec;
12251225 goto out;
12261226 }
12271227
@@ -3634,7 +3634,7 @@ static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, · 2406 unmodified lines
36343634 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
36353635 int nid, unsigned int lru_mask)
36363636 {
3637− struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
3637+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
36383638 unsigned long nr = 0;
36393639 enum lru_list lru;
36403640
@@ -5338,8 +5338,8 @@ static int mem_cgroup_move_account(struct page *page, · 1697 unmodified lines
53385338 anon = PageAnon(page);
53395339
53405340 pgdat = page_pgdat(page);
5341− from_vec = mem_cgroup_lruvec(pgdat, from);
5342− to_vec = mem_cgroup_lruvec(pgdat, to);
5341+ from_vec = mem_cgroup_lruvec(from, pgdat);
5342+ to_vec = mem_cgroup_lruvec(to, pgdat);
53435343
53445344 spin_lock_irqsave(&from->move_lock, flags);
53455345
mm/page_alloc.c+1−1
@@ -6713,7 +6713,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) · 6712 unmodified lines
67136713
67146714 pgdat_page_ext_init(pgdat);
67156715 spin_lock_init(&pgdat->lru_lock);
6716− lruvec_init(node_lruvec(pgdat));
6716+ lruvec_init(&pgdat->__lruvec);
67176717 }
67186718
67196719 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
mm/slab.h+2−2
@@ -369,7 +369,7 @@ static __always_inline int memcg_charge_slab(struct page *page, · 368 unmodified lines
369369 if (ret)
370370 goto out;
371371
372− lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
372+ lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
373373 mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order);
374374
375375 /* transer try_charge() page references to kmem_cache */
@@ -393,7 +393,7 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order, · 17 unmodified lines
393393 rcu_read_lock();
394394 memcg = READ_ONCE(s->memcg_params.memcg);
395395 if (likely(!mem_cgroup_is_root(memcg))) {
396− lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
396+ lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
397397 mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order));
398398 memcg_kmem_uncharge_memcg(page, order, memcg);
399399 } else {
mm/vmscan.c+3−3
@@ -2545,7 +2545,7 @@ out: · 2544 unmodified lines
25452545 static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg,
25462546 struct scan_control *sc)
25472547 {
2548− struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
2548+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
25492549 unsigned long nr[NR_LRU_LISTS];
25502550 unsigned long targets[NR_LRU_LISTS];
25512551 unsigned long nr_to_scan;
@@ -3023,7 +3023,7 @@ static void snapshot_refaults(struct mem_cgroup *root_memcg, pg_data_t *pgdat) · 471 unmodified lines
30233023 unsigned long refaults;
30243024 struct lruvec *lruvec;
30253025
3026− lruvec = mem_cgroup_lruvec(pgdat, memcg);
3026+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
30273027 refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE);
30283028 lruvec->refaults = refaults;
30293029 } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
@@ -3379,7 +3379,7 @@ static void age_active_anon(struct pglist_data *pgdat, · 349 unmodified lines
33793379
33803380 memcg = mem_cgroup_iter(NULL, NULL, NULL);
33813381 do {
3382− struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
3382+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
33833383
33843384 if (inactive_list_is_low(lruvec, false, sc, true))
33853385 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
mm/workingset.c+4−4
@@ -233,7 +233,7 @@ void *workingset_eviction(struct page *page) · 232 unmodified lines
233233 VM_BUG_ON_PAGE(page_count(page), page);
234234 VM_BUG_ON_PAGE(!PageLocked(page), page);
235235
236− lruvec = mem_cgroup_lruvec(pgdat, memcg);
236+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
237237 eviction = atomic_long_inc_return(&lruvec->inactive_age);
238238 return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));
239239 }
@@ -280,7 +280,7 @@ void workingset_refault(struct page *page, void *shadow) · 40 unmodified lines
280280 memcg = mem_cgroup_from_id(memcgid);
281281 if (!mem_cgroup_disabled() && !memcg)
282282 goto out;
283− lruvec = mem_cgroup_lruvec(pgdat, memcg);
283+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
284284 refault = atomic_long_read(&lruvec->inactive_age);
285285 active_file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES);
286286
@@ -345,7 +345,7 @@ void workingset_activation(struct page *page) · 58 unmodified lines
345345 memcg = page_memcg_rcu(page);
346346 if (!mem_cgroup_disabled() && !memcg)
347347 goto out;
348− lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
348+ lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
349349 atomic_long_inc(&lruvec->inactive_age);
350350 out:
351351 rcu_read_unlock();
@@ -426,7 +426,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 74 unmodified lines
426426 struct lruvec *lruvec;
427427 int i;
428428
429− lruvec = mem_cgroup_lruvec(NODE_DATA(sc->nid), sc->memcg);
429+ lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid));
430430 for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)
431431 pages += lruvec_page_state_local(lruvec,
432432 NR_LRU_BASE + i);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t