History

mm: switch deferred split shrinker to list_lru

The deferred split queue handles cgroups in a suboptimal fashion. The queue is per-NUMA node or per-cgroup, not the intersection. That means on a cgrouped system, a node-restricted allocation entering reclaim can end up splitting large pages on other nodes: alloc/unmap deferred_split_folio() list_add_tail(memcg->split_queue) set_shrinker_bit(memcg, node, deferred_shrinker_id) for_each_zone_zonelist_nodemask(restricted_nodes) mem_cgroup_iter() shrink_slab(node, memcg) shrink_slab_memcg(node, memcg) if test_shrinker_bit(memcg, node, deferred_shrinker_id) deferred_split_scan() walks memcg->split_queue The shrinker bit adds an imperfect guard rail. As soon as the cgroup has a single large page on the node of interest, all large pages owned by that memcg, including those on other nodes, will be split. list_lru properly sets up per-node, per-cgroup lists. As a bonus, it streamlines a lot of the list operations and reclaim walks. It's used widely by other major shrinkers already. Convert the deferred split queue as well. The list_lru per-memcg heads are instantiated on demand when the first object of interest is allocated for a cgroup, by calling folio_memcg_alloc_deferred(). Add calls to where splittable pages are created: anon faults, swapin faults, khugepaged collapse. These calls create all possible node heads for the cgroup at once, so the migration code (between nodes) doesn't need any special care. [akpm@linux-foundation.org: fix build with CONFIG_TRANSPARENT_HUGEPAGE=n] Link: https://lore.kernel.org/202605281620.lc3rtkBm-lkp@intel.com [hannes@cmpxchg.org: fix cgroup.memory=nokmem handling] Link: https://lore.kernel.org/ah9PGv12mqai84ES@cmpxchg.org Link: https://lore.kernel.org/20260527204757.2544958-10-hannes@cmpxchg.org Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reported-by: Mikhail Zaslonko <zaslonko@linux.ibm.com> Tested-by: Mikhail Zaslonko <zaslonko@linux.ibm.com> Acked-by: Shakeel Butt <shakeel.butt@linux.dev> Reviewed-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org> Acked-by: Usama Arif <usama.arif@linux.dev> Reviewed-by: Kairui Song <kasong@tencent.com> Cc: Baolin Wang <baolin.wang@linux.alibaba.com> Cc: Barry Song <baohua@kernel.org> Cc: Dave Chinner <david@fromorbit.com> Cc: David Hildenbrand (Arm) <david@kernel.org> Cc: Dev Jain <dev.jain@arm.com> Cc: Lance Yang <lance.yang@linux.dev> Cc: Liam R. Howlett <liam@infradead.org> Cc: Michal Hocko <mhocko@kernel.org> Cc: Muchun Song <muchun.song@linux.dev> Cc: Nico Pache <npache@redhat.com> Cc: Roman Gushchin <roman.gushchin@linux.dev> Cc: Ryan Roberts <ryan.roberts@arm.com> Cc: Vasily Gorbik <gor@linux.ibm.com> Cc: Vlastimil Babka <vbabka@kernel.org> Cc: Zi Yan <ziy@nvidia.com> Cc: kernel test robot <lkp@intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Johannes Weiner committed 5 months ago

Commit fafaece1 parent 65180e910 files · +160 −288Browse files

Unified · Split

10 files
  1. include/linux/huge_mm.h +12 −5
  2. include/linux/memcontrol.h +0 −4
  3. include/linux/mmzone.h +0 −12
  4. mm/huge_memory.c +125 −242
  5. mm/internal.h +1 −1
  6. mm/khugepaged.c +5 −0
  7. mm/memcontrol.c +3 −9
  8. mm/memory.c +4 −0
  9. mm/mm_init.c +0 −15
  10. mm/swap_state.c +10 −0
include/linux/huge_mm.h+12−5
@@ -439,10 +439,10 @@ static inline int split_huge_page(struct page *page) · 438 unmodified lines
439439 {
440440 return split_huge_page_to_list_to_order(page, NULL, 0);
441441 }
442+
443+int folio_memcg_alloc_deferred(struct folio *folio);
444+
442445 void deferred_split_folio(struct folio *folio, bool partially_mapped);
443−#ifdef CONFIG_MEMCG
444−void reparent_deferred_split_queue(struct mem_cgroup *memcg);
445−#endif
446446
447447 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
448448 unsigned long address, bool freeze);
@@ -679,8 +679,15 @@ static inline int try_folio_split_to_order(struct folio *folio, · 230 unmodified lines
679679 return -EINVAL;
680680 }
681681
682−static inline void deferred_split_folio(struct folio *folio, bool partially_mapped) {}
683−static inline void reparent_deferred_split_queue(struct mem_cgroup *memcg) {}
682+static inline int folio_memcg_alloc_deferred(struct folio *folio)
683+{
684+ return 0;
685+}
686+
687+static inline void deferred_split_folio(struct folio *folio, bool partially_mapped)
688+{
689+}
690+
684691 #define split_huge_pmd(__vma, __pmd, __address) \
685692 do { } while (0)
686693
include/linux/memcontrol.h+0−4
@@ -278,10 +278,6 @@ struct mem_cgroup { · 277 unmodified lines
278278 struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
279279 #endif
280280
281−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
282− struct deferred_split deferred_split_queue;
283−#endif
284−
285281 #ifdef CONFIG_LRU_GEN_WALKS_MMU
286282 /* per-memcg mm_struct list */
287283 struct lru_gen_mm_list mm_list;
include/linux/mmzone.h+0−12
@@ -1431,14 +1431,6 @@ struct zonelist { · 1430 unmodified lines
14311431 */
14321432 extern struct page *mem_map;
14331433
1434−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1435−struct deferred_split {
1436− spinlock_t split_queue_lock;
1437− struct list_head split_queue;
1438− unsigned long split_queue_len;
1439−};
1440−#endif
1441−
14421434 #ifdef CONFIG_MEMORY_FAILURE
14431435 /*
14441436 * Per NUMA node memory failure handling statistics.
@@ -1564,10 +1556,6 @@ typedef struct pglist_data { · 119 unmodified lines
15641556 unsigned long first_deferred_pfn;
15651557 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
15661558
1567−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1568− struct deferred_split deferred_split_queue;
1569−#endif
1570−
15711559 #ifdef CONFIG_NUMA_BALANCING
15721560 /* start time in ms of current promote rate limit period */
15731561 unsigned int nbp_rl_start;
mm/huge_memory.c+125−242
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <linux/mmu_notifier.h>
1515 #include <linux/rmap.h>
1616 #include <linux/swap.h>
17+#include <linux/list_lru.h>
1718 #include <linux/shrinker.h>
1819 #include <linux/mm_inline.h>
1920 #include <linux/swapops.h>
@@ -67,6 +68,8 @@ unsigned long transparent_hugepage_flags __read_mostly = · 47 unmodified lines
6768 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
6869 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
6970
71+static struct lock_class_key deferred_split_key;
72+static struct list_lru deferred_split_lru;
7073 static struct shrinker *deferred_split_shrinker;
7174 static unsigned long deferred_split_count(struct shrinker *shrink,
7275 struct shrink_control *sc);
@@ -932,15 +935,28 @@ static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) · 859 unmodified lines
932935 }
933936 #endif /* CONFIG_SYSFS */
934937
938+int folio_memcg_alloc_deferred(struct folio *folio)
939+{
940+ if (mem_cgroup_disabled())
941+ return 0;
942+ return folio_memcg_list_lru_alloc(folio, &deferred_split_lru, GFP_KERNEL);
943+}
944+
935945 static int __init thp_shrinker_init(void)
936946 {
937947 deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE |
938− SHRINKER_MEMCG_AWARE |
939− SHRINKER_NONSLAB,
948+ SHRINKER_MEMCG_AWARE,
940949 "thp-deferred_split");
941950 if (!deferred_split_shrinker)
942951 return -ENOMEM;
943952
953+ if (list_lru_init_memcg_key(&deferred_split_lru,
954+ deferred_split_shrinker,
955+ &deferred_split_key)) {
956+ shrinker_free(deferred_split_shrinker);
957+ return -ENOMEM;
958+ }
959+
944960 deferred_split_shrinker->count_objects = deferred_split_count;
945961 deferred_split_shrinker->scan_objects = deferred_split_scan;
946962 shrinker_register(deferred_split_shrinker);
@@ -962,6 +978,7 @@ static int __init thp_shrinker_init(void) · 15 unmodified lines
962978 huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero");
963979 if (!huge_zero_folio_shrinker) {
964980 shrinker_free(deferred_split_shrinker);
981+ list_lru_destroy(&deferred_split_lru);
965982 return -ENOMEM;
966983 }
967984
@@ -976,6 +993,7 @@ static void __init thp_shrinker_exit(void) · 8 unmodified lines
976993 {
977994 shrinker_free(huge_zero_folio_shrinker);
978995 shrinker_free(deferred_split_shrinker);
996+ list_lru_destroy(&deferred_split_lru);
979997 }
980998
981999 static int __init hugepage_init(void)
@@ -1155,119 +1173,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) · 173 unmodified lines
11551173 return pmd;
11561174 }
11571175
1158−static struct deferred_split *split_queue_node(int nid)
1159−{
1160− struct pglist_data *pgdata = NODE_DATA(nid);
1161−
1162− return &pgdata->deferred_split_queue;
1163−}
1164−
1165−#ifdef CONFIG_MEMCG
1166−static inline
1167−struct mem_cgroup *folio_split_queue_memcg(struct folio *folio,
1168− struct deferred_split *queue)
1169−{
1170− if (mem_cgroup_disabled())
1171− return NULL;
1172− if (split_queue_node(folio_nid(folio)) == queue)
1173− return NULL;
1174− return container_of(queue, struct mem_cgroup, deferred_split_queue);
1175−}
1176−
1177−static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg)
1178−{
1179− return memcg ? &memcg->deferred_split_queue : split_queue_node(nid);
1180−}
1181−#else
1182−static inline
1183−struct mem_cgroup *folio_split_queue_memcg(struct folio *folio,
1184− struct deferred_split *queue)
1185−{
1186− return NULL;
1187−}
1188−
1189−static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg)
1190−{
1191− return split_queue_node(nid);
1192−}
1193−#endif
1194−
1195−static struct deferred_split *split_queue_lock(int nid, struct mem_cgroup *memcg)
1196−{
1197− struct deferred_split *queue;
1198−
1199−retry:
1200− queue = memcg_split_queue(nid, memcg);
1201− spin_lock(&queue->split_queue_lock);
1202− /*
1203− * There is a period between setting memcg to dying and reparenting
1204− * deferred split queue, and during this period the THPs in the deferred
1205− * split queue will be hidden from the shrinker side.
1206− */
1207− if (unlikely(memcg_is_dying(memcg))) {
1208− spin_unlock(&queue->split_queue_lock);
1209− memcg = parent_mem_cgroup(memcg);
1210− goto retry;
1211− }
1212−
1213− return queue;
1214−}
1215−
1216−static struct deferred_split *
1217−split_queue_lock_irqsave(int nid, struct mem_cgroup *memcg, unsigned long *flags)
1218−{
1219− struct deferred_split *queue;
1220−
1221−retry:
1222− queue = memcg_split_queue(nid, memcg);
1223− spin_lock_irqsave(&queue->split_queue_lock, *flags);
1224− if (unlikely(memcg_is_dying(memcg))) {
1225− spin_unlock_irqrestore(&queue->split_queue_lock, *flags);
1226− memcg = parent_mem_cgroup(memcg);
1227− goto retry;
1228− }
1229−
1230− return queue;
1231−}
1232−
1233−static struct deferred_split *folio_split_queue_lock(struct folio *folio)
1234−{
1235− struct deferred_split *queue;
1236−
1237− rcu_read_lock();
1238− queue = split_queue_lock(folio_nid(folio), folio_memcg(folio));
1239− /*
1240− * The memcg destruction path is acquiring the split queue lock for
1241− * reparenting. Once you have it locked, it's safe to drop the rcu lock.
1242− */
1243− rcu_read_unlock();
1244−
1245− return queue;
1246−}
1247−
1248−static struct deferred_split *
1249−folio_split_queue_lock_irqsave(struct folio *folio, unsigned long *flags)
1250−{
1251− struct deferred_split *queue;
1252−
1253− rcu_read_lock();
1254− queue = split_queue_lock_irqsave(folio_nid(folio), folio_memcg(folio), flags);
1255− rcu_read_unlock();
1256−
1257− return queue;
1258−}
1259−
1260−static inline void split_queue_unlock(struct deferred_split *queue)
1261−{
1262− spin_unlock(&queue->split_queue_lock);
1263−}
1264−
1265−static inline void split_queue_unlock_irqrestore(struct deferred_split *queue,
1266− unsigned long flags)
1267−{
1268− spin_unlock_irqrestore(&queue->split_queue_lock, flags);
1269−}
1270−
12711176 static inline bool is_transparent_hugepage(const struct folio *folio)
12721177 {
12731178 if (!folio_test_large(folio))
@@ -1368,6 +1273,14 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, · 94 unmodified lines
13681273 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
13691274 return NULL;
13701275 }
1276+
1277+ if (folio_memcg_alloc_deferred(folio)) {
1278+ folio_put(folio);
1279+ count_vm_event(THP_FAULT_FALLBACK);
1280+ count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
1281+ return NULL;
1282+ }
1283+
13711284 folio_throttle_swaprate(folio, gfp);
13721285
13731286 /*
@@ -3903,34 +3816,43 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n · 2529 unmodified lines
39033816 struct folio *end_folio = folio_next(folio);
39043817 struct folio *new_folio, *next;
39053818 int old_order = folio_order(folio);
3819+ struct list_lru_one *lru;
3820+ bool dequeue_deferred;
39063821 int ret = 0;
3907− struct deferred_split *ds_queue;
39083822
39093823 VM_WARN_ON_ONCE(!mapping && end);
3910− /* Prevent deferred_split_scan() touching ->_refcount */
3911− ds_queue = folio_split_queue_lock(folio);
3824+ /*
3825+ * If this folio can be on the deferred split queue, lock out
3826+ * the shrinker before freezing the ref. If the shrinker sees
3827+ * a 0-ref folio, it assumes it beat folio_put() to the list
3828+ * lock and must clean up the LRU state - the same dequeue we
3829+ * will do below as part of the split.
3830+ */
3831+ dequeue_deferred = folio_test_anon(folio) && old_order > 1;
3832+ if (dequeue_deferred) {
3833+ struct mem_cgroup *memcg;
3834+
3835+ rcu_read_lock();
3836+ memcg = folio_memcg(folio);
3837+ lru = list_lru_lock(&deferred_split_lru,
3838+ folio_nid(folio), &memcg);
3839+ }
39123840 if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) {
39133841 struct swap_cluster_info *ci = NULL;
39143842 struct lruvec *lruvec;
39153843
3916− if (old_order > 1) {
3917− if (!list_empty(&folio->_deferred_list)) {
3918− ds_queue->split_queue_len--;
3919− /*
3920− * Reinitialize page_deferred_list after removing the
3921− * page from the split_queue, otherwise a subsequent
3922− * split will see list corruption when checking the
3923− * page_deferred_list.
3924− */
3925− list_del_init(&folio->_deferred_list);
3926− }
3844+ if (dequeue_deferred) {
3845+ __list_lru_del(&deferred_split_lru, lru,
3846+ &folio->_deferred_list, folio_nid(folio));
39273847 if (folio_test_partially_mapped(folio)) {
39283848 folio_clear_partially_mapped(folio);
39293849 mod_mthp_stat(old_order,
39303850 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
39313851 }
3852+ list_lru_unlock(lru);
3853+ rcu_read_unlock();
39323854 }
3933− split_queue_unlock(ds_queue);
3855+
39343856 if (mapping) {
39353857 int nr = folio_nr_pages(folio);
39363858
@@ -4031,7 +3953,10 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n · 94 unmodified lines
40313953 if (ci)
40323954 swap_cluster_unlock(ci);
40333955 } else {
4034− split_queue_unlock(ds_queue);
3956+ if (dequeue_deferred) {
3957+ list_lru_unlock(lru);
3958+ rcu_read_unlock();
3959+ }
40353960 return -EAGAIN;
40363961 }
40373962
@@ -4397,33 +4322,37 @@ int split_folio_to_list(struct folio *folio, struct list_head *list) · 359 unmodified lines
43974322 * queueing THP splits, and that list is (racily observed to be) non-empty.
43984323 *
43994324 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is
4400− * zero: because even when split_queue_lock is held, a non-empty _deferred_list
4401− * might be in use on deferred_split_scan()'s unlocked on-stack list.
4325+ * zero: because even when the list_lru lock is held, a non-empty
4326+ * _deferred_list might be in use on deferred_split_scan()'s unlocked
4327+ * on-stack list.
44024328 *
4403− * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is
4404− * therefore important to unqueue deferred split before changing folio memcg.
4329+ * The list_lru sublist is determined by folio's memcg: it is therefore
4330+ * important to unqueue deferred split before changing folio memcg.
44054331 */
44064332 bool __folio_unqueue_deferred_split(struct folio *folio)
44074333 {
4408− struct deferred_split *ds_queue;
4334+ struct mem_cgroup *memcg;
4335+ struct list_lru_one *lru;
4336+ int nid = folio_nid(folio);
44094337 unsigned long flags;
44104338 bool unqueued = false;
44114339
44124340 WARN_ON_ONCE(folio_ref_count(folio));
44134341 WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg_charged(folio));
44144342
4415− ds_queue = folio_split_queue_lock_irqsave(folio, &flags);
4416− if (!list_empty(&folio->_deferred_list)) {
4417− ds_queue->split_queue_len--;
4343+ rcu_read_lock();
4344+ memcg = folio_memcg(folio);
4345+ lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags);
4346+ if (__list_lru_del(&deferred_split_lru, lru, &folio->_deferred_list, nid)) {
44184347 if (folio_test_partially_mapped(folio)) {
44194348 folio_clear_partially_mapped(folio);
44204349 mod_mthp_stat(folio_order(folio),
44214350 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
44224351 }
4423− list_del_init(&folio->_deferred_list);
44244352 unqueued = true;
44254353 }
4426− split_queue_unlock_irqrestore(ds_queue, flags);
4354+ list_lru_unlock_irqrestore(lru, &flags);
4355+ rcu_read_unlock();
44274356
44284357 return unqueued; /* useful for debug warnings */
44294358 }
@@ -4431,7 +4360,9 @@ bool __folio_unqueue_deferred_split(struct folio *folio) · 1 unmodified line
44314360 /* partially_mapped=false won't clear PG_partially_mapped folio flag */
44324361 void deferred_split_folio(struct folio *folio, bool partially_mapped)
44334362 {
4434− struct deferred_split *ds_queue;
4363+ struct list_lru_one *lru;
4364+ int nid;
4365+ struct mem_cgroup *memcg;
44354366 unsigned long flags;
44364367
44374368 /*
@@ -4454,7 +4385,11 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) · 16 unmodified lines
44544385 if (folio_test_swapcache(folio))
44554386 return;
44564387
4457− ds_queue = folio_split_queue_lock_irqsave(folio, &flags);
4388+ nid = folio_nid(folio);
4389+
4390+ rcu_read_lock();
4391+ memcg = folio_memcg(folio);
4392+ lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags);
44584393 if (partially_mapped) {
44594394 if (!folio_test_partially_mapped(folio)) {
44604395 folio_set_partially_mapped(folio);
@@ -4462,36 +4397,20 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) · 1 unmodified line
44624397 count_vm_event(THP_DEFERRED_SPLIT_PAGE);
44634398 count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED);
44644399 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1);
4465−
44664400 }
44674401 } else {
44684402 /* partially mapped folios cannot become non-partially mapped */
44694403 VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio);
44704404 }
4471− if (list_empty(&folio->_deferred_list)) {
4472− struct mem_cgroup *memcg;
4473−
4474− memcg = folio_split_queue_memcg(folio, ds_queue);
4475− list_add_tail(&folio->_deferred_list, &ds_queue->split_queue);
4476− ds_queue->split_queue_len++;
4477− if (memcg)
4478− set_shrinker_bit(memcg, folio_nid(folio),
4479− shrinker_id(deferred_split_shrinker));
4480− }
4481− split_queue_unlock_irqrestore(ds_queue, flags);
4405+ __list_lru_add(&deferred_split_lru, lru, &folio->_deferred_list, nid, memcg);
4406+ list_lru_unlock_irqrestore(lru, &flags);
4407+ rcu_read_unlock();
44824408 }
44834409
44844410 static unsigned long deferred_split_count(struct shrinker *shrink,
44854411 struct shrink_control *sc)
44864412 {
4487− struct pglist_data *pgdata = NODE_DATA(sc->nid);
4488− struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
4489−
4490−#ifdef CONFIG_MEMCG
4491− if (sc->memcg)
4492− ds_queue = &sc->memcg->deferred_split_queue;
4493−#endif
4494− return READ_ONCE(ds_queue->split_queue_len);
4413+ return list_lru_shrink_count(&deferred_split_lru, sc);
44954414 }
44964415
44974416 static bool thp_underused(struct folio *folio)
@@ -4521,45 +4440,49 @@ static bool thp_underused(struct folio *folio) · 23 unmodified lines
45214440 return false;
45224441 }
45234442
4443+static enum lru_status deferred_split_isolate(struct list_head *item,
4444+ struct list_lru_one *lru,
4445+ void *cb_arg)
4446+{
4447+ struct folio *folio = container_of(item, struct folio, _deferred_list);
4448+ struct list_head *freeable = cb_arg;
4449+
4450+ if (folio_try_get(folio)) {
4451+ list_lru_isolate_move(lru, item, freeable);
4452+ return LRU_REMOVED;
4453+ }
4454+
4455+ /*
4456+ * We lost race with folio_put(). Read folio state before the
4457+ * isolate: folio_unqueue_deferred_split() checks list_empty()
4458+ * locklessly, so once removed the folio can be freed any time.
4459+ */
4460+ if (folio_test_partially_mapped(folio)) {
4461+ folio_clear_partially_mapped(folio);
4462+ mod_mthp_stat(folio_order(folio),
4463+ MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
4464+ }
4465+ list_lru_isolate(lru, item);
4466+ return LRU_REMOVED;
4467+}
4468+
45244469 static unsigned long deferred_split_scan(struct shrinker *shrink,
45254470 struct shrink_control *sc)
45264471 {
4527− struct deferred_split *ds_queue;
4528− unsigned long flags;
4472+ LIST_HEAD(dispose);
45294473 struct folio *folio, *next;
4530− int split = 0, i;
4531− struct folio_batch fbatch;
4532−
4533− folio_batch_init(&fbatch);
4474+ int split = 0;
4475+ unsigned long isolated;
45344476
4535−retry:
4536− ds_queue = split_queue_lock_irqsave(sc->nid, sc->memcg, &flags);
4537− /* Take pin on all head pages to avoid freeing them under us */
4538− list_for_each_entry_safe(folio, next, &ds_queue->split_queue,
4539− _deferred_list) {
4540− if (folio_try_get(folio)) {
4541− folio_batch_add(&fbatch, folio);
4542− } else if (folio_test_partially_mapped(folio)) {
4543− /* We lost race with folio_put() */
4544− folio_clear_partially_mapped(folio);
4545− mod_mthp_stat(folio_order(folio),
4546− MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
4547− }
4548− list_del_init(&folio->_deferred_list);
4549− ds_queue->split_queue_len--;
4550− if (!--sc->nr_to_scan)
4551− break;
4552− if (!folio_batch_space(&fbatch))
4553− break;
4554− }
4555− split_queue_unlock_irqrestore(ds_queue, flags);
4477+ isolated = list_lru_shrink_walk_irq(&deferred_split_lru, sc,
4478+ deferred_split_isolate, &dispose);
45564479
4557− for (i = 0; i < folio_batch_count(&fbatch); i++) {
4480+ list_for_each_entry_safe(folio, next, &dispose, _deferred_list) {
45584481 bool did_split = false;
45594482 bool underused = false;
4560− struct deferred_split *fqueue;
45614483
4562− folio = fbatch.folios[i];
4484+ list_del_init(&folio->_deferred_list);
4485+
45634486 if (!folio_test_partially_mapped(folio)) {
45644487 /*
45654488 * See try_to_map_unused_to_zeropage(): we cannot
@@ -4588,63 +4511,23 @@ next: · 22 unmodified lines
45884511 * underused, then consider it used and don't add it back to
45894512 * split_queue.
45904513 */
4591− if (did_split || !folio_test_partially_mapped(folio))
4592− continue;
4514+ if (!did_split && folio_test_partially_mapped(folio)) {
45934515 requeue:
4594− /*
4595− * Add back partially mapped folios, or underused folios that
4596− * we could not lock this round.
4597− */
4598− fqueue = folio_split_queue_lock_irqsave(folio, &flags);
4599− if (list_empty(&folio->_deferred_list)) {
4600− list_add_tail(&folio->_deferred_list, &fqueue->split_queue);
4601− fqueue->split_queue_len++;
4516+ rcu_read_lock();
4517+ list_lru_add_irq(&deferred_split_lru,
4518+ &folio->_deferred_list,
4519+ folio_nid(folio),
4520+ folio_memcg(folio));
4521+ rcu_read_unlock();
46024522 }
4603− split_queue_unlock_irqrestore(fqueue, flags);
4604− }
4605− folios_put(&fbatch);
4606−
4607− if (sc->nr_to_scan && !list_empty(&ds_queue->split_queue)) {
4608− cond_resched();
4609− goto retry;
4523+ folio_put(folio);
46104524 }
46114525
4612− /*
4613− * Stop shrinker if we didn't split any page, but the queue is empty.
4614− * This can happen if pages were freed under us.
4615− */
4616− if (!split && list_empty(&ds_queue->split_queue))
4526+ if (!split && !isolated)
46174527 return SHRINK_STOP;
46184528 return split;
46194529 }
46204530
4621−#ifdef CONFIG_MEMCG
4622−void reparent_deferred_split_queue(struct mem_cgroup *memcg)
4623−{
4624− struct mem_cgroup *parent = parent_mem_cgroup(memcg);
4625− struct deferred_split *ds_queue = &memcg->deferred_split_queue;
4626− struct deferred_split *parent_ds_queue = &parent->deferred_split_queue;
4627− int nid;
4628−
4629− spin_lock_irq(&ds_queue->split_queue_lock);
4630− spin_lock_nested(&parent_ds_queue->split_queue_lock, SINGLE_DEPTH_NESTING);
4631−
4632− if (!ds_queue->split_queue_len)
4633− goto unlock;
4634−
4635− list_splice_tail_init(&ds_queue->split_queue, &parent_ds_queue->split_queue);
4636− parent_ds_queue->split_queue_len += ds_queue->split_queue_len;
4637− ds_queue->split_queue_len = 0;
4638−
4639− for_each_node(nid)
4640− set_shrinker_bit(parent, nid, shrinker_id(deferred_split_shrinker));
4641−
4642−unlock:
4643− spin_unlock(&parent_ds_queue->split_queue_lock);
4644− spin_unlock_irq(&ds_queue->split_queue_lock);
4645−}
4646−#endif
4647−
46484531 #ifdef CONFIG_DEBUG_FS
46494532 static void split_huge_pages_all(void)
46504533 {
mm/internal.h+1−1
@@ -852,7 +852,7 @@ static inline bool folio_unqueue_deferred_split(struct folio *folio) · 851 unmodified lines
852852 /*
853853 * At this point, there is no one trying to add the folio to
854854 * deferred_list. If folio is not in deferred_list, it's safe
855− * to check without acquiring the split_queue_lock.
855+ * to check without acquiring the list_lru lock.
856856 */
857857 if (data_race(list_empty(&folio->_deferred_list)))
858858 return false;
mm/khugepaged.c+5−0
@@ -1123,6 +1123,11 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a · 1122 unmodified lines
11231123 if (result != SCAN_SUCCEED)
11241124 goto out_nolock;
11251125
1126+ if (folio_memcg_alloc_deferred(folio)) {
1127+ result = SCAN_ALLOC_HUGE_PAGE_FAIL;
1128+ goto out_nolock;
1129+ }
1130+
11261131 mmap_read_lock(mm);
11271132 result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
11281133 if (result != SCAN_SUCCEED) {
mm/memcontrol.c+3−9
@@ -4143,11 +4143,6 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) · 4142 unmodified lines
41434143 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
41444144 memcg->cgwb_frn[i].done =
41454145 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
4146−#endif
4147−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4148− spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
4149− INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
4150− memcg->deferred_split_queue.split_queue_len = 0;
41514146 #endif
41524147 lru_gen_init_memcg(memcg);
41534148 return memcg;
@@ -4299,11 +4294,10 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) · 145 unmodified lines
42994294 zswap_memcg_offline_cleanup(memcg);
43004295
43014296 memcg_offline_kmem(memcg);
4302− reparent_deferred_split_queue(memcg);
43034297 /*
4304− * The reparenting of objcg must be after the reparenting of the
4305− * list_lru and deferred_split_queue above, which ensures that they will
4306− * not mistakenly get the parent list_lru and deferred_split_queue.
4298+ * The reparenting of objcg must be after the reparenting of
4299+ * the list_lru in memcg_offline_kmem(), which ensures that
4300+ * they will not mistakenly get the parent list_lru.
43074301 */
43084302 memcg_reparent_objcgs(memcg);
43094303 reparent_shrinker_deferred(memcg);
mm/memory.c+4−0
@@ -5222,6 +5222,10 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) · 5221 unmodified lines
52225222 folio_put(folio);
52235223 goto next;
52245224 }
5225+ if (order > 1 && folio_memcg_alloc_deferred(folio)) {
5226+ folio_put(folio);
5227+ goto fallback;
5228+ }
52255229 folio_throttle_swaprate(folio, gfp);
52265230 /*
52275231 * When a folio is not zeroed during allocation
mm/mm_init.c+0−15
@@ -1373,19 +1373,6 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, · 1372 unmodified lines
13731373 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);
13741374 }
13751375
1376−#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1377−static void pgdat_init_split_queue(struct pglist_data *pgdat)
1378−{
1379− struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
1380−
1381− spin_lock_init(&ds_queue->split_queue_lock);
1382− INIT_LIST_HEAD(&ds_queue->split_queue);
1383− ds_queue->split_queue_len = 0;
1384−}
1385−#else
1386−static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
1387−#endif
1388−
13891376 #ifdef CONFIG_COMPACTION
13901377 static void pgdat_init_kcompactd(struct pglist_data *pgdat)
13911378 {
@@ -1401,8 +1388,6 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) · 9 unmodified lines
14011388
14021389 pgdat_resize_init(pgdat);
14031390 pgdat_kswapd_lock_init(pgdat);
1404−
1405− pgdat_init_split_queue(pgdat);
14061391 pgdat_init_kcompactd(pgdat);
14071392
14081393 init_waitqueue_head(&pgdat->kswapd_wait);
mm/swap_state.c+10−0
@@ -465,6 +465,16 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, · 464 unmodified lines
465465 return ERR_PTR(-ENOMEM);
466466 }
467467
468+ if (order > 1 && folio_memcg_alloc_deferred(folio)) {
469+ spin_lock(&ci->lock);
470+ __swap_cache_do_del_folio(ci, folio, entry, shadow);
471+ spin_unlock(&ci->lock);
472+ folio_unlock(folio);
473+ /* nr_pages refs from swap cache, 1 from allocation */
474+ folio_put_refs(folio, nr_pages + 1);
475+ return ERR_PTR(-ENOMEM);
476+ }
477+
468478 /* memsw uncharges swap when folio is added to swap cache */
469479 memcg1_swapin(folio);
470480 if (shadow)