| @@ -14,6 +14,7 @@ · 13 unmodified lines |
| 14 | #include <linux/mmu_notifier.h> | 14 | #include <linux/mmu_notifier.h> |
| 15 | #include <linux/rmap.h> | 15 | #include <linux/rmap.h> |
| 16 | #include <linux/swap.h> | 16 | #include <linux/swap.h> |
| | 17 | #include <linux/list_lru.h> |
| 17 | #include <linux/shrinker.h> | 18 | #include <linux/shrinker.h> |
| 18 | #include <linux/mm_inline.h> | 19 | #include <linux/mm_inline.h> |
| 19 | #include <linux/swapops.h> | 20 | #include <linux/swapops.h> |
| @@ -67,6 +68,8 @@ unsigned long transparent_hugepage_flags __read_mostly = · 47 unmodified lines |
| 67 | (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| | 68 | (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| |
| 68 | (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); | 69 | (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); |
| 69 | | 70 | |
| | 71 | static struct lock_class_key deferred_split_key; |
| | 72 | static struct list_lru deferred_split_lru; |
| 70 | static struct shrinker *deferred_split_shrinker; | 73 | static struct shrinker *deferred_split_shrinker; |
| 71 | static unsigned long deferred_split_count(struct shrinker *shrink, | 74 | static unsigned long deferred_split_count(struct shrinker *shrink, |
| 72 | struct shrink_control *sc); | 75 | struct shrink_control *sc); |
| @@ -932,15 +935,28 @@ static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) · 859 unmodified lines |
| 932 | } | 935 | } |
| 933 | #endif /* CONFIG_SYSFS */ | 936 | #endif /* CONFIG_SYSFS */ |
| 934 | | 937 | |
| | 938 | int folio_memcg_alloc_deferred(struct folio *folio) |
| | 939 | { |
| | 940 | if (mem_cgroup_disabled()) |
| | 941 | return 0; |
| | 942 | return folio_memcg_list_lru_alloc(folio, &deferred_split_lru, GFP_KERNEL); |
| | 943 | } |
| | 944 | |
| 935 | static int __init thp_shrinker_init(void) | 945 | static int __init thp_shrinker_init(void) |
| 936 | { | 946 | { |
| 937 | deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | | 947 | deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | |
| 938 | SHRINKER_MEMCG_AWARE | | 948 | SHRINKER_MEMCG_AWARE, |
| 939 | SHRINKER_NONSLAB, | | |
| 940 | "thp-deferred_split"); | 949 | "thp-deferred_split"); |
| 941 | if (!deferred_split_shrinker) | 950 | if (!deferred_split_shrinker) |
| 942 | return -ENOMEM; | 951 | return -ENOMEM; |
| 943 | | 952 | |
| | 953 | if (list_lru_init_memcg_key(&deferred_split_lru, |
| | 954 | deferred_split_shrinker, |
| | 955 | &deferred_split_key)) { |
| | 956 | shrinker_free(deferred_split_shrinker); |
| | 957 | return -ENOMEM; |
| | 958 | } |
| | 959 | |
| 944 | deferred_split_shrinker->count_objects = deferred_split_count; | 960 | deferred_split_shrinker->count_objects = deferred_split_count; |
| 945 | deferred_split_shrinker->scan_objects = deferred_split_scan; | 961 | deferred_split_shrinker->scan_objects = deferred_split_scan; |
| 946 | shrinker_register(deferred_split_shrinker); | 962 | shrinker_register(deferred_split_shrinker); |
| @@ -962,6 +978,7 @@ static int __init thp_shrinker_init(void) · 15 unmodified lines |
| 962 | huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); | 978 | huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); |
| 963 | if (!huge_zero_folio_shrinker) { | 979 | if (!huge_zero_folio_shrinker) { |
| 964 | shrinker_free(deferred_split_shrinker); | 980 | shrinker_free(deferred_split_shrinker); |
| | 981 | list_lru_destroy(&deferred_split_lru); |
| 965 | return -ENOMEM; | 982 | return -ENOMEM; |
| 966 | } | 983 | } |
| 967 | | 984 | |
| @@ -976,6 +993,7 @@ static void __init thp_shrinker_exit(void) · 8 unmodified lines |
| 976 | { | 993 | { |
| 977 | shrinker_free(huge_zero_folio_shrinker); | 994 | shrinker_free(huge_zero_folio_shrinker); |
| 978 | shrinker_free(deferred_split_shrinker); | 995 | shrinker_free(deferred_split_shrinker); |
| | 996 | list_lru_destroy(&deferred_split_lru); |
| 979 | } | 997 | } |
| 980 | | 998 | |
| 981 | static int __init hugepage_init(void) | 999 | static int __init hugepage_init(void) |
| @@ -1155,119 +1173,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) · 173 unmodified lines |
| 1155 | return pmd; | 1173 | return pmd; |
| 1156 | } | 1174 | } |
| 1157 | | 1175 | |
| 1158 | static struct deferred_split *split_queue_node(int nid) | | |
| 1159 | { | | |
| 1160 | struct pglist_data *pgdata = NODE_DATA(nid); | | |
| 1161 | | | |
| 1162 | return &pgdata->deferred_split_queue; | | |
| 1163 | } | | |
| 1164 | | | |
| 1165 | #ifdef CONFIG_MEMCG | | |
| 1166 | static inline | | |
| 1167 | struct mem_cgroup *folio_split_queue_memcg(struct folio *folio, | | |
| 1168 | struct deferred_split *queue) | | |
| 1169 | { | | |
| 1170 | if (mem_cgroup_disabled()) | | |
| 1171 | return NULL; | | |
| 1172 | if (split_queue_node(folio_nid(folio)) == queue) | | |
| 1173 | return NULL; | | |
| 1174 | return container_of(queue, struct mem_cgroup, deferred_split_queue); | | |
| 1175 | } | | |
| 1176 | | | |
| 1177 | static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg) | | |
| 1178 | { | | |
| 1179 | return memcg ? &memcg->deferred_split_queue : split_queue_node(nid); | | |
| 1180 | } | | |
| 1181 | #else | | |
| 1182 | static inline | | |
| 1183 | struct mem_cgroup *folio_split_queue_memcg(struct folio *folio, | | |
| 1184 | struct deferred_split *queue) | | |
| 1185 | { | | |
| 1186 | return NULL; | | |
| 1187 | } | | |
| 1188 | | | |
| 1189 | static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg) | | |
| 1190 | { | | |
| 1191 | return split_queue_node(nid); | | |
| 1192 | } | | |
| 1193 | #endif | | |
| 1194 | | | |
| 1195 | static struct deferred_split *split_queue_lock(int nid, struct mem_cgroup *memcg) | | |
| 1196 | { | | |
| 1197 | struct deferred_split *queue; | | |
| 1198 | | | |
| 1199 | retry: | | |
| 1200 | queue = memcg_split_queue(nid, memcg); | | |
| 1201 | spin_lock(&queue->split_queue_lock); | | |
| 1202 | /* | | |
| 1203 | * There is a period between setting memcg to dying and reparenting | | |
| 1204 | * deferred split queue, and during this period the THPs in the deferred | | |
| 1205 | * split queue will be hidden from the shrinker side. | | |
| 1206 | */ | | |
| 1207 | if (unlikely(memcg_is_dying(memcg))) { | | |
| 1208 | spin_unlock(&queue->split_queue_lock); | | |
| 1209 | memcg = parent_mem_cgroup(memcg); | | |
| 1210 | goto retry; | | |
| 1211 | } | | |
| 1212 | | | |
| 1213 | return queue; | | |
| 1214 | } | | |
| 1215 | | | |
| 1216 | static struct deferred_split * | | |
| 1217 | split_queue_lock_irqsave(int nid, struct mem_cgroup *memcg, unsigned long *flags) | | |
| 1218 | { | | |
| 1219 | struct deferred_split *queue; | | |
| 1220 | | | |
| 1221 | retry: | | |
| 1222 | queue = memcg_split_queue(nid, memcg); | | |
| 1223 | spin_lock_irqsave(&queue->split_queue_lock, *flags); | | |
| 1224 | if (unlikely(memcg_is_dying(memcg))) { | | |
| 1225 | spin_unlock_irqrestore(&queue->split_queue_lock, *flags); | | |
| 1226 | memcg = parent_mem_cgroup(memcg); | | |
| 1227 | goto retry; | | |
| 1228 | } | | |
| 1229 | | | |
| 1230 | return queue; | | |
| 1231 | } | | |
| 1232 | | | |
| 1233 | static struct deferred_split *folio_split_queue_lock(struct folio *folio) | | |
| 1234 | { | | |
| 1235 | struct deferred_split *queue; | | |
| 1236 | | | |
| 1237 | rcu_read_lock(); | | |
| 1238 | queue = split_queue_lock(folio_nid(folio), folio_memcg(folio)); | | |
| 1239 | /* | | |
| 1240 | * The memcg destruction path is acquiring the split queue lock for | | |
| 1241 | * reparenting. Once you have it locked, it's safe to drop the rcu lock. | | |
| 1242 | */ | | |
| 1243 | rcu_read_unlock(); | | |
| 1244 | | | |
| 1245 | return queue; | | |
| 1246 | } | | |
| 1247 | | | |
| 1248 | static struct deferred_split * | | |
| 1249 | folio_split_queue_lock_irqsave(struct folio *folio, unsigned long *flags) | | |
| 1250 | { | | |
| 1251 | struct deferred_split *queue; | | |
| 1252 | | | |
| 1253 | rcu_read_lock(); | | |
| 1254 | queue = split_queue_lock_irqsave(folio_nid(folio), folio_memcg(folio), flags); | | |
| 1255 | rcu_read_unlock(); | | |
| 1256 | | | |
| 1257 | return queue; | | |
| 1258 | } | | |
| 1259 | | | |
| 1260 | static inline void split_queue_unlock(struct deferred_split *queue) | | |
| 1261 | { | | |
| 1262 | spin_unlock(&queue->split_queue_lock); | | |
| 1263 | } | | |
| 1264 | | | |
| 1265 | static inline void split_queue_unlock_irqrestore(struct deferred_split *queue, | | |
| 1266 | unsigned long flags) | | |
| 1267 | { | | |
| 1268 | spin_unlock_irqrestore(&queue->split_queue_lock, flags); | | |
| 1269 | } | | |
| 1270 | | | |
| 1271 | static inline bool is_transparent_hugepage(const struct folio *folio) | 1176 | static inline bool is_transparent_hugepage(const struct folio *folio) |
| 1272 | { | 1177 | { |
| 1273 | if (!folio_test_large(folio)) | 1178 | if (!folio_test_large(folio)) |
| @@ -1368,6 +1273,14 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, · 94 unmodified lines |
| 1368 | count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); | 1273 | count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); |
| 1369 | return NULL; | 1274 | return NULL; |
| 1370 | } | 1275 | } |
| | 1276 | |
| | 1277 | if (folio_memcg_alloc_deferred(folio)) { |
| | 1278 | folio_put(folio); |
| | 1279 | count_vm_event(THP_FAULT_FALLBACK); |
| | 1280 | count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); |
| | 1281 | return NULL; |
| | 1282 | } |
| | 1283 | |
| 1371 | folio_throttle_swaprate(folio, gfp); | 1284 | folio_throttle_swaprate(folio, gfp); |
| 1372 | | 1285 | |
| 1373 | /* | 1286 | /* |
| @@ -3903,34 +3816,43 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n · 2529 unmodified lines |
| 3903 | struct folio *end_folio = folio_next(folio); | 3816 | struct folio *end_folio = folio_next(folio); |
| 3904 | struct folio *new_folio, *next; | 3817 | struct folio *new_folio, *next; |
| 3905 | int old_order = folio_order(folio); | 3818 | int old_order = folio_order(folio); |
| | 3819 | struct list_lru_one *lru; |
| | 3820 | bool dequeue_deferred; |
| 3906 | int ret = 0; | 3821 | int ret = 0; |
| 3907 | struct deferred_split *ds_queue; | | |
| 3908 | | 3822 | |
| 3909 | VM_WARN_ON_ONCE(!mapping && end); | 3823 | VM_WARN_ON_ONCE(!mapping && end); |
| 3910 | /* Prevent deferred_split_scan() touching ->_refcount */ | 3824 | /* |
| 3911 | ds_queue = folio_split_queue_lock(folio); | 3825 | * If this folio can be on the deferred split queue, lock out |
| | 3826 | * the shrinker before freezing the ref. If the shrinker sees |
| | 3827 | * a 0-ref folio, it assumes it beat folio_put() to the list |
| | 3828 | * lock and must clean up the LRU state - the same dequeue we |
| | 3829 | * will do below as part of the split. |
| | 3830 | */ |
| | 3831 | dequeue_deferred = folio_test_anon(folio) && old_order > 1; |
| | 3832 | if (dequeue_deferred) { |
| | 3833 | struct mem_cgroup *memcg; |
| | 3834 | |
| | 3835 | rcu_read_lock(); |
| | 3836 | memcg = folio_memcg(folio); |
| | 3837 | lru = list_lru_lock(&deferred_split_lru, |
| | 3838 | folio_nid(folio), &memcg); |
| | 3839 | } |
| 3912 | if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { | 3840 | if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { |
| 3913 | struct swap_cluster_info *ci = NULL; | 3841 | struct swap_cluster_info *ci = NULL; |
| 3914 | struct lruvec *lruvec; | 3842 | struct lruvec *lruvec; |
| 3915 | | 3843 | |
| 3916 | if (old_order > 1) { | 3844 | if (dequeue_deferred) { |
| 3917 | if (!list_empty(&folio->_deferred_list)) { | 3845 | __list_lru_del(&deferred_split_lru, lru, |
| 3918 | ds_queue->split_queue_len--; | 3846 | &folio->_deferred_list, folio_nid(folio)); |
| 3919 | /* | | |
| 3920 | * Reinitialize page_deferred_list after removing the | | |
| 3921 | * page from the split_queue, otherwise a subsequent | | |
| 3922 | * split will see list corruption when checking the | | |
| 3923 | * page_deferred_list. | | |
| 3924 | */ | | |
| 3925 | list_del_init(&folio->_deferred_list); | | |
| 3926 | } | | |
| 3927 | if (folio_test_partially_mapped(folio)) { | 3847 | if (folio_test_partially_mapped(folio)) { |
| 3928 | folio_clear_partially_mapped(folio); | 3848 | folio_clear_partially_mapped(folio); |
| 3929 | mod_mthp_stat(old_order, | 3849 | mod_mthp_stat(old_order, |
| 3930 | MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); | 3850 | MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); |
| 3931 | } | 3851 | } |
| | 3852 | list_lru_unlock(lru); |
| | 3853 | rcu_read_unlock(); |
| 3932 | } | 3854 | } |
| 3933 | split_queue_unlock(ds_queue); | 3855 | |
| 3934 | if (mapping) { | 3856 | if (mapping) { |
| 3935 | int nr = folio_nr_pages(folio); | 3857 | int nr = folio_nr_pages(folio); |
| 3936 | | 3858 | |
| @@ -4031,7 +3953,10 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n · 94 unmodified lines |
| 4031 | if (ci) | 3953 | if (ci) |
| 4032 | swap_cluster_unlock(ci); | 3954 | swap_cluster_unlock(ci); |
| 4033 | } else { | 3955 | } else { |
| 4034 | split_queue_unlock(ds_queue); | 3956 | if (dequeue_deferred) { |
| | 3957 | list_lru_unlock(lru); |
| | 3958 | rcu_read_unlock(); |
| | 3959 | } |
| 4035 | return -EAGAIN; | 3960 | return -EAGAIN; |
| 4036 | } | 3961 | } |
| 4037 | | 3962 | |
| @@ -4397,33 +4322,37 @@ int split_folio_to_list(struct folio *folio, struct list_head *list) · 359 unmodified lines |
| 4397 | * queueing THP splits, and that list is (racily observed to be) non-empty. | 4322 | * queueing THP splits, and that list is (racily observed to be) non-empty. |
| 4398 | * | 4323 | * |
| 4399 | * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is | 4324 | * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is |
| 4400 | * zero: because even when split_queue_lock is held, a non-empty _deferred_list | 4325 | * zero: because even when the list_lru lock is held, a non-empty |
| 4401 | * might be in use on deferred_split_scan()'s unlocked on-stack list. | 4326 | * _deferred_list might be in use on deferred_split_scan()'s unlocked |
| | 4327 | * on-stack list. |
| 4402 | * | 4328 | * |
| 4403 | * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is | 4329 | * The list_lru sublist is determined by folio's memcg: it is therefore |
| 4404 | * therefore important to unqueue deferred split before changing folio memcg. | 4330 | * important to unqueue deferred split before changing folio memcg. |
| 4405 | */ | 4331 | */ |
| 4406 | bool __folio_unqueue_deferred_split(struct folio *folio) | 4332 | bool __folio_unqueue_deferred_split(struct folio *folio) |
| 4407 | { | 4333 | { |
| 4408 | struct deferred_split *ds_queue; | 4334 | struct mem_cgroup *memcg; |
| | 4335 | struct list_lru_one *lru; |
| | 4336 | int nid = folio_nid(folio); |
| 4409 | unsigned long flags; | 4337 | unsigned long flags; |
| 4410 | bool unqueued = false; | 4338 | bool unqueued = false; |
| 4411 | | 4339 | |
| 4412 | WARN_ON_ONCE(folio_ref_count(folio)); | 4340 | WARN_ON_ONCE(folio_ref_count(folio)); |
| 4413 | WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg_charged(folio)); | 4341 | WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg_charged(folio)); |
| 4414 | | 4342 | |
| 4415 | ds_queue = folio_split_queue_lock_irqsave(folio, &flags); | 4343 | rcu_read_lock(); |
| 4416 | if (!list_empty(&folio->_deferred_list)) { | 4344 | memcg = folio_memcg(folio); |
| 4417 | ds_queue->split_queue_len--; | 4345 | lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags); |
| | 4346 | if (__list_lru_del(&deferred_split_lru, lru, &folio->_deferred_list, nid)) { |
| 4418 | if (folio_test_partially_mapped(folio)) { | 4347 | if (folio_test_partially_mapped(folio)) { |
| 4419 | folio_clear_partially_mapped(folio); | 4348 | folio_clear_partially_mapped(folio); |
| 4420 | mod_mthp_stat(folio_order(folio), | 4349 | mod_mthp_stat(folio_order(folio), |
| 4421 | MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); | 4350 | MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); |
| 4422 | } | 4351 | } |
| 4423 | list_del_init(&folio->_deferred_list); | | |
| 4424 | unqueued = true; | 4352 | unqueued = true; |
| 4425 | } | 4353 | } |
| 4426 | split_queue_unlock_irqrestore(ds_queue, flags); | 4354 | list_lru_unlock_irqrestore(lru, &flags); |
| | 4355 | rcu_read_unlock(); |
| 4427 | | 4356 | |
| 4428 | return unqueued; /* useful for debug warnings */ | 4357 | return unqueued; /* useful for debug warnings */ |
| 4429 | } | 4358 | } |
| @@ -4431,7 +4360,9 @@ bool __folio_unqueue_deferred_split(struct folio *folio) · 1 unmodified line |
| 4431 | /* partially_mapped=false won't clear PG_partially_mapped folio flag */ | 4360 | /* partially_mapped=false won't clear PG_partially_mapped folio flag */ |
| 4432 | void deferred_split_folio(struct folio *folio, bool partially_mapped) | 4361 | void deferred_split_folio(struct folio *folio, bool partially_mapped) |
| 4433 | { | 4362 | { |
| 4434 | struct deferred_split *ds_queue; | 4363 | struct list_lru_one *lru; |
| | 4364 | int nid; |
| | 4365 | struct mem_cgroup *memcg; |
| 4435 | unsigned long flags; | 4366 | unsigned long flags; |
| 4436 | | 4367 | |
| 4437 | /* | 4368 | /* |
| @@ -4454,7 +4385,11 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) · 16 unmodified lines |
| 4454 | if (folio_test_swapcache(folio)) | 4385 | if (folio_test_swapcache(folio)) |
| 4455 | return; | 4386 | return; |
| 4456 | | 4387 | |
| 4457 | ds_queue = folio_split_queue_lock_irqsave(folio, &flags); | 4388 | nid = folio_nid(folio); |
| | 4389 | |
| | 4390 | rcu_read_lock(); |
| | 4391 | memcg = folio_memcg(folio); |
| | 4392 | lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags); |
| 4458 | if (partially_mapped) { | 4393 | if (partially_mapped) { |
| 4459 | if (!folio_test_partially_mapped(folio)) { | 4394 | if (!folio_test_partially_mapped(folio)) { |
| 4460 | folio_set_partially_mapped(folio); | 4395 | folio_set_partially_mapped(folio); |
| @@ -4462,36 +4397,20 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) · 1 unmodified line |
| 4462 | count_vm_event(THP_DEFERRED_SPLIT_PAGE); | 4397 | count_vm_event(THP_DEFERRED_SPLIT_PAGE); |
| 4463 | count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); | 4398 | count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); |
| 4464 | mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); | 4399 | mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); |
| 4465 | | | |
| 4466 | } | 4400 | } |
| 4467 | } else { | 4401 | } else { |
| 4468 | /* partially mapped folios cannot become non-partially mapped */ | 4402 | /* partially mapped folios cannot become non-partially mapped */ |
| 4469 | VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); | 4403 | VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); |
| 4470 | } | 4404 | } |
| 4471 | if (list_empty(&folio->_deferred_list)) { | 4405 | __list_lru_add(&deferred_split_lru, lru, &folio->_deferred_list, nid, memcg); |
| 4472 | struct mem_cgroup *memcg; | 4406 | list_lru_unlock_irqrestore(lru, &flags); |
| 4473 | | 4407 | rcu_read_unlock(); |
| 4474 | memcg = folio_split_queue_memcg(folio, ds_queue); | | |
| 4475 | list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); | | |
| 4476 | ds_queue->split_queue_len++; | | |
| 4477 | if (memcg) | | |
| 4478 | set_shrinker_bit(memcg, folio_nid(folio), | | |
| 4479 | shrinker_id(deferred_split_shrinker)); | | |
| 4480 | } | | |
| 4481 | split_queue_unlock_irqrestore(ds_queue, flags); | | |
| 4482 | } | 4408 | } |
| 4483 | | 4409 | |
| 4484 | static unsigned long deferred_split_count(struct shrinker *shrink, | 4410 | static unsigned long deferred_split_count(struct shrinker *shrink, |
| 4485 | struct shrink_control *sc) | 4411 | struct shrink_control *sc) |
| 4486 | { | 4412 | { |
| 4487 | struct pglist_data *pgdata = NODE_DATA(sc->nid); | 4413 | return list_lru_shrink_count(&deferred_split_lru, sc); |
| 4488 | struct deferred_split *ds_queue = &pgdata->deferred_split_queue; | | |
| 4489 | | | |
| 4490 | #ifdef CONFIG_MEMCG | | |
| 4491 | if (sc->memcg) | | |
| 4492 | ds_queue = &sc->memcg->deferred_split_queue; | | |
| 4493 | #endif | | |
| 4494 | return READ_ONCE(ds_queue->split_queue_len); | | |
| 4495 | } | 4414 | } |
| 4496 | | 4415 | |
| 4497 | static bool thp_underused(struct folio *folio) | 4416 | static bool thp_underused(struct folio *folio) |
| @@ -4521,45 +4440,49 @@ static bool thp_underused(struct folio *folio) · 23 unmodified lines |
| 4521 | return false; | 4440 | return false; |
| 4522 | } | 4441 | } |
| 4523 | | 4442 | |
| | 4443 | static enum lru_status deferred_split_isolate(struct list_head *item, |
| | 4444 | struct list_lru_one *lru, |
| | 4445 | void *cb_arg) |
| | 4446 | { |
| | 4447 | struct folio *folio = container_of(item, struct folio, _deferred_list); |
| | 4448 | struct list_head *freeable = cb_arg; |
| | 4449 | |
| | 4450 | if (folio_try_get(folio)) { |
| | 4451 | list_lru_isolate_move(lru, item, freeable); |
| | 4452 | return LRU_REMOVED; |
| | 4453 | } |
| | 4454 | |
| | 4455 | /* |
| | 4456 | * We lost race with folio_put(). Read folio state before the |
| | 4457 | * isolate: folio_unqueue_deferred_split() checks list_empty() |
| | 4458 | * locklessly, so once removed the folio can be freed any time. |
| | 4459 | */ |
| | 4460 | if (folio_test_partially_mapped(folio)) { |
| | 4461 | folio_clear_partially_mapped(folio); |
| | 4462 | mod_mthp_stat(folio_order(folio), |
| | 4463 | MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); |
| | 4464 | } |
| | 4465 | list_lru_isolate(lru, item); |
| | 4466 | return LRU_REMOVED; |
| | 4467 | } |
| | 4468 | |
| 4524 | static unsigned long deferred_split_scan(struct shrinker *shrink, | 4469 | static unsigned long deferred_split_scan(struct shrinker *shrink, |
| 4525 | struct shrink_control *sc) | 4470 | struct shrink_control *sc) |
| 4526 | { | 4471 | { |
| 4527 | struct deferred_split *ds_queue; | 4472 | LIST_HEAD(dispose); |
| 4528 | unsigned long flags; | | |
| 4529 | struct folio *folio, *next; | 4473 | struct folio *folio, *next; |
| 4530 | int split = 0, i; | 4474 | int split = 0; |
| 4531 | struct folio_batch fbatch; | 4475 | unsigned long isolated; |
| 4532 | | | |
| 4533 | folio_batch_init(&fbatch); | | |
| 4534 | | 4476 | |
| 4535 | retry: | 4477 | isolated = list_lru_shrink_walk_irq(&deferred_split_lru, sc, |
| 4536 | ds_queue = split_queue_lock_irqsave(sc->nid, sc->memcg, &flags); | 4478 | deferred_split_isolate, &dispose); |
| 4537 | /* Take pin on all head pages to avoid freeing them under us */ | | |
| 4538 | list_for_each_entry_safe(folio, next, &ds_queue->split_queue, | | |
| 4539 | _deferred_list) { | | |
| 4540 | if (folio_try_get(folio)) { | | |
| 4541 | folio_batch_add(&fbatch, folio); | | |
| 4542 | } else if (folio_test_partially_mapped(folio)) { | | |
| 4543 | /* We lost race with folio_put() */ | | |
| 4544 | folio_clear_partially_mapped(folio); | | |
| 4545 | mod_mthp_stat(folio_order(folio), | | |
| 4546 | MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); | | |
| 4547 | } | | |
| 4548 | list_del_init(&folio->_deferred_list); | | |
| 4549 | ds_queue->split_queue_len--; | | |
| 4550 | if (!--sc->nr_to_scan) | | |
| 4551 | break; | | |
| 4552 | if (!folio_batch_space(&fbatch)) | | |
| 4553 | break; | | |
| 4554 | } | | |
| 4555 | split_queue_unlock_irqrestore(ds_queue, flags); | | |
| 4556 | | 4479 | |
| 4557 | for (i = 0; i < folio_batch_count(&fbatch); i++) { | 4480 | list_for_each_entry_safe(folio, next, &dispose, _deferred_list) { |
| 4558 | bool did_split = false; | 4481 | bool did_split = false; |
| 4559 | bool underused = false; | 4482 | bool underused = false; |
| 4560 | struct deferred_split *fqueue; | | |
| 4561 | | 4483 | |
| 4562 | folio = fbatch.folios[i]; | 4484 | list_del_init(&folio->_deferred_list); |
| | 4485 | |
| 4563 | if (!folio_test_partially_mapped(folio)) { | 4486 | if (!folio_test_partially_mapped(folio)) { |
| 4564 | /* | 4487 | /* |
| 4565 | * See try_to_map_unused_to_zeropage(): we cannot | 4488 | * See try_to_map_unused_to_zeropage(): we cannot |
| @@ -4588,63 +4511,23 @@ next: · 22 unmodified lines |
| 4588 | * underused, then consider it used and don't add it back to | 4511 | * underused, then consider it used and don't add it back to |
| 4589 | * split_queue. | 4512 | * split_queue. |
| 4590 | */ | 4513 | */ |
| 4591 | if (did_split || !folio_test_partially_mapped(folio)) | 4514 | if (!did_split && folio_test_partially_mapped(folio)) { |
| 4592 | continue; | | |
| 4593 | requeue: | 4515 | requeue: |
| 4594 | /* | 4516 | rcu_read_lock(); |
| 4595 | * Add back partially mapped folios, or underused folios that | 4517 | list_lru_add_irq(&deferred_split_lru, |
| 4596 | * we could not lock this round. | 4518 | &folio->_deferred_list, |
| 4597 | */ | 4519 | folio_nid(folio), |
| 4598 | fqueue = folio_split_queue_lock_irqsave(folio, &flags); | 4520 | folio_memcg(folio)); |
| 4599 | if (list_empty(&folio->_deferred_list)) { | 4521 | rcu_read_unlock(); |
| 4600 | list_add_tail(&folio->_deferred_list, &fqueue->split_queue); | | |
| 4601 | fqueue->split_queue_len++; | | |
| 4602 | } | 4522 | } |
| 4603 | split_queue_unlock_irqrestore(fqueue, flags); | 4523 | folio_put(folio); |
| 4604 | } | | |
| 4605 | folios_put(&fbatch); | | |
| 4606 | | | |
| 4607 | if (sc->nr_to_scan && !list_empty(&ds_queue->split_queue)) { | | |
| 4608 | cond_resched(); | | |
| 4609 | goto retry; | | |
| 4610 | } | 4524 | } |
| 4611 | | 4525 | |
| 4612 | /* | 4526 | if (!split && !isolated) |
| 4613 | * Stop shrinker if we didn't split any page, but the queue is empty. | | |
| 4614 | * This can happen if pages were freed under us. | | |
| 4615 | */ | | |
| 4616 | if (!split && list_empty(&ds_queue->split_queue)) | | |
| 4617 | return SHRINK_STOP; | 4527 | return SHRINK_STOP; |
| 4618 | return split; | 4528 | return split; |
| 4619 | } | 4529 | } |
| 4620 | | 4530 | |
| 4621 | #ifdef CONFIG_MEMCG | | |
| 4622 | void reparent_deferred_split_queue(struct mem_cgroup *memcg) | | |
| 4623 | { | | |
| 4624 | struct mem_cgroup *parent = parent_mem_cgroup(memcg); | | |
| 4625 | struct deferred_split *ds_queue = &memcg->deferred_split_queue; | | |
| 4626 | struct deferred_split *parent_ds_queue = &parent->deferred_split_queue; | | |
| 4627 | int nid; | | |
| 4628 | | | |
| 4629 | spin_lock_irq(&ds_queue->split_queue_lock); | | |
| 4630 | spin_lock_nested(&parent_ds_queue->split_queue_lock, SINGLE_DEPTH_NESTING); | | |
| 4631 | | | |
| 4632 | if (!ds_queue->split_queue_len) | | |
| 4633 | goto unlock; | | |
| 4634 | | | |
| 4635 | list_splice_tail_init(&ds_queue->split_queue, &parent_ds_queue->split_queue); | | |
| 4636 | parent_ds_queue->split_queue_len += ds_queue->split_queue_len; | | |
| 4637 | ds_queue->split_queue_len = 0; | | |
| 4638 | | | |
| 4639 | for_each_node(nid) | | |
| 4640 | set_shrinker_bit(parent, nid, shrinker_id(deferred_split_shrinker)); | | |
| 4641 | | | |
| 4642 | unlock: | | |
| 4643 | spin_unlock(&parent_ds_queue->split_queue_lock); | | |
| 4644 | spin_unlock_irq(&ds_queue->split_queue_lock); | | |
| 4645 | } | | |
| 4646 | #endif | | |
| 4647 | | | |
| 4648 | #ifdef CONFIG_DEBUG_FS | 4531 | #ifdef CONFIG_DEBUG_FS |
| 4649 | static void split_huge_pages_all(void) | 4532 | static void split_huge_pages_all(void) |
| 4650 | { | 4533 | { |