mm/swap.c and mm/mlock.c agree to drain any per-CPU batch as soon as a large folio is added: so collect_longterm_unpinnable_folios() just wastes effort when calling lru_add_drain[_all]() on a large folio. But although there is good reason not to batch up PMD-sized folios, we might well benefit from batching a small number of low-order mTHPs (though unclear how that "small number" limitation will be implemented). So ask if folio_may_be_lru_cached() rather than !folio_test_large(), to insulate those particular checks from future change. Name preferred to "folio_is_batchable" because large folios can well be put on a batch: it's just the per-CPU LRU caches, drained much later, which need care. Marked for stable, to counter the increase in lru_add_drain_all()s from "mm/gup: check ref_count instead of lru before migration".
Unified · Split
| @@ -385,6 +385,16 @@ void folio_add_lru_vma(struct folio *, struct vm_area_struct *); · 384 unmodified lines | |||
| 385 | void mark_page_accessed(struct page *); | 385 | void mark_page_accessed(struct page *); |
| 386 | void folio_mark_accessed(struct folio *); | 386 | void folio_mark_accessed(struct folio *); |
| 387 | 387 | ||
| 388 | static inline bool folio_may_be_lru_cached(struct folio *folio) | ||
| 389 | { | ||
| 390 | /* | ||
| 391 | * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. | ||
| 392 | * Holding small numbers of low-order mTHP folios in per-CPU LRU cache | ||
| 393 | * will be sensible, but nobody has implemented and tested that yet. | ||
| 394 | */ | ||
| 395 | return !folio_test_large(folio); | ||
| 396 | } | ||
| 397 | |||
| 388 | extern atomic_t lru_disable_count; | 398 | extern atomic_t lru_disable_count; |
| 389 | 399 | ||
| 390 | static inline bool lru_cache_disabled(void) | 400 | static inline bool lru_cache_disabled(void) |
| @@ -2307,13 +2307,13 @@ static unsigned long collect_longterm_unpinnable_folios( · 2306 unmodified lines | |||
| 2307 | continue; | 2307 | continue; |
| 2308 | } | 2308 | } |
| 2309 | 2309 | ||
| 2310 | if (drained == 0 && | 2310 | if (drained == 0 && folio_may_be_lru_cached(folio) && |
| 2311 | folio_ref_count(folio) != | 2311 | folio_ref_count(folio) != |
| 2312 | folio_expected_ref_count(folio) + 1) { | 2312 | folio_expected_ref_count(folio) + 1) { |
| 2313 | lru_add_drain(); | 2313 | lru_add_drain(); |
| 2314 | drained = 1; | 2314 | drained = 1; |
| 2315 | } | 2315 | } |
| 2316 | if (drained == 1 && | 2316 | if (drained == 1 && folio_may_be_lru_cached(folio) && |
| 2317 | folio_ref_count(folio) != | 2317 | folio_ref_count(folio) != |
| 2318 | folio_expected_ref_count(folio) + 1) { | 2318 | folio_expected_ref_count(folio) + 1) { |
| 2319 | lru_add_drain_all(); | 2319 | lru_add_drain_all(); |
| @@ -255,7 +255,7 @@ void mlock_folio(struct folio *folio) · 254 unmodified lines | |||
| 255 | 255 | ||
| 256 | folio_get(folio); | 256 | folio_get(folio); |
| 257 | if (!folio_batch_add(fbatch, mlock_lru(folio)) || | 257 | if (!folio_batch_add(fbatch, mlock_lru(folio)) || |
| 258 | folio_test_large(folio) || lru_cache_disabled()) | 258 | !folio_may_be_lru_cached(folio) || lru_cache_disabled()) |
| 259 | mlock_folio_batch(fbatch); | 259 | mlock_folio_batch(fbatch); |
| 260 | local_unlock(&mlock_fbatch.lock); | 260 | local_unlock(&mlock_fbatch.lock); |
| 261 | } | 261 | } |
| @@ -278,7 +278,7 @@ void mlock_new_folio(struct folio *folio) · 16 unmodified lines | |||
| 278 | 278 | ||
| 279 | folio_get(folio); | 279 | folio_get(folio); |
| 280 | if (!folio_batch_add(fbatch, mlock_new(folio)) || | 280 | if (!folio_batch_add(fbatch, mlock_new(folio)) || |
| 281 | folio_test_large(folio) || lru_cache_disabled()) | 281 | !folio_may_be_lru_cached(folio) || lru_cache_disabled()) |
| 282 | mlock_folio_batch(fbatch); | 282 | mlock_folio_batch(fbatch); |
| 283 | local_unlock(&mlock_fbatch.lock); | 283 | local_unlock(&mlock_fbatch.lock); |
| 284 | } | 284 | } |
| @@ -299,7 +299,7 @@ void munlock_folio(struct folio *folio) · 14 unmodified lines | |||
| 299 | */ | 299 | */ |
| 300 | folio_get(folio); | 300 | folio_get(folio); |
| 301 | if (!folio_batch_add(fbatch, folio) || | 301 | if (!folio_batch_add(fbatch, folio) || |
| 302 | folio_test_large(folio) || lru_cache_disabled()) | 302 | !folio_may_be_lru_cached(folio) || lru_cache_disabled()) |
| 303 | mlock_folio_batch(fbatch); | 303 | mlock_folio_batch(fbatch); |
| 304 | local_unlock(&mlock_fbatch.lock); | 304 | local_unlock(&mlock_fbatch.lock); |
| 305 | } | 305 | } |
| @@ -192,7 +192,7 @@ static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch, · 191 unmodified lines | |||
| 192 | local_lock(&cpu_fbatches.lock); | 192 | local_lock(&cpu_fbatches.lock); |
| 193 | 193 | ||
| 194 | if (!folio_batch_add(this_cpu_ptr(fbatch), folio) || | 194 | if (!folio_batch_add(this_cpu_ptr(fbatch), folio) || |
| 195 | folio_test_large(folio) || lru_cache_disabled()) | 195 | !folio_may_be_lru_cached(folio) || lru_cache_disabled()) |
| 196 | folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn); | 196 | folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn); |
| 197 | 197 | ||
| 198 | if (disable_irq) | 198 | if (disable_irq) |