mm: folio_may_be_lru_cached() unless folio_test_large()

mm/swap.c and mm/mlock.c agree to drain any per-CPU batch as soon as a large folio is added: so collect_longterm_unpinnable_folios() just wastes effort when calling lru_add_drain[_all]() on a large folio. But although there is good reason not to batch up PMD-sized folios, we might well benefit from batching a small number of low-order mTHPs (though unclear how that "small number" limitation will be implemented). So ask if folio_may_be_lru_cached() rather than !folio_test_large(), to insulate those particular checks from future change. Name preferred to "folio_is_batchable" because large folios can well be put on a batch: it's just the per-CPU LRU caches, drained much later, which need care. Marked for stable, to counter the increase in lru_add_drain_all()s from "mm/gup: check ref_count instead of lru before migration".

Hugh Dickins committed 1 year ago

Commit 2da6de31 parent 8d79ed34 files · +16 −6Browse files

Unified · Split

4 files

include/linux/swap.h +10 −0
mm/gup.c +2 −2
mm/mlock.c +3 −3
mm/swap.c +1 −1

include/linux/swap.h+10−0

@@ -385,6 +385,16 @@ void folio_add_lru_vma(struct folio , struct vm_area_struct ); · 384 unmodified lines
385	385	void mark_page_accessed(struct page *);
386	386	void folio_mark_accessed(struct folio *);
387	387
	388	+static inline bool folio_may_be_lru_cached(struct folio *folio)
	389	+{
	390	+ /*
	391	+ * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting.
	392	+ * Holding small numbers of low-order mTHP folios in per-CPU LRU cache
	393	+ * will be sensible, but nobody has implemented and tested that yet.
	394	+ */
	395	+ return !folio_test_large(folio);
	396	+}
	397	+
388	398	extern atomic_t lru_disable_count;
389	399
390	400	static inline bool lru_cache_disabled(void)

mm/gup.c+2−2

@@ -2307,13 +2307,13 @@ static unsigned long collect_longterm_unpinnable_folios( · 2306 unmodified lines
2307	2307	continue;
2308	2308	}
2309	2309
2310		− if (drained == 0 &&
	2310	+ if (drained == 0 && folio_may_be_lru_cached(folio) &&
2311	2311	folio_ref_count(folio) !=
2312	2312	folio_expected_ref_count(folio) + 1) {
2313	2313	lru_add_drain();
2314	2314	drained = 1;
2315	2315	}
2316		− if (drained == 1 &&
	2316	+ if (drained == 1 && folio_may_be_lru_cached(folio) &&
2317	2317	folio_ref_count(folio) !=
2318	2318	folio_expected_ref_count(folio) + 1) {
2319	2319	lru_add_drain_all();

mm/mlock.c+3−3

@@ -255,7 +255,7 @@ void mlock_folio(struct folio *folio) · 254 unmodified lines
255	255
256	256	folio_get(folio);
257	257	if (!folio_batch_add(fbatch, mlock_lru(folio)) \|\|
258		− folio_test_large(folio) \|\| lru_cache_disabled())
	258	+ !folio_may_be_lru_cached(folio) \|\| lru_cache_disabled())
259	259	mlock_folio_batch(fbatch);
260	260	local_unlock(&mlock_fbatch.lock);
261	261	}
@@ -278,7 +278,7 @@ void mlock_new_folio(struct folio *folio) · 16 unmodified lines
278	278
279	279	folio_get(folio);
280	280	if (!folio_batch_add(fbatch, mlock_new(folio)) \|\|
281		− folio_test_large(folio) \|\| lru_cache_disabled())
	281	+ !folio_may_be_lru_cached(folio) \|\| lru_cache_disabled())
282	282	mlock_folio_batch(fbatch);
283	283	local_unlock(&mlock_fbatch.lock);
284	284	}
@@ -299,7 +299,7 @@ void munlock_folio(struct folio *folio) · 14 unmodified lines
299	299	*/
300	300	folio_get(folio);
301	301	if (!folio_batch_add(fbatch, folio) \|\|
302		− folio_test_large(folio) \|\| lru_cache_disabled())
	302	+ !folio_may_be_lru_cached(folio) \|\| lru_cache_disabled())
303	303	mlock_folio_batch(fbatch);
304	304	local_unlock(&mlock_fbatch.lock);
305	305	}

mm/swap.c+1−1

@@ -192,7 +192,7 @@ static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch, · 191 unmodified lines
192	192	local_lock(&cpu_fbatches.lock);
193	193
194	194	if (!folio_batch_add(this_cpu_ptr(fbatch), folio) \|\|
195		− folio_test_large(folio) \|\| lru_cache_disabled())
	195	+ !folio_may_be_lru_cached(folio) \|\| lru_cache_disabled())
196	196	folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn);
197	197
198	198	if (disable_irq)