mm: Convert find_lock_entries() to use a folio_batch

find_lock_entries() already only returned the head page of folios, so convert it to return a folio_batch instead of a pagevec. That cascades through converting truncate_inode_pages_range() to delete_from_page_cache_batch() and page_cache_delete_batch().

Matthew Wilcox (Oracle) committed 4 years ago

Commit 51dcbda1 parent 0e499ed6 files · +69 −80Browse files

Unified · Split

6 files

fs/f2fs/f2fs.h +2 −0
include/linux/pagemap.h +2 −2
mm/filemap.c +29 −31
mm/internal.h +1 −1
mm/shmem.c +6 −8
mm/truncate.c +29 −38

fs/f2fs/f2fs.h+2−0

@@ -28,6 +28,8 @@ · 27 unmodified lines
28	#include <linux/fscrypt.h>	28	#include <linux/fscrypt.h>
29	#include <linux/fsverity.h>	29	#include <linux/fsverity.h>
30		30
		31	struct pagevec;
		32
31	#ifdef CONFIG_F2FS_CHECK_FS	33	#ifdef CONFIG_F2FS_CHECK_FS
32	#define f2fs_bug_on(sbi, condition) BUG_ON(condition)	34	#define f2fs_bug_on(sbi, condition) BUG_ON(condition)
33	#else	35	#else

include/linux/pagemap.h+2−2

@@ -16,7 +16,7 @@ · 15 unmodified lines
16	#include <linux/hardirq.h> /* for in_interrupt() */	16	#include <linux/hardirq.h> /* for in_interrupt() */
17	#include <linux/hugetlb_inline.h>	17	#include <linux/hugetlb_inline.h>
18		18
19	struct pagevec;	19	struct folio_batch;
20		20
21	static inline bool mapping_empty(struct address_space *mapping)	21	static inline bool mapping_empty(struct address_space *mapping)
22	{	22	{
@@ -936,7 +936,7 @@ static inline void __delete_from_page_cache(struct page page, void shadow) · 913 unmodified lines
936	}	936	}
937	void replace_page_cache_page(struct page old, struct page new);	937	void replace_page_cache_page(struct page old, struct page new);
938	void delete_from_page_cache_batch(struct address_space *mapping,	938	void delete_from_page_cache_batch(struct address_space *mapping,
939	struct pagevec *pvec);	939	struct folio_batch *fbatch);
940	int try_to_release_page(struct page *page, gfp_t gfp);	940	int try_to_release_page(struct page *page, gfp_t gfp);
941	bool filemap_release_folio(struct folio *folio, gfp_t gfp);	941	bool filemap_release_folio(struct folio *folio, gfp_t gfp);
942	loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end,	942	loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end,

mm/filemap.c+29−31

 }
 /*
- * page_cache_delete_batch - delete several pages from page cache
+ * page_cache_delete_batch - delete several folios from page cache
- * @mapping: the mapping to which pages belong
+ * @mapping: the mapping to which folios belong
- * @pvec: pagevec with pages to delete
+ * @fbatch: batch of folios to delete
  *
- * The function walks over mapping->i_pages and removes pages passed in @pvec
+ * The function walks over mapping->i_pages and removes folios passed in
- * from the mapping. The function expects @pvec to be sorted by page index
+ * @fbatch from the mapping. The function expects @fbatch to be sorted
- * and is optimised for it to be dense.
+ * by page index and is optimised for it to be dense.
- * It tolerates holes in @pvec (mapping entries at those indices are not
+ * It tolerates holes in @fbatch (mapping entries at those indices are not
- * modified). The function expects only THP head pages to be present in the
+ * modified).
- * @pvec.
  *
  * The function expects the i_pages lock to be held.
  */
 static void page_cache_delete_batch(struct address_space *mapping,
-			     struct pagevec *pvec)
+			     struct folio_batch *fbatch)
 {
-	XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index);
+	XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index);
 	int total_pages = 0;
 	int i = 0;
 	struct folio *folio;
 	mapping_set_update(&xas, mapping);
 	xas_for_each(&xas, folio, ULONG_MAX) {
-		if (i >= pagevec_count(pvec))
+		if (i >= folio_batch_count(fbatch))
 			break;
 		/* A swap/dax/shadow entry got inserted? Skip it. */
 		 * means our page has been removed, which shouldn't be
 		 * possible because we're holding the PageLock.
 		 */
-		if (&folio->page != pvec->pages[i]) {
+		if (folio != fbatch->folios[i]) {
 			VM_BUG_ON_FOLIO(folio->index >
-						pvec->pages[i]->index, folio);
+					fbatch->folios[i]->index, folio);
 			continue;
 		}
 		if (folio->index == xas.xa_index)
 			folio->mapping = NULL;
-		/* Leave page->index set: truncation lookup relies on it */
+		/* Leave folio->index set: truncation lookup relies on it */
 		/*
-		 * Move to the next page in the vector if this is a regular
+		 * Move to the next folio in the batch if this is a regular
-		 * page or the index is of the last sub-page of this compound
+		 * folio or the index is of the last sub-page of this folio.
-		 * page.
 		 */
 		if (folio->index + folio_nr_pages(folio) - 1 == xas.xa_index)
 			i++;
 }
 void delete_from_page_cache_batch(struct address_space *mapping,
-				  struct pagevec *pvec)
+				  struct folio_batch *fbatch)
 {
 	int i;
-	if (!pagevec_count(pvec))
+	if (!folio_batch_count(fbatch))
 		return;
 	spin_lock(&mapping->host->i_lock);
 	xa_lock_irq(&mapping->i_pages);
-	for (i = 0; i < pagevec_count(pvec); i++) {
+	for (i = 0; i < folio_batch_count(fbatch); i++) {
-		struct folio *folio = page_folio(pvec->pages[i]);
+		struct folio *folio = fbatch->folios[i];
 		trace_mm_filemap_delete_from_page_cache(folio);
 		filemap_unaccount_folio(mapping, folio);
 	}
-	page_cache_delete_batch(mapping, pvec);
+	page_cache_delete_batch(mapping, fbatch);
 	xa_unlock_irq(&mapping->i_pages);
 	if (mapping_shrinkable(mapping))
 		inode_add_lru(mapping->host);
 	spin_unlock(&mapping->host->i_lock);
-	for (i = 0; i < pagevec_count(pvec); i++)
+	for (i = 0; i < folio_batch_count(fbatch); i++)
-		filemap_free_folio(mapping, page_folio(pvec->pages[i]));
+		filemap_free_folio(mapping, fbatch->folios[i]);
 }
 int filemap_check_errors(struct address_space *mapping)
  * @mapping:	The address_space to search.
  * @start:	The starting page cache index.
  * @end:	The final page index (inclusive).
- * @pvec:	Where the resulting entries are placed.
+ * @fbatch:	Where the resulting entries are placed.
- * @indices:	The cache indices of the entries in @pvec.
+ * @indices:	The cache indices of the entries in @fbatch.
  *
  * find_lock_entries() will return a batch of entries from @mapping.
  * Swap, shadow and DAX entries are included.  Folios are returned
  * Return: The number of entries which were found.
  */
 unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
-		pgoff_t end, struct pagevec *pvec, pgoff_t *indices)
+		pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices)
 {
 	XA_STATE(xas, &mapping->i_pages, start);
 	struct folio *folio;
 			VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
 					folio);
 		}
-		indices[pvec->nr] = xas.xa_index;
+		indices[fbatch->nr] = xas.xa_index;
-		if (!pagevec_add(pvec, &folio->page))
+		if (!folio_batch_add(fbatch, folio))
 			break;
 		goto next;
 unlock:
 	}
 	rcu_read_unlock();
-	return pagevec_count(pvec);
+	return folio_batch_count(fbatch);
 }
 /**

mm/internal.h+1−1

@@ -93,7 +93,7 @@ static inline void force_page_cache_readahead(struct address_space *mapping, · 92 unmodified lines
93	}	93	}
94		94
95	unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,	95	unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
96	pgoff_t end, struct pagevec pvec, pgoff_t indices);	96	pgoff_t end, struct folio_batch fbatch, pgoff_t indices);
97	unsigned find_get_entries(struct address_space *mapping, pgoff_t start,	97	unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
98	pgoff_t end, struct folio_batch fbatch, pgoff_t indices);	98	pgoff_t end, struct folio_batch fbatch, pgoff_t indices);
99	void filemap_free_folio(struct address_space mapping, struct folio folio);	99	void filemap_free_folio(struct address_space mapping, struct folio folio);

mm/shmem.c+6−8

 	pgoff_t end = (lend + 1) >> PAGE_SHIFT;
 	unsigned int partial_start = lstart & (PAGE_SIZE - 1);
 	unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
-	struct pagevec pvec;
 	struct folio_batch fbatch;
 	pgoff_t indices[PAGEVEC_SIZE];
 	long nr_swaps_freed = 0;
 	if (info->fallocend > start && info->fallocend <= end && !unfalloc)
 		info->fallocend = start;
-	pagevec_init(&pvec);
+	folio_batch_init(&fbatch);
 	index = start;
 	while (index < end && find_lock_entries(mapping, index, end - 1,
-			&pvec, indices)) {
+			&fbatch, indices)) {
-		for (i = 0; i < pagevec_count(&pvec); i++) {
+		for (i = 0; i < folio_batch_count(&fbatch); i++) {
-			struct folio *folio = (struct folio *)pvec.pages[i];
+			struct folio *folio = fbatch.folios[i];
 			index = indices[i];
 				truncate_inode_folio(mapping, folio);
 			folio_unlock(folio);
 		}
-		pagevec_remove_exceptionals(&pvec);
+		folio_batch_remove_exceptionals(&fbatch);
-		pagevec_release(&pvec);
+		folio_batch_release(&fbatch);
 		cond_resched();
 		index++;
 	}
 	if (start >= end)
 		return;
-	folio_batch_init(&fbatch);
 	index = start;
 	while (index < end) {
 		cond_resched();

mm/truncate.c+29−38

 /*
  * Unconditionally remove exceptional entries. Usually called from truncate
- * path. Note that the pagevec may be altered by this function by removing
+ * path. Note that the folio_batch may be altered by this function by removing
  * exceptional entries similar to what pagevec_remove_exceptionals does.
  */
-static void truncate_exceptional_pvec_entries(struct address_space *mapping,
+static void truncate_folio_batch_exceptionals(struct address_space *mapping,
-				struct pagevec *pvec, pgoff_t *indices)
+				struct folio_batch *fbatch, pgoff_t *indices)
 {
 	int i, j;
 	bool dax;
 	if (shmem_mapping(mapping))
 		return;
-	for (j = 0; j < pagevec_count(pvec); j++)
+	for (j = 0; j < folio_batch_count(fbatch); j++)
-		if (xa_is_value(pvec->pages[j]))
+		if (xa_is_value(fbatch->folios[j]))
 			break;
-	if (j == pagevec_count(pvec))
+	if (j == folio_batch_count(fbatch))
 		return;
 	dax = dax_mapping(mapping);
 		xa_lock_irq(&mapping->i_pages);
 	}
-	for (i = j; i < pagevec_count(pvec); i++) {
+	for (i = j; i < folio_batch_count(fbatch); i++) {
-		struct page *page = pvec->pages[i];
+		struct folio *folio = fbatch->folios[i];
 		pgoff_t index = indices[i];
-		if (!xa_is_value(page)) {
+		if (!xa_is_value(folio)) {
-			pvec->pages[j++] = page;
+			fbatch->folios[j++] = folio;
 			continue;
 		}
 			continue;
 		}
-		__clear_shadow_entry(mapping, index, page);
+		__clear_shadow_entry(mapping, index, folio);
 	}
 	if (!dax) {
 			inode_add_lru(mapping->host);
 		spin_unlock(&mapping->host->i_lock);
 	}
-	pvec->nr = j;
+	fbatch->nr = j;
-}
-static void truncate_folio_batch_exceptionals(struct address_space *mapping,
-				struct folio_batch *fbatch, pgoff_t *indices)
-{
-	truncate_exceptional_pvec_entries(mapping, (struct pagevec *)fbatch,
-						indices);
 }
 /*
 	pgoff_t		end;		/* exclusive */
 	unsigned int	partial_start;	/* inclusive */
 	unsigned int	partial_end;	/* exclusive */
-	struct pagevec	pvec;
 	struct folio_batch fbatch;
 	pgoff_t		indices[PAGEVEC_SIZE];
 	pgoff_t		index;
 	else
 		end = (lend + 1) >> PAGE_SHIFT;
-	pagevec_init(&pvec);
+	folio_batch_init(&fbatch);
 	index = start;
 	while (index < end && find_lock_entries(mapping, index, end - 1,
-			&pvec, indices)) {
+			&fbatch, indices)) {
-		index = indices[pagevec_count(&pvec) - 1] + 1;
+		index = indices[folio_batch_count(&fbatch) - 1] + 1;
-		truncate_exceptional_pvec_entries(mapping, &pvec, indices);
+		truncate_folio_batch_exceptionals(mapping, &fbatch, indices);
-		for (i = 0; i < pagevec_count(&pvec); i++)
+		for (i = 0; i < folio_batch_count(&fbatch); i++)
-			truncate_cleanup_folio(page_folio(pvec.pages[i]));
+			truncate_cleanup_folio(fbatch.folios[i]);
-		delete_from_page_cache_batch(mapping, &pvec);
+		delete_from_page_cache_batch(mapping, &fbatch);
-		for (i = 0; i < pagevec_count(&pvec); i++)
+		for (i = 0; i < folio_batch_count(&fbatch); i++)
-			unlock_page(pvec.pages[i]);
+			folio_unlock(fbatch.folios[i]);
-		pagevec_release(&pvec);
+		folio_batch_release(&fbatch);
 		cond_resched();
 	}
 	if (start >= end)
 		goto out;
-	folio_batch_init(&fbatch);
 	index = start;
 	for ( ; ; ) {
 		cond_resched();
 		pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
 {
 	pgoff_t indices[PAGEVEC_SIZE];
-	struct pagevec pvec;
+	struct folio_batch fbatch;
 	pgoff_t index = start;
 	unsigned long ret;
 	unsigned long count = 0;
 	int i;
-	pagevec_init(&pvec);
+	folio_batch_init(&fbatch);
-	while (find_lock_entries(mapping, index, end, &pvec, indices)) {
+	while (find_lock_entries(mapping, index, end, &fbatch, indices)) {
-		for (i = 0; i < pagevec_count(&pvec); i++) {
+		for (i = 0; i < folio_batch_count(&fbatch); i++) {
-			struct page *page = pvec.pages[i];
+			struct page *page = &fbatch.folios[i]->page;
 			/* We rely upon deletion not changing page->index */
 			index = indices[i];
 			}
 			count += ret;
 		}
-		pagevec_remove_exceptionals(&pvec);
+		folio_batch_remove_exceptionals(&fbatch);
-		pagevec_release(&pvec);
+		folio_batch_release(&fbatch);
 		cond_resched();
 		index++;
 	}

mm: Convert find_lock_entries() to use a folio_batch

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code