find_lock_entries() already only returned the head page of folios, so convert it to return a folio_batch instead of a pagevec. That cascades through converting truncate_inode_pages_range() to delete_from_page_cache_batch() and page_cache_delete_batch().
Unified · Split
| @@ -28,6 +28,8 @@ · 27 unmodified lines | ||
| 28 | 28 | #include <linux/fscrypt.h> |
| 29 | 29 | #include <linux/fsverity.h> |
| 30 | 30 | |
| 31 | +struct pagevec; | |
| 32 | + | |
| 31 | 33 | #ifdef CONFIG_F2FS_CHECK_FS |
| 32 | 34 | #define f2fs_bug_on(sbi, condition) BUG_ON(condition) |
| 33 | 35 | #else |
| @@ -16,7 +16,7 @@ · 15 unmodified lines | ||
| 16 | 16 | #include <linux/hardirq.h> /* for in_interrupt() */ |
| 17 | 17 | #include <linux/hugetlb_inline.h> |
| 18 | 18 | |
| 19 | −struct pagevec; | |
| 19 | +struct folio_batch; | |
| 20 | 20 | |
| 21 | 21 | static inline bool mapping_empty(struct address_space *mapping) |
| 22 | 22 | { |
| @@ -936,7 +936,7 @@ static inline void __delete_from_page_cache(struct page *page, void *shadow) · 913 unmodified lines | ||
| 936 | 936 | } |
| 937 | 937 | void replace_page_cache_page(struct page *old, struct page *new); |
| 938 | 938 | void delete_from_page_cache_batch(struct address_space *mapping, |
| 939 | − struct pagevec *pvec); | |
| 939 | + struct folio_batch *fbatch); | |
| 940 | 940 | int try_to_release_page(struct page *page, gfp_t gfp); |
| 941 | 941 | bool filemap_release_folio(struct folio *folio, gfp_t gfp); |
| 942 | 942 | loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end, |
| @@ -270,30 +270,29 @@ void filemap_remove_folio(struct folio *folio) · 269 unmodified lines | ||
| 270 | 270 | } |
| 271 | 271 | |
| 272 | 272 | /* |
| 273 | − * page_cache_delete_batch - delete several pages from page cache | |
| 274 | − * @mapping: the mapping to which pages belong | |
| 275 | − * @pvec: pagevec with pages to delete | |
| 273 | + * page_cache_delete_batch - delete several folios from page cache | |
| 274 | + * @mapping: the mapping to which folios belong | |
| 275 | + * @fbatch: batch of folios to delete | |
| 276 | 276 | * |
| 277 | − * The function walks over mapping->i_pages and removes pages passed in @pvec | |
| 278 | − * from the mapping. The function expects @pvec to be sorted by page index | |
| 279 | − * and is optimised for it to be dense. | |
| 280 | − * It tolerates holes in @pvec (mapping entries at those indices are not | |
| 281 | − * modified). The function expects only THP head pages to be present in the | |
| 282 | − * @pvec. | |
| 277 | + * The function walks over mapping->i_pages and removes folios passed in | |
| 278 | + * @fbatch from the mapping. The function expects @fbatch to be sorted | |
| 279 | + * by page index and is optimised for it to be dense. | |
| 280 | + * It tolerates holes in @fbatch (mapping entries at those indices are not | |
| 281 | + * modified). | |
| 283 | 282 | * |
| 284 | 283 | * The function expects the i_pages lock to be held. |
| 285 | 284 | */ |
| 286 | 285 | static void page_cache_delete_batch(struct address_space *mapping, |
| 287 | − struct pagevec *pvec) | |
| 286 | + struct folio_batch *fbatch) | |
| 288 | 287 | { |
| 289 | − XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index); | |
| 288 | + XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); | |
| 290 | 289 | int total_pages = 0; |
| 291 | 290 | int i = 0; |
| 292 | 291 | struct folio *folio; |
| 293 | 292 | |
| 294 | 293 | mapping_set_update(&xas, mapping); |
| 295 | 294 | xas_for_each(&xas, folio, ULONG_MAX) { |
| 296 | − if (i >= pagevec_count(pvec)) | |
| 295 | + if (i >= folio_batch_count(fbatch)) | |
| 297 | 296 | break; |
| 298 | 297 | |
| 299 | 298 | /* A swap/dax/shadow entry got inserted? Skip it. */ |
| @@ -306,9 +305,9 @@ static void page_cache_delete_batch(struct address_space *mapping, · 6 unmodified lines | ||
| 306 | 305 | * means our page has been removed, which shouldn't be |
| 307 | 306 | * possible because we're holding the PageLock. |
| 308 | 307 | */ |
| 309 | − if (&folio->page != pvec->pages[i]) { | |
| 308 | + if (folio != fbatch->folios[i]) { | |
| 310 | 309 | VM_BUG_ON_FOLIO(folio->index > |
| 311 | − pvec->pages[i]->index, folio); | |
| 310 | + fbatch->folios[i]->index, folio); | |
| 312 | 311 | continue; |
| 313 | 312 | } |
| 314 | 313 | |
| @@ -316,12 +315,11 @@ static void page_cache_delete_batch(struct address_space *mapping, · 1 unmodified line | ||
| 316 | 315 | |
| 317 | 316 | if (folio->index == xas.xa_index) |
| 318 | 317 | folio->mapping = NULL; |
| 319 | − /* Leave page->index set: truncation lookup relies on it */ | |
| 318 | + /* Leave folio->index set: truncation lookup relies on it */ | |
| 320 | 319 | |
| 321 | 320 | /* |
| 322 | − * Move to the next page in the vector if this is a regular | |
| 323 | − * page or the index is of the last sub-page of this compound | |
| 324 | − * page. | |
| 321 | + * Move to the next folio in the batch if this is a regular | |
| 322 | + * folio or the index is of the last sub-page of this folio. | |
| 325 | 323 | */ |
| 326 | 324 | if (folio->index + folio_nr_pages(folio) - 1 == xas.xa_index) |
| 327 | 325 | i++; |
| @@ -332,29 +330,29 @@ static void page_cache_delete_batch(struct address_space *mapping, · 4 unmodified lines | ||
| 332 | 330 | } |
| 333 | 331 | |
| 334 | 332 | void delete_from_page_cache_batch(struct address_space *mapping, |
| 335 | − struct pagevec *pvec) | |
| 333 | + struct folio_batch *fbatch) | |
| 336 | 334 | { |
| 337 | 335 | int i; |
| 338 | 336 | |
| 339 | − if (!pagevec_count(pvec)) | |
| 337 | + if (!folio_batch_count(fbatch)) | |
| 340 | 338 | return; |
| 341 | 339 | |
| 342 | 340 | spin_lock(&mapping->host->i_lock); |
| 343 | 341 | xa_lock_irq(&mapping->i_pages); |
| 344 | − for (i = 0; i < pagevec_count(pvec); i++) { | |
| 345 | − struct folio *folio = page_folio(pvec->pages[i]); | |
| 342 | + for (i = 0; i < folio_batch_count(fbatch); i++) { | |
| 343 | + struct folio *folio = fbatch->folios[i]; | |
| 346 | 344 | |
| 347 | 345 | trace_mm_filemap_delete_from_page_cache(folio); |
| 348 | 346 | filemap_unaccount_folio(mapping, folio); |
| 349 | 347 | } |
| 350 | − page_cache_delete_batch(mapping, pvec); | |
| 348 | + page_cache_delete_batch(mapping, fbatch); | |
| 351 | 349 | xa_unlock_irq(&mapping->i_pages); |
| 352 | 350 | if (mapping_shrinkable(mapping)) |
| 353 | 351 | inode_add_lru(mapping->host); |
| 354 | 352 | spin_unlock(&mapping->host->i_lock); |
| 355 | 353 | |
| 356 | − for (i = 0; i < pagevec_count(pvec); i++) | |
| 357 | − filemap_free_folio(mapping, page_folio(pvec->pages[i])); | |
| 354 | + for (i = 0; i < folio_batch_count(fbatch); i++) | |
| 355 | + filemap_free_folio(mapping, fbatch->folios[i]); | |
| 358 | 356 | } |
| 359 | 357 | |
| 360 | 358 | int filemap_check_errors(struct address_space *mapping) |
| @@ -2052,8 +2050,8 @@ unsigned find_get_entries(struct address_space *mapping, pgoff_t start, · 1691 unmodified lines | ||
| 2052 | 2050 | * @mapping: The address_space to search. |
| 2053 | 2051 | * @start: The starting page cache index. |
| 2054 | 2052 | * @end: The final page index (inclusive). |
| 2055 | − * @pvec: Where the resulting entries are placed. | |
| 2056 | − * @indices: The cache indices of the entries in @pvec. | |
| 2053 | + * @fbatch: Where the resulting entries are placed. | |
| 2054 | + * @indices: The cache indices of the entries in @fbatch. | |
| 2057 | 2055 | * |
| 2058 | 2056 | * find_lock_entries() will return a batch of entries from @mapping. |
| 2059 | 2057 | * Swap, shadow and DAX entries are included. Folios are returned |
| @@ -2068,7 +2066,7 @@ unsigned find_get_entries(struct address_space *mapping, pgoff_t start, · 8 unmodified lines | ||
| 2068 | 2066 | * Return: The number of entries which were found. |
| 2069 | 2067 | */ |
| 2070 | 2068 | unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, |
| 2071 | − pgoff_t end, struct pagevec *pvec, pgoff_t *indices) | |
| 2069 | + pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) | |
| 2072 | 2070 | { |
| 2073 | 2071 | XA_STATE(xas, &mapping->i_pages, start); |
| 2074 | 2072 | struct folio *folio; |
| @@ -2088,8 +2086,8 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, · 13 unmodified lines | ||
| 2088 | 2086 | VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index), |
| 2089 | 2087 | folio); |
| 2090 | 2088 | } |
| 2091 | − indices[pvec->nr] = xas.xa_index; | |
| 2092 | − if (!pagevec_add(pvec, &folio->page)) | |
| 2089 | + indices[fbatch->nr] = xas.xa_index; | |
| 2090 | + if (!folio_batch_add(fbatch, folio)) | |
| 2093 | 2091 | break; |
| 2094 | 2092 | goto next; |
| 2095 | 2093 | unlock: |
| @@ -2106,7 +2104,7 @@ next: · 10 unmodified lines | ||
| 2106 | 2104 | } |
| 2107 | 2105 | rcu_read_unlock(); |
| 2108 | 2106 | |
| 2109 | − return pagevec_count(pvec); | |
| 2107 | + return folio_batch_count(fbatch); | |
| 2110 | 2108 | } |
| 2111 | 2109 | |
| 2112 | 2110 | /** |
| @@ -93,7 +93,7 @@ static inline void force_page_cache_readahead(struct address_space *mapping, · 92 unmodified lines | ||
| 93 | 93 | } |
| 94 | 94 | |
| 95 | 95 | unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, |
| 96 | − pgoff_t end, struct pagevec *pvec, pgoff_t *indices); | |
| 96 | + pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); | |
| 97 | 97 | unsigned find_get_entries(struct address_space *mapping, pgoff_t start, |
| 98 | 98 | pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); |
| 99 | 99 | void filemap_free_folio(struct address_space *mapping, struct folio *folio); |
| @@ -919,7 +919,6 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, · 918 unmodified lines | ||
| 919 | 919 | pgoff_t end = (lend + 1) >> PAGE_SHIFT; |
| 920 | 920 | unsigned int partial_start = lstart & (PAGE_SIZE - 1); |
| 921 | 921 | unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); |
| 922 | − struct pagevec pvec; | |
| 923 | 922 | struct folio_batch fbatch; |
| 924 | 923 | pgoff_t indices[PAGEVEC_SIZE]; |
| 925 | 924 | long nr_swaps_freed = 0; |
| @@ -932,12 +931,12 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, · 6 unmodified lines | ||
| 932 | 931 | if (info->fallocend > start && info->fallocend <= end && !unfalloc) |
| 933 | 932 | info->fallocend = start; |
| 934 | 933 | |
| 935 | − pagevec_init(&pvec); | |
| 934 | + folio_batch_init(&fbatch); | |
| 936 | 935 | index = start; |
| 937 | 936 | while (index < end && find_lock_entries(mapping, index, end - 1, |
| 938 | − &pvec, indices)) { | |
| 939 | − for (i = 0; i < pagevec_count(&pvec); i++) { | |
| 940 | − struct folio *folio = (struct folio *)pvec.pages[i]; | |
| 937 | + &fbatch, indices)) { | |
| 938 | + for (i = 0; i < folio_batch_count(&fbatch); i++) { | |
| 939 | + struct folio *folio = fbatch.folios[i]; | |
| 941 | 940 | |
| 942 | 941 | index = indices[i]; |
| 943 | 942 | |
| @@ -954,8 +953,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, · 10 unmodified lines | ||
| 954 | 953 | truncate_inode_folio(mapping, folio); |
| 955 | 954 | folio_unlock(folio); |
| 956 | 955 | } |
| 957 | − pagevec_remove_exceptionals(&pvec); | |
| 958 | − pagevec_release(&pvec); | |
| 956 | + folio_batch_remove_exceptionals(&fbatch); | |
| 957 | + folio_batch_release(&fbatch); | |
| 959 | 958 | cond_resched(); |
| 960 | 959 | index++; |
| 961 | 960 | } |
| @@ -988,7 +987,6 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, · 26 unmodified lines | ||
| 988 | 987 | if (start >= end) |
| 989 | 988 | return; |
| 990 | 989 | |
| 991 | − folio_batch_init(&fbatch); | |
| 992 | 990 | index = start; |
| 993 | 991 | while (index < end) { |
| 994 | 992 | cond_resched(); |
| @@ -56,11 +56,11 @@ static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, · 55 unmodified lines | ||
| 56 | 56 | |
| 57 | 57 | /* |
| 58 | 58 | * Unconditionally remove exceptional entries. Usually called from truncate |
| 59 | − * path. Note that the pagevec may be altered by this function by removing | |
| 59 | + * path. Note that the folio_batch may be altered by this function by removing | |
| 60 | 60 | * exceptional entries similar to what pagevec_remove_exceptionals does. |
| 61 | 61 | */ |
| 62 | −static void truncate_exceptional_pvec_entries(struct address_space *mapping, | |
| 63 | − struct pagevec *pvec, pgoff_t *indices) | |
| 62 | +static void truncate_folio_batch_exceptionals(struct address_space *mapping, | |
| 63 | + struct folio_batch *fbatch, pgoff_t *indices) | |
| 64 | 64 | { |
| 65 | 65 | int i, j; |
| 66 | 66 | bool dax; |
| @@ -69,11 +69,11 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, · 2 unmodified lines | ||
| 69 | 69 | if (shmem_mapping(mapping)) |
| 70 | 70 | return; |
| 71 | 71 | |
| 72 | − for (j = 0; j < pagevec_count(pvec); j++) | |
| 73 | − if (xa_is_value(pvec->pages[j])) | |
| 72 | + for (j = 0; j < folio_batch_count(fbatch); j++) | |
| 73 | + if (xa_is_value(fbatch->folios[j])) | |
| 74 | 74 | break; |
| 75 | 75 | |
| 76 | − if (j == pagevec_count(pvec)) | |
| 76 | + if (j == folio_batch_count(fbatch)) | |
| 77 | 77 | return; |
| 78 | 78 | |
| 79 | 79 | dax = dax_mapping(mapping); |
| @@ -82,12 +82,12 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, · 2 unmodified lines | ||
| 82 | 82 | xa_lock_irq(&mapping->i_pages); |
| 83 | 83 | } |
| 84 | 84 | |
| 85 | − for (i = j; i < pagevec_count(pvec); i++) { | |
| 86 | − struct page *page = pvec->pages[i]; | |
| 85 | + for (i = j; i < folio_batch_count(fbatch); i++) { | |
| 86 | + struct folio *folio = fbatch->folios[i]; | |
| 87 | 87 | pgoff_t index = indices[i]; |
| 88 | 88 | |
| 89 | − if (!xa_is_value(page)) { | |
| 90 | − pvec->pages[j++] = page; | |
| 89 | + if (!xa_is_value(folio)) { | |
| 90 | + fbatch->folios[j++] = folio; | |
| 91 | 91 | continue; |
| 92 | 92 | } |
| 93 | 93 | |
| @@ -96,7 +96,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, · 2 unmodified lines | ||
| 96 | 96 | continue; |
| 97 | 97 | } |
| 98 | 98 | |
| 99 | − __clear_shadow_entry(mapping, index, page); | |
| 99 | + __clear_shadow_entry(mapping, index, folio); | |
| 100 | 100 | } |
| 101 | 101 | |
| 102 | 102 | if (!dax) { |
| @@ -105,14 +105,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, · 2 unmodified lines | ||
| 105 | 105 | inode_add_lru(mapping->host); |
| 106 | 106 | spin_unlock(&mapping->host->i_lock); |
| 107 | 107 | } |
| 108 | − pvec->nr = j; | |
| 109 | −} | |
| 110 | − | |
| 111 | −static void truncate_folio_batch_exceptionals(struct address_space *mapping, | |
| 112 | − struct folio_batch *fbatch, pgoff_t *indices) | |
| 113 | −{ | |
| 114 | − truncate_exceptional_pvec_entries(mapping, (struct pagevec *)fbatch, | |
| 115 | − indices); | |
| 108 | + fbatch->nr = j; | |
| 116 | 109 | } |
| 117 | 110 | |
| 118 | 111 | /* |
| @@ -303,7 +296,6 @@ void truncate_inode_pages_range(struct address_space *mapping, · 184 unmodified lines | ||
| 303 | 296 | pgoff_t end; /* exclusive */ |
| 304 | 297 | unsigned int partial_start; /* inclusive */ |
| 305 | 298 | unsigned int partial_end; /* exclusive */ |
| 306 | − struct pagevec pvec; | |
| 307 | 299 | struct folio_batch fbatch; |
| 308 | 300 | pgoff_t indices[PAGEVEC_SIZE]; |
| 309 | 301 | pgoff_t index; |
| @@ -333,18 +325,18 @@ void truncate_inode_pages_range(struct address_space *mapping, · 23 unmodified lines | ||
| 333 | 325 | else |
| 334 | 326 | end = (lend + 1) >> PAGE_SHIFT; |
| 335 | 327 | |
| 336 | − pagevec_init(&pvec); | |
| 328 | + folio_batch_init(&fbatch); | |
| 337 | 329 | index = start; |
| 338 | 330 | while (index < end && find_lock_entries(mapping, index, end - 1, |
| 339 | − &pvec, indices)) { | |
| 340 | − index = indices[pagevec_count(&pvec) - 1] + 1; | |
| 341 | − truncate_exceptional_pvec_entries(mapping, &pvec, indices); | |
| 342 | − for (i = 0; i < pagevec_count(&pvec); i++) | |
| 343 | − truncate_cleanup_folio(page_folio(pvec.pages[i])); | |
| 344 | − delete_from_page_cache_batch(mapping, &pvec); | |
| 345 | − for (i = 0; i < pagevec_count(&pvec); i++) | |
| 346 | − unlock_page(pvec.pages[i]); | |
| 347 | − pagevec_release(&pvec); | |
| 331 | + &fbatch, indices)) { | |
| 332 | + index = indices[folio_batch_count(&fbatch) - 1] + 1; | |
| 333 | + truncate_folio_batch_exceptionals(mapping, &fbatch, indices); | |
| 334 | + for (i = 0; i < folio_batch_count(&fbatch); i++) | |
| 335 | + truncate_cleanup_folio(fbatch.folios[i]); | |
| 336 | + delete_from_page_cache_batch(mapping, &fbatch); | |
| 337 | + for (i = 0; i < folio_batch_count(&fbatch); i++) | |
| 338 | + folio_unlock(fbatch.folios[i]); | |
| 339 | + folio_batch_release(&fbatch); | |
| 348 | 340 | cond_resched(); |
| 349 | 341 | } |
| 350 | 342 | |
| @@ -387,7 +379,6 @@ void truncate_inode_pages_range(struct address_space *mapping, · 36 unmodified lines | ||
| 387 | 379 | if (start >= end) |
| 388 | 380 | goto out; |
| 389 | 381 | |
| 390 | − folio_batch_init(&fbatch); | |
| 391 | 382 | index = start; |
| 392 | 383 | for ( ; ; ) { |
| 393 | 384 | cond_resched(); |
| @@ -489,16 +480,16 @@ static unsigned long __invalidate_mapping_pages(struct address_space *mapping, · 95 unmodified lines | ||
| 489 | 480 | pgoff_t start, pgoff_t end, unsigned long *nr_pagevec) |
| 490 | 481 | { |
| 491 | 482 | pgoff_t indices[PAGEVEC_SIZE]; |
| 492 | − struct pagevec pvec; | |
| 483 | + struct folio_batch fbatch; | |
| 493 | 484 | pgoff_t index = start; |
| 494 | 485 | unsigned long ret; |
| 495 | 486 | unsigned long count = 0; |
| 496 | 487 | int i; |
| 497 | 488 | |
| 498 | − pagevec_init(&pvec); | |
| 499 | − while (find_lock_entries(mapping, index, end, &pvec, indices)) { | |
| 500 | − for (i = 0; i < pagevec_count(&pvec); i++) { | |
| 501 | − struct page *page = pvec.pages[i]; | |
| 489 | + folio_batch_init(&fbatch); | |
| 490 | + while (find_lock_entries(mapping, index, end, &fbatch, indices)) { | |
| 491 | + for (i = 0; i < folio_batch_count(&fbatch); i++) { | |
| 492 | + struct page *page = &fbatch.folios[i]->page; | |
| 502 | 493 | |
| 503 | 494 | /* We rely upon deletion not changing page->index */ |
| 504 | 495 | index = indices[i]; |
| @@ -525,8 +516,8 @@ static unsigned long __invalidate_mapping_pages(struct address_space *mapping, · 20 unmodified lines | ||
| 525 | 516 | } |
| 526 | 517 | count += ret; |
| 527 | 518 | } |
| 528 | − pagevec_remove_exceptionals(&pvec); | |
| 529 | − pagevec_release(&pvec); | |
| 519 | + folio_batch_remove_exceptionals(&fbatch); | |
| 520 | + folio_batch_release(&fbatch); | |
| 530 | 521 | cond_resched(); |
| 531 | 522 | index++; |
| 532 | 523 | } |