// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2008 Oracle. All rights reserved. */ #include <linux/kernel.h>#include <linux/bio.h>#include <linux/file.h>#include <linux/fs.h>#include <linux/pagemap.h>#include <linux/folio_batch.h>#include <linux/highmem.h>#include <linux/kthread.h>#include <linux/time.h>#include <linux/init.h>#include <linux/string.h>#include <linux/backing-dev.h>#include <linux/writeback.h>#include <linux/psi.h>#include <linux/slab.h>#include <linux/sched/mm.h>#include <linux/log2.h>#include <linux/shrinker.h>#include "misc.h"#include "ctree.h"#include "fs.h"#include "btrfs_inode.h"#include "bio.h"#include "ordered-data.h"#include "compression.h"#include "extent_io.h"#include "extent_map.h"#include "subpage.h"#include "messages.h"#include "super.h" static struct bio_set btrfs_compressed_bioset; static const char* const btrfs_compress_types[] = { "", "zlib", "lzo", "zstd" }; const char* btrfs_compress_type2str(enum btrfs_compression_type type){ switch (type) { case BTRFS_COMPRESS_ZLIB: case BTRFS_COMPRESS_LZO: case BTRFS_COMPRESS_ZSTD: case BTRFS_COMPRESS_NONE: return btrfs_compress_types[type]; default: break; } return NULL;} static inline struct compressed_bio *to_compressed_bio(struct btrfs_bio *bbio){ return container_of(bbio, struct compressed_bio, bbio);} static struct compressed_bio *alloc_compressed_bio(struct btrfs_inode *inode, u64 start, blk_opf_t op, btrfs_bio_end_io_t end_io){ struct btrfs_bio *bbio; bbio = btrfs_bio(bio_alloc_bioset(NULL, BTRFS_MAX_COMPRESSED_PAGES, op, GFP_NOFS, &btrfs_compressed_bioset)); btrfs_bio_init(bbio, inode, start, end_io, NULL); return to_compressed_bio(bbio);} bool btrfs_compress_is_valid_type(const char *str, size_t len){ int i; for (i = 1; i < ARRAY_SIZE(btrfs_compress_types); i++) { size_t comp_len = strlen(btrfs_compress_types[i]); if (len < comp_len) continue; if (!strncmp(btrfs_compress_types[i], str, comp_len)) return true; } return false;} static int compression_decompress_bio(struct list_head *ws, struct compressed_bio *cb){ switch (cb->compress_type) { case BTRFS_COMPRESS_ZLIB: return zlib_decompress_bio(ws, cb); case BTRFS_COMPRESS_LZO: return lzo_decompress_bio(ws, cb); case BTRFS_COMPRESS_ZSTD: return zstd_decompress_bio(ws, cb); case BTRFS_COMPRESS_NONE: default: /* * This can't happen, the type is validated several times * before we get here. */ BUG(); }} static int compression_decompress(int type, struct list_head *ws, const u8 *data_in, struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, size_t destlen){ switch (type) { case BTRFS_COMPRESS_ZLIB: return zlib_decompress(ws, data_in, dest_folio, dest_pgoff, srclen, destlen); case BTRFS_COMPRESS_LZO: return lzo_decompress(ws, data_in, dest_folio, dest_pgoff, srclen, destlen); case BTRFS_COMPRESS_ZSTD: return zstd_decompress(ws, data_in, dest_folio, dest_pgoff, srclen, destlen); case BTRFS_COMPRESS_NONE: default: /* * This can't happen, the type is validated several times * before we get here. */ BUG(); }} static int btrfs_decompress_bio(struct compressed_bio *cb); /* * Global cache of last unused pages for compression/decompression. */static struct btrfs_compr_pool { struct shrinker *shrinker; spinlock_t lock; struct list_head list; int count; int thresh;} compr_pool; static unsigned long btrfs_compr_pool_count(struct shrinker *sh, struct shrink_control *sc){ int ret; /* * We must not read the values more than once if 'ret' gets expanded in * the return statement so we don't accidentally return a negative * number, even if the first condition finds it positive. */ ret = READ_ONCE(compr_pool.count) - READ_ONCE(compr_pool.thresh); return ret > 0 ? ret : 0;} static unsigned long btrfs_compr_pool_scan(struct shrinker *sh, struct shrink_control *sc){ LIST_HEAD(remove); struct list_head *tmp, *next; int freed; if (compr_pool.count == 0) return SHRINK_STOP; /* For now, just simply drain the whole list. */ spin_lock(&compr_pool.lock); list_splice_init(&compr_pool.list, &remove); freed = compr_pool.count; compr_pool.count = 0; spin_unlock(&compr_pool.lock); list_for_each_safe(tmp, next, &remove) { struct page *page = list_entry(tmp, struct page, lru); ASSERT(page_ref_count(page) == 1); put_page(page); } return freed;} /* * Common wrappers for page allocation from compression wrappers */struct folio *btrfs_alloc_compr_folio(struct btrfs_fs_info *fs_info, gfp_t gfp){ struct folio *folio = NULL; /* For bs > ps cases, no cached folio pool for now. */ if (fs_info->block_min_order) goto alloc; spin_lock(&compr_pool.lock); if (compr_pool.count > 0) { folio = list_first_entry(&compr_pool.list, struct folio, lru); list_del_init(&folio->lru); compr_pool.count--; } spin_unlock(&compr_pool.lock); if (folio) return folio; alloc: return folio_alloc(gfp, fs_info->block_min_order);} void btrfs_free_compr_folio(struct folio *folio){ bool do_free = false; /* The folio is from bs > ps fs, no cached pool for now. */ if (folio_order(folio)) goto free; spin_lock(&compr_pool.lock); if (compr_pool.count > compr_pool.thresh) { do_free = true; } else { list_add(&folio->lru, &compr_pool.list); compr_pool.count++; } spin_unlock(&compr_pool.lock); if (!do_free) return; free: ASSERT(folio_ref_count(folio) == 1); folio_put(folio);} static void end_bbio_compressed_read(struct btrfs_bio *bbio){ struct compressed_bio *cb = to_compressed_bio(bbio); blk_status_t status = bbio->bio.bi_status; struct folio_iter fi; if (!status) status = errno_to_blk_status(btrfs_decompress_bio(cb)); btrfs_bio_end_io(cb->orig_bbio, status); bio_for_each_folio_all(fi, &bbio->bio) btrfs_free_compr_folio(fi.folio); bio_put(&bbio->bio);} /* * Clear the writeback bits on all of the file * pages for a compressed write */static noinline void end_compressed_writeback(const struct compressed_bio *cb){ struct inode *inode = &cb->bbio.inode->vfs_inode; struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); pgoff_t index = cb->start >> PAGE_SHIFT; const pgoff_t end_index = (cb->start + cb->len - 1) >> PAGE_SHIFT; struct folio_batch fbatch; int i; int ret; ret = blk_status_to_errno(cb->bbio.bio.bi_status); if (ret) mapping_set_error(inode->i_mapping, ret); folio_batch_init(&fbatch); while (index <= end_index) { ret = filemap_get_folios(inode->i_mapping, &index, end_index, &fbatch); if (ret == 0) return; for (i = 0; i < ret; i++) { struct folio *folio = fbatch.folios[i]; btrfs_folio_clamp_clear_writeback(fs_info, folio, cb->start, cb->len); } folio_batch_release(&fbatch); } /* the inode may be gone now */} /* * Do the cleanup once all the compressed pages hit the disk. This will clear * writeback on the file pages and free the compressed pages. * * This also calls the writeback end hooks for the file pages so that metadata * and checksums can be updated in the file. */static void end_bbio_compressed_write(struct btrfs_bio *bbio){ struct compressed_bio *cb = to_compressed_bio(bbio); struct folio_iter fi; btrfs_finish_ordered_extent(cb->bbio.ordered, cb->start, cb->len, cb->bbio.bio.bi_status == BLK_STS_OK); if (cb->writeback) end_compressed_writeback(cb); /* Note, our inode could be gone now. */ bio_for_each_folio_all(fi, &bbio->bio) btrfs_free_compr_folio(fi.folio); bio_put(&cb->bbio.bio);} /* * worker function to build and submit bios for previously compressed pages. * The corresponding pages in the inode should be marked for writeback * and the compressed pages should have a reference on them for dropping * when the IO is complete. * * This also checksums the file bytes and gets things ready for * the end io hooks. */void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered, struct compressed_bio *cb){ struct btrfs_inode *inode = ordered->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; ASSERT(IS_ALIGNED(ordered->file_offset, fs_info->sectorsize)); ASSERT(IS_ALIGNED(ordered->num_bytes, fs_info->sectorsize)); /* * This flag determines if we should clear the writeback flag from the * page cache. But this function is only utilized by encoded writes, it * never goes through the page cache. */ ASSERT(!cb->writeback); cb->start = ordered->file_offset; cb->len = ordered->num_bytes; ASSERT(cb->bbio.bio.bi_iter.bi_size == ordered->disk_num_bytes); cb->bbio.bio.bi_iter.bi_sector = ordered->disk_bytenr >> SECTOR_SHIFT; cb->bbio.ordered = ordered; btrfs_submit_bbio(&cb->bbio, 0);} /* * Allocate a compressed write bio for @inode file offset @start length @len. * * The caller still needs to properly queue all folios and populate involved * members. */struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, u64 start, u64 len){ struct compressed_bio *cb; cb = alloc_compressed_bio(inode, start, REQ_OP_WRITE, end_bbio_compressed_write); cb->start = start; cb->len = len; cb->writeback = false; return cb;} /* * Add extra folios in the same compressed file extent so that we don't need to * re-read the same extent again and again. * * If in the same folio, we have several non-contiguous blocks which are pointing * to the same on-disk compressed data, we will re-read the same extent many * times, as this function can only help cross folio situations. */static noinline int add_ra_bio_folios(struct inode *inode, u64 compressed_end, struct compressed_bio *cb, int *memstall, unsigned long *pflags, bool direct_reclaim){ struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); pgoff_t end_index; struct bio *orig_bio = &cb->orig_bbio->bio; u64 cur = cb->orig_bbio->file_offset + orig_bio->bi_iter.bi_size; u64 isize = i_size_read(inode); int ret; gfp_t constraint_gfp, cache_gfp; struct folio *folio; struct extent_map *em; struct address_space *mapping = inode->i_mapping; struct extent_map_tree *em_tree; struct extent_io_tree *tree; int sectors_missed = 0; em_tree = &BTRFS_I(inode)->extent_tree; tree = &BTRFS_I(inode)->io_tree; if (isize == 0) return 0; /* For bs > ps cases, we don't support readahead for compressed folios for now. */ if (fs_info->block_min_order) return 0; end_index = (i_size_read(inode) - 1) >> PAGE_SHIFT; /* Avoid direct reclaim when the caller does not allow it. */ constraint_gfp = ~__GFP_FS; cache_gfp = GFP_NOFS | __GFP_NOWARN; if (!direct_reclaim) { constraint_gfp &= ~__GFP_DIRECT_RECLAIM; cache_gfp &= ~__GFP_DIRECT_RECLAIM; } while (cur < compressed_end) { u64 folio_end; pgoff_t pg_index = cur >> PAGE_SHIFT; gfp_t masked_constraint_gfp; u32 add_size; if (pg_index > end_index) break; folio = filemap_get_folio(mapping, pg_index); if (!IS_ERR(folio)) { u64 folio_sz = folio_size(folio); u64 offset = offset_in_folio(folio, cur); folio_put(folio); sectors_missed += (folio_sz - offset) >> fs_info->sectorsize_bits; /* Beyond threshold, no need to continue */ if (sectors_missed > 4) break; /* * Jump to the next folio as we already have a folio for * the current offset. */ cur += (folio_sz - offset); continue; } /* * Since add_ra_bio_pages() is always speculative, suppress * allocation warnings. */ masked_constraint_gfp = mapping_gfp_constraint(mapping, constraint_gfp); masked_constraint_gfp |= __GFP_NOWARN; folio = filemap_alloc_folio(masked_constraint_gfp, 0, NULL); if (!folio) break; if (filemap_add_folio(mapping, folio, pg_index, cache_gfp)) { /* There is already a folio, skip to the folio end. */ cur += folio_size(folio) - offset_in_folio(folio, cur); folio_put(folio); continue; } if (!*memstall && folio_test_workingset(folio)) { psi_memstall_enter(pflags); *memstall = 1; } ret = set_folio_extent_mapped(folio); if (ret < 0) { folio_unlock(folio); folio_put(folio); break; } folio_end = folio_next_pos(folio) - 1; btrfs_lock_extent(tree, cur, folio_end, NULL); read_lock(&em_tree->lock); em = btrfs_lookup_extent_mapping(em_tree, cur, folio_end + 1 - cur); read_unlock(&em_tree->lock); /* * At this point, we have a locked folio in the page cache for * these bytes in the file. But, we have to make sure they map * to this compressed extent on disk. */ if (!em || cur < em->start || (cur + fs_info->sectorsize > btrfs_extent_map_end(em)) || (btrfs_extent_map_block_start(em) >> SECTOR_SHIFT) != orig_bio->bi_iter.bi_sector) { btrfs_free_extent_map(em); btrfs_unlock_extent(tree, cur, folio_end, NULL); folio_unlock(folio); folio_put(folio); break; } add_size = min(btrfs_extent_map_end(em), folio_end + 1) - cur; btrfs_free_extent_map(em); btrfs_unlock_extent(tree, cur, folio_end, NULL); if (folio_contains(folio, end_index)) { size_t zero_offset = offset_in_folio(folio, isize); if (zero_offset) { int zeros; zeros = folio_size(folio) - zero_offset; folio_zero_range(folio, zero_offset, zeros); } } if (!bio_add_folio(orig_bio, folio, add_size, offset_in_folio(folio, cur))) { folio_unlock(folio);Lines 1–500 of 1652. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2008 Oracle. All rights reserved. */ #include <linux/kernel.h> #include <linux/bio.h> #include <linux/file.h> #include <linux/fs.h> #include <linux/pagemap.h> #include <linux/folio_batch.h> #include <linux/highmem.h> #include <linux/kthread.h> #include <linux/time.h> #include <linux/init.h> #include <linux/string.h> #include <linux/backing-dev.h> #include <linux/writeback.h> #include <linux/psi.h> #include <linux/slab.h> #include <linux/sched/mm.h> #include <linux/log2.h> #include <linux/shrinker.h> #include "misc.h" #include "ctree.h" #include "fs.h" #include "btrfs_inode.h" #include "bio.h" #include "ordered-data.h" #include "compression.h" #include "extent_io.h" #include "extent_map.h" #include "subpage.h" #include "messages.h" #include "super.h" static struct bio_set btrfs_compressed_bioset; static const char* const btrfs_compress_types[] = { "", "zlib", "lzo", "zstd" }; const char* btrfs_compress_type2str(enum btrfs_compression_type type) { switch (type) { case BTRFS_COMPRESS_ZLIB: case BTRFS_COMPRESS_LZO: case BTRFS_COMPRESS_ZSTD: case BTRFS_COMPRESS_NONE: return btrfs_compress_types[type]; default: break; } return NULL; } static inline struct compressed_bio *to_compressed_bio(struct btrfs_bio *bbio) { return container_of(bbio, struct compressed_bio, bbio); } static struct compressed_bio *alloc_compressed_bio(struct btrfs_inode *inode, u64 start, blk_opf_t op, btrfs_bio_end_io_t end_io) { struct btrfs_bio *bbio; bbio = btrfs_bio(bio_alloc_bioset(NULL, BTRFS_MAX_COMPRESSED_PAGES, op, GFP_NOFS, &btrfs_compressed_bioset)); btrfs_bio_init(bbio, inode, start, end_io, NULL); return to_compressed_bio(bbio); } bool btrfs_compress_is_valid_type(const char *str, size_t len) { int i; for (i = 1; i < ARRAY_SIZE(btrfs_compress_types); i++) { size_t comp_len = strlen(btrfs_compress_types[i]); if (len < comp_len) continue; if (!strncmp(btrfs_compress_types[i], str, comp_len)) return true; } return false; } static int compression_decompress_bio(struct list_head *ws, struct compressed_bio *cb) { switch (cb->compress_type) { case BTRFS_COMPRESS_ZLIB: return zlib_decompress_bio(ws, cb); case BTRFS_COMPRESS_LZO: return lzo_decompress_bio(ws, cb); case BTRFS_COMPRESS_ZSTD: return zstd_decompress_bio(ws, cb); case BTRFS_COMPRESS_NONE: default: /* * This can't happen, the type is validated several times * before we get here. */ BUG(); } } static int compression_decompress(int type, struct list_head *ws, const u8 *data_in, struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, size_t destlen) { switch (type) { case BTRFS_COMPRESS_ZLIB: return zlib_decompress(ws, data_in, dest_folio, dest_pgoff, srclen, destlen); case BTRFS_COMPRESS_LZO: return lzo_decompress(ws, data_in, dest_folio, dest_pgoff, srclen, destlen); case BTRFS_COMPRESS_ZSTD: return zstd_decompress(ws, data_in, dest_folio, dest_pgoff, srclen, destlen); case BTRFS_COMPRESS_NONE: default: /* * This can't happen, the type is validated several times * before we get here. */ BUG(); } } static int btrfs_decompress_bio(struct compressed_bio *cb); /* * Global cache of last unused pages for compression/decompression. */ static struct btrfs_compr_pool { struct shrinker *shrinker; spinlock_t lock; struct list_head list; int count; int thresh; } compr_pool; static unsigned long btrfs_compr_pool_count(struct shrinker *sh, struct shrink_control *sc) { int ret; /* * We must not read the values more than once if 'ret' gets expanded in * the return statement so we don't accidentally return a negative * number, even if the first condition finds it positive. */ ret = READ_ONCE(compr_pool.count) - READ_ONCE(compr_pool.thresh); return ret > 0 ? ret : 0; } static unsigned long btrfs_compr_pool_scan(struct shrinker *sh, struct shrink_control *sc) { LIST_HEAD(remove); struct list_head *tmp, *next; int freed; if (compr_pool.count == 0) return SHRINK_STOP; /* For now, just simply drain the whole list. */ spin_lock(&compr_pool.lock); list_splice_init(&compr_pool.list, &remove); freed = compr_pool.count; compr_pool.count = 0; spin_unlock(&compr_pool.lock); list_for_each_safe(tmp, next, &remove) { struct page *page = list_entry(tmp, struct page, lru); ASSERT(page_ref_count(page) == 1); put_page(page); } return freed; } /* * Common wrappers for page allocation from compression wrappers */ struct folio *btrfs_alloc_compr_folio(struct btrfs_fs_info *fs_info, gfp_t gfp) { struct folio *folio = NULL; /* For bs > ps cases, no cached folio pool for now. */ if (fs_info->block_min_order) goto alloc; spin_lock(&compr_pool.lock); if (compr_pool.count > 0) { folio = list_first_entry(&compr_pool.list, struct folio, lru); list_del_init(&folio->lru); compr_pool.count--; } spin_unlock(&compr_pool.lock); if (folio) return folio; alloc: return folio_alloc(gfp, fs_info->block_min_order); } void btrfs_free_compr_folio(struct folio *folio) { bool do_free = false; /* The folio is from bs > ps fs, no cached pool for now. */ if (folio_order(folio)) goto free; spin_lock(&compr_pool.lock); if (compr_pool.count > compr_pool.thresh) { do_free = true; } else { list_add(&folio->lru, &compr_pool.list); compr_pool.count++; } spin_unlock(&compr_pool.lock); if (!do_free) return; free: ASSERT(folio_ref_count(folio) == 1); folio_put(folio); } static void end_bbio_compressed_read(struct btrfs_bio *bbio) { struct compressed_bio *cb = to_compressed_bio(bbio); blk_status_t status = bbio->bio.bi_status; struct folio_iter fi; if (!status) status = errno_to_blk_status(btrfs_decompress_bio(cb)); btrfs_bio_end_io(cb->orig_bbio, status); bio_for_each_folio_all(fi, &bbio->bio) btrfs_free_compr_folio(fi.folio); bio_put(&bbio->bio); } /* * Clear the writeback bits on all of the file * pages for a compressed write */ static noinline void end_compressed_writeback(const struct compressed_bio *cb) { struct inode *inode = &cb->bbio.inode->vfs_inode; struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); pgoff_t index = cb->start >> PAGE_SHIFT; const pgoff_t end_index = (cb->start + cb->len - 1) >> PAGE_SHIFT; struct folio_batch fbatch; int i; int ret; ret = blk_status_to_errno(cb->bbio.bio.bi_status); if (ret) mapping_set_error(inode->i_mapping, ret); folio_batch_init(&fbatch); while (index <= end_index) { ret = filemap_get_folios(inode->i_mapping, &index, end_index, &fbatch); if (ret == 0) return; for (i = 0; i < ret; i++) { struct folio *folio = fbatch.folios[i]; btrfs_folio_clamp_clear_writeback(fs_info, folio, cb->start, cb->len); } folio_batch_release(&fbatch); } /* the inode may be gone now */ } /* * Do the cleanup once all the compressed pages hit the disk. This will clear * writeback on the file pages and free the compressed pages. * * This also calls the writeback end hooks for the file pages so that metadata * and checksums can be updated in the file. */ static void end_bbio_compressed_write(struct btrfs_bio *bbio) { struct compressed_bio *cb = to_compressed_bio(bbio); struct folio_iter fi; btrfs_finish_ordered_extent(cb->bbio.ordered, cb->start, cb->len, cb->bbio.bio.bi_status == BLK_STS_OK); if (cb->writeback) end_compressed_writeback(cb); /* Note, our inode could be gone now. */ bio_for_each_folio_all(fi, &bbio->bio) btrfs_free_compr_folio(fi.folio); bio_put(&cb->bbio.bio); } /* * worker function to build and submit bios for previously compressed pages. * The corresponding pages in the inode should be marked for writeback * and the compressed pages should have a reference on them for dropping * when the IO is complete. * * This also checksums the file bytes and gets things ready for * the end io hooks. */ void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered, struct compressed_bio *cb) { struct btrfs_inode *inode = ordered->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; ASSERT(IS_ALIGNED(ordered->file_offset, fs_info->sectorsize)); ASSERT(IS_ALIGNED(ordered->num_bytes, fs_info->sectorsize)); /* * This flag determines if we should clear the writeback flag from the * page cache. But this function is only utilized by encoded writes, it * never goes through the page cache. */ ASSERT(!cb->writeback); cb->start = ordered->file_offset; cb->len = ordered->num_bytes; ASSERT(cb->bbio.bio.bi_iter.bi_size == ordered->disk_num_bytes); cb->bbio.bio.bi_iter.bi_sector = ordered->disk_bytenr >> SECTOR_SHIFT; cb->bbio.ordered = ordered; btrfs_submit_bbio(&cb->bbio, 0); } /* * Allocate a compressed write bio for @inode file offset @start length @len. * * The caller still needs to properly queue all folios and populate involved * members. */ struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, u64 start, u64 len) { struct compressed_bio *cb; cb = alloc_compressed_bio(inode, start, REQ_OP_WRITE, end_bbio_compressed_write); cb->start = start; cb->len = len; cb->writeback = false; return cb; } /* * Add extra folios in the same compressed file extent so that we don't need to * re-read the same extent again and again. * * If in the same folio, we have several non-contiguous blocks which are pointing * to the same on-disk compressed data, we will re-read the same extent many * times, as this function can only help cross folio situations. */ static noinline int add_ra_bio_folios(struct inode *inode, u64 compressed_end, struct compressed_bio *cb, int *memstall, unsigned long *pflags, bool direct_reclaim) { struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); pgoff_t end_index; struct bio *orig_bio = &cb->orig_bbio->bio; u64 cur = cb->orig_bbio->file_offset + orig_bio->bi_iter.bi_size; u64 isize = i_size_read(inode); int ret; gfp_t constraint_gfp, cache_gfp; struct folio *folio; struct extent_map *em; struct address_space *mapping = inode->i_mapping; struct extent_map_tree *em_tree; struct extent_io_tree *tree; int sectors_missed = 0; em_tree = &BTRFS_I(inode)->extent_tree; tree = &BTRFS_I(inode)->io_tree; if (isize == 0) return 0; /* For bs > ps cases, we don't support readahead for compressed folios for now. */ if (fs_info->block_min_order) return 0; end_index = (i_size_read(inode) - 1) >> PAGE_SHIFT; /* Avoid direct reclaim when the caller does not allow it. */ constraint_gfp = ~__GFP_FS; cache_gfp = GFP_NOFS | __GFP_NOWARN; if (!direct_reclaim) { constraint_gfp &= ~__GFP_DIRECT_RECLAIM; cache_gfp &= ~__GFP_DIRECT_RECLAIM; } while (cur < compressed_end) { u64 folio_end; pgoff_t pg_index = cur >> PAGE_SHIFT; gfp_t masked_constraint_gfp; u32 add_size; if (pg_index > end_index) break; folio = filemap_get_folio(mapping, pg_index); if (!IS_ERR(folio)) { u64 folio_sz = folio_size(folio); u64 offset = offset_in_folio(folio, cur); folio_put(folio); sectors_missed += (folio_sz - offset) >> fs_info->sectorsize_bits; /* Beyond threshold, no need to continue */ if (sectors_missed > 4) break; /* * Jump to the next folio as we already have a folio for * the current offset. */ cur += (folio_sz - offset); continue; } /* * Since add_ra_bio_pages() is always speculative, suppress * allocation warnings. */ masked_constraint_gfp = mapping_gfp_constraint(mapping, constraint_gfp); masked_constraint_gfp |= __GFP_NOWARN; folio = filemap_alloc_folio(masked_constraint_gfp, 0, NULL); if (!folio) break; if (filemap_add_folio(mapping, folio, pg_index, cache_gfp)) { /* There is already a folio, skip to the folio end. */ cur += folio_size(folio) - offset_in_folio(folio, cur); folio_put(folio); continue; } if (!*memstall && folio_test_workingset(folio)) { psi_memstall_enter(pflags); *memstall = 1; } ret = set_folio_extent_mapped(folio); if (ret < 0) { folio_unlock(folio); folio_put(folio); break; } folio_end = folio_next_pos(folio) - 1; btrfs_lock_extent(tree, cur, folio_end, NULL); read_lock(&em_tree->lock); em = btrfs_lookup_extent_mapping(em_tree, cur, folio_end + 1 - cur); read_unlock(&em_tree->lock); /* * At this point, we have a locked folio in the page cache for * these bytes in the file. But, we have to make sure they map * to this compressed extent on disk. */ if (!em || cur < em->start || (cur + fs_info->sectorsize > btrfs_extent_map_end(em)) || (btrfs_extent_map_block_start(em) >> SECTOR_SHIFT) != orig_bio->bi_iter.bi_sector) { btrfs_free_extent_map(em); btrfs_unlock_extent(tree, cur, folio_end, NULL); folio_unlock(folio); folio_put(folio); break; } add_size = min(btrfs_extent_map_end(em), folio_end + 1) - cur; btrfs_free_extent_map(em); btrfs_unlock_extent(tree, cur, folio_end, NULL); if (folio_contains(folio, end_index)) { size_t zero_offset = offset_in_folio(folio, isize); if (zero_offset) { int zeros; zeros = folio_size(folio) - zero_offset; folio_zero_range(folio, zero_offset, zeros); } } if (!bio_add_folio(orig_bio, folio, add_size, offset_in_folio(folio, cur))) { folio_unlock(folio);