// SPDX-License-Identifier: GPL-2.0/* * fs/f2fs/data.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */#include <linux/fs.h>#include <linux/f2fs_fs.h>#include <linux/buffer_head.h>#include <linux/mpage.h>#include <linux/writeback.h>#include <linux/backing-dev.h>#include <linux/pagevec.h>#include <linux/blkdev.h>#include <linux/bio.h>#include <linux/blk-crypto.h>#include <linux/swap.h>#include <linux/prefetch.h>#include <linux/uio.h>#include <linux/cleancache.h>#include <linux/sched/signal.h>#include <linux/fiemap.h> #include "f2fs.h"#include "node.h"#include "segment.h"#include "trace.h"#include <trace/events/f2fs.h> #define NUM_PREALLOC_POST_READ_CTXS 128 static struct kmem_cache *bio_post_read_ctx_cache;static struct kmem_cache *bio_entry_slab;static mempool_t *bio_post_read_ctx_pool;static struct bio_set f2fs_bioset; #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE int __init f2fs_init_bioset(void){ if (bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS)) return -ENOMEM; return 0;} void f2fs_destroy_bioset(void){ bioset_exit(&f2fs_bioset);} static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs){ return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset);} struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio){ if (noio) { /* No failure on bio allocation */ return __f2fs_bio_alloc(GFP_NOIO, npages); } if (time_to_inject(sbi, FAULT_ALLOC_BIO)) { f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO); return NULL; } return __f2fs_bio_alloc(GFP_KERNEL, npages);} static bool __is_cp_guaranteed(struct page *page){ struct address_space *mapping = page->mapping; struct inode *inode; struct f2fs_sb_info *sbi; if (!mapping) return false; if (f2fs_is_compressed_page(page)) return false; inode = mapping->host; sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi) || inode->i_ino == F2FS_NODE_INO(sbi) || S_ISDIR(inode->i_mode) || (S_ISREG(inode->i_mode) && (f2fs_is_atomic_file(inode) || IS_NOQUOTA(inode))) || is_cold_data(page)) return true; return false;} static enum count_type __read_io_type(struct page *page){ struct address_space *mapping = page_file_mapping(page); if (mapping) { struct inode *inode = mapping->host; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi)) return F2FS_RD_META; if (inode->i_ino == F2FS_NODE_INO(sbi)) return F2FS_RD_NODE; } return F2FS_RD_DATA;} /* postprocessing steps for read bios */enum bio_post_read_step { STEP_DECRYPT, STEP_DECOMPRESS_NOWQ, /* handle normal cluster data inplace */ STEP_DECOMPRESS, /* handle compressed cluster data in workqueue */ STEP_VERITY,}; struct bio_post_read_ctx { struct bio *bio; struct f2fs_sb_info *sbi; struct work_struct work; unsigned int enabled_steps;}; static void __read_end_io(struct bio *bio, bool compr, bool verity){ struct page *page; struct bio_vec *bv; struct bvec_iter_all iter_all; bio_for_each_segment_all(bv, bio, iter_all) { page = bv->bv_page; #ifdef CONFIG_F2FS_FS_COMPRESSION if (compr && f2fs_is_compressed_page(page)) { f2fs_decompress_pages(bio, page, verity); continue; } if (verity) continue;#endif /* PG_error was set if any post_read step failed */ if (bio->bi_status || PageError(page)) { ClearPageUptodate(page); /* will re-read again later */ ClearPageError(page); } else { SetPageUptodate(page); } dec_page_count(F2FS_P_SB(page), __read_io_type(page)); unlock_page(page); }} static void f2fs_release_read_bio(struct bio *bio);static void __f2fs_read_end_io(struct bio *bio, bool compr, bool verity){ if (!compr) __read_end_io(bio, false, verity); f2fs_release_read_bio(bio);} static void f2fs_decompress_bio(struct bio *bio, bool verity){ __read_end_io(bio, true, verity);} static void bio_post_read_processing(struct bio_post_read_ctx *ctx); static void f2fs_decrypt_work(struct bio_post_read_ctx *ctx){ fscrypt_decrypt_bio(ctx->bio);} static void f2fs_decompress_work(struct bio_post_read_ctx *ctx){ f2fs_decompress_bio(ctx->bio, ctx->enabled_steps & (1 << STEP_VERITY));} #ifdef CONFIG_F2FS_FS_COMPRESSIONstatic void f2fs_verify_pages(struct page **rpages, unsigned int cluster_size){ f2fs_decompress_end_io(rpages, cluster_size, false, true);} static void f2fs_verify_bio(struct bio *bio){ struct bio_vec *bv; struct bvec_iter_all iter_all; bio_for_each_segment_all(bv, bio, iter_all) { struct page *page = bv->bv_page; struct decompress_io_ctx *dic; dic = (struct decompress_io_ctx *)page_private(page); if (dic) { if (refcount_dec_not_one(&dic->ref)) continue; f2fs_verify_pages(dic->rpages, dic->cluster_size); f2fs_free_dic(dic); continue; } if (bio->bi_status || PageError(page)) goto clear_uptodate; if (fsverity_verify_page(page)) { SetPageUptodate(page); goto unlock; }clear_uptodate: ClearPageUptodate(page); ClearPageError(page);unlock: dec_page_count(F2FS_P_SB(page), __read_io_type(page)); unlock_page(page); }}#endif static void f2fs_verity_work(struct work_struct *work){ struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio;#ifdef CONFIG_F2FS_FS_COMPRESSION unsigned int enabled_steps = ctx->enabled_steps;#endif /* * fsverity_verify_bio() may call readpages() again, and while verity * will be disabled for this, decryption may still be needed, resulting * in another bio_post_read_ctx being allocated. So to prevent * deadlocks we need to release the current ctx to the mempool first. * This assumes that verity is the last post-read step. */ mempool_free(ctx, bio_post_read_ctx_pool); bio->bi_private = NULL; #ifdef CONFIG_F2FS_FS_COMPRESSION /* previous step is decompression */ if (enabled_steps & (1 << STEP_DECOMPRESS)) { f2fs_verify_bio(bio); f2fs_release_read_bio(bio); return; }#endif fsverity_verify_bio(bio); __f2fs_read_end_io(bio, false, false);} static void f2fs_post_read_work(struct work_struct *work){ struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); if (ctx->enabled_steps & (1 << STEP_DECRYPT)) f2fs_decrypt_work(ctx); if (ctx->enabled_steps & (1 << STEP_DECOMPRESS)) f2fs_decompress_work(ctx); if (ctx->enabled_steps & (1 << STEP_VERITY)) { INIT_WORK(&ctx->work, f2fs_verity_work); fsverity_enqueue_verify_work(&ctx->work); return; } __f2fs_read_end_io(ctx->bio, ctx->enabled_steps & (1 << STEP_DECOMPRESS), false);} static void f2fs_enqueue_post_read_work(struct f2fs_sb_info *sbi, struct work_struct *work){ queue_work(sbi->post_read_wq, work);} static void bio_post_read_processing(struct bio_post_read_ctx *ctx){ /* * We use different work queues for decryption and for verity because * verity may require reading metadata pages that need decryption, and * we shouldn't recurse to the same workqueue. */ if (ctx->enabled_steps & (1 << STEP_DECRYPT) || ctx->enabled_steps & (1 << STEP_DECOMPRESS)) { INIT_WORK(&ctx->work, f2fs_post_read_work); f2fs_enqueue_post_read_work(ctx->sbi, &ctx->work); return; } if (ctx->enabled_steps & (1 << STEP_VERITY)) { INIT_WORK(&ctx->work, f2fs_verity_work); fsverity_enqueue_verify_work(&ctx->work); return; } __f2fs_read_end_io(ctx->bio, false, false);} static bool f2fs_bio_post_read_required(struct bio *bio){ return bio->bi_private;} static void f2fs_read_end_io(struct bio *bio){ struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio)); if (time_to_inject(sbi, FAULT_READ_IO)) { f2fs_show_injection_info(sbi, FAULT_READ_IO); bio->bi_status = BLK_STS_IOERR; } if (f2fs_bio_post_read_required(bio)) { struct bio_post_read_ctx *ctx = bio->bi_private; bio_post_read_processing(ctx); return; } __f2fs_read_end_io(bio, false, false);} static void f2fs_write_end_io(struct bio *bio){ struct f2fs_sb_info *sbi = bio->bi_private; struct bio_vec *bvec; struct bvec_iter_all iter_all; if (time_to_inject(sbi, FAULT_WRITE_IO)) { f2fs_show_injection_info(sbi, FAULT_WRITE_IO); bio->bi_status = BLK_STS_IOERR; } bio_for_each_segment_all(bvec, bio, iter_all) { struct page *page = bvec->bv_page; enum count_type type = WB_DATA_TYPE(page); if (IS_DUMMY_WRITTEN_PAGE(page)) { set_page_private(page, (unsigned long)NULL); ClearPagePrivate(page); unlock_page(page); mempool_free(page, sbi->write_io_dummy); if (unlikely(bio->bi_status)) f2fs_stop_checkpoint(sbi, true); continue; } fscrypt_finalize_bounce_page(&page); #ifdef CONFIG_F2FS_FS_COMPRESSION if (f2fs_is_compressed_page(page)) { f2fs_compress_write_end_io(bio, page); continue; }#endif if (unlikely(bio->bi_status)) { mapping_set_error(page->mapping, -EIO); if (type == F2FS_WB_CP_DATA) f2fs_stop_checkpoint(sbi, true); } f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) && page->index != nid_of_node(page)); dec_page_count(sbi, type); if (f2fs_in_warm_node_list(sbi, page)) f2fs_del_fsync_node_entry(sbi, page); clear_cold_data(page); end_page_writeback(page); } if (!get_pages(sbi, F2FS_WB_CP_DATA) && wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); bio_put(bio);} struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, block_t blk_addr, struct bio *bio){ struct block_device *bdev = sbi->sb->s_bdev; int i; if (f2fs_is_multi_device(sbi)) { for (i = 0; i < sbi->s_ndevs; i++) { if (FDEV(i).start_blk <= blk_addr && FDEV(i).end_blk >= blk_addr) { blk_addr -= FDEV(i).start_blk; bdev = FDEV(i).bdev; break; } } } if (bio) { bio_set_dev(bio, bdev); bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); } return bdev;} int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr){ int i; if (!f2fs_is_multi_device(sbi)) return 0; for (i = 0; i < sbi->s_ndevs; i++) if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) return i; return 0;} /* * Return true, if pre_bio's bdev is same as its target device. */static bool __same_bdev(struct f2fs_sb_info *sbi, block_t blk_addr, struct bio *bio){ struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL); return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;} static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages){ struct f2fs_sb_info *sbi = fio->sbi; struct bio *bio; bio = f2fs_bio_alloc(sbi, npages, true); f2fs_target_device(sbi, fio->new_blkaddr, bio); if (is_read_io(fio->op)) { bio->bi_end_io = f2fs_read_end_io; bio->bi_private = NULL; } else { bio->bi_end_io = f2fs_write_end_io; bio->bi_private = sbi; bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, fio->type, fio->temp); } if (fio->io_wbc) wbc_init_bio(fio->io_wbc, bio); return bio;} static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, pgoff_t first_idx, const struct f2fs_io_info *fio, gfp_t gfp_mask){ /* * The f2fs garbage collector sets ->encrypted_page when it wants to * read/write raw data without encryption. */ if (!fio || !fio->encrypted_page) fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask);} static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, pgoff_t next_idx, const struct f2fs_io_info *fio){ /* * The f2fs garbage collector sets ->encrypted_page when it wants to * read/write raw data without encryption. */ if (fio && fio->encrypted_page) return !bio_has_crypt_ctx(bio); return fscrypt_mergeable_bio(bio, inode, next_idx);} static inline void __submit_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type){ if (!is_read_io(bio_op(bio))) { unsigned int start; if (type != DATA && type != NODE) goto submit_io; if (f2fs_lfs_mode(sbi) && current->plug) blk_finish_plug(current->plug);Lines 1–500 of 4109. Load more
// SPDX-License-Identifier: GPL-2.0 /* * fs/f2fs/data.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/buffer_head.h> #include <linux/mpage.h> #include <linux/writeback.h> #include <linux/backing-dev.h> #include <linux/pagevec.h> #include <linux/blkdev.h> #include <linux/bio.h> #include <linux/blk-crypto.h> #include <linux/swap.h> #include <linux/prefetch.h> #include <linux/uio.h> #include <linux/cleancache.h> #include <linux/sched/signal.h> #include <linux/fiemap.h> #include "f2fs.h" #include "node.h" #include "segment.h" #include "trace.h" #include <trace/events/f2fs.h> #define NUM_PREALLOC_POST_READ_CTXS 128 static struct kmem_cache *bio_post_read_ctx_cache; static struct kmem_cache *bio_entry_slab; static mempool_t *bio_post_read_ctx_pool; static struct bio_set f2fs_bioset; #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE int __init f2fs_init_bioset(void) { if (bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS)) return -ENOMEM; return 0; } void f2fs_destroy_bioset(void) { bioset_exit(&f2fs_bioset); } static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) { return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset); } struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio) { if (noio) { /* No failure on bio allocation */ return __f2fs_bio_alloc(GFP_NOIO, npages); } if (time_to_inject(sbi, FAULT_ALLOC_BIO)) { f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO); return NULL; } return __f2fs_bio_alloc(GFP_KERNEL, npages); } static bool __is_cp_guaranteed(struct page *page) { struct address_space *mapping = page->mapping; struct inode *inode; struct f2fs_sb_info *sbi; if (!mapping) return false; if (f2fs_is_compressed_page(page)) return false; inode = mapping->host; sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi) || inode->i_ino == F2FS_NODE_INO(sbi) || S_ISDIR(inode->i_mode) || (S_ISREG(inode->i_mode) && (f2fs_is_atomic_file(inode) || IS_NOQUOTA(inode))) || is_cold_data(page)) return true; return false; } static enum count_type __read_io_type(struct page *page) { struct address_space *mapping = page_file_mapping(page); if (mapping) { struct inode *inode = mapping->host; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi)) return F2FS_RD_META; if (inode->i_ino == F2FS_NODE_INO(sbi)) return F2FS_RD_NODE; } return F2FS_RD_DATA; } /* postprocessing steps for read bios */ enum bio_post_read_step { STEP_DECRYPT, STEP_DECOMPRESS_NOWQ, /* handle normal cluster data inplace */ STEP_DECOMPRESS, /* handle compressed cluster data in workqueue */ STEP_VERITY, }; struct bio_post_read_ctx { struct bio *bio; struct f2fs_sb_info *sbi; struct work_struct work; unsigned int enabled_steps; }; static void __read_end_io(struct bio *bio, bool compr, bool verity) { struct page *page; struct bio_vec *bv; struct bvec_iter_all iter_all; bio_for_each_segment_all(bv, bio, iter_all) { page = bv->bv_page; #ifdef CONFIG_F2FS_FS_COMPRESSION if (compr && f2fs_is_compressed_page(page)) { f2fs_decompress_pages(bio, page, verity); continue; } if (verity) continue; #endif /* PG_error was set if any post_read step failed */ if (bio->bi_status || PageError(page)) { ClearPageUptodate(page); /* will re-read again later */ ClearPageError(page); } else { SetPageUptodate(page); } dec_page_count(F2FS_P_SB(page), __read_io_type(page)); unlock_page(page); } } static void f2fs_release_read_bio(struct bio *bio); static void __f2fs_read_end_io(struct bio *bio, bool compr, bool verity) { if (!compr) __read_end_io(bio, false, verity); f2fs_release_read_bio(bio); } static void f2fs_decompress_bio(struct bio *bio, bool verity) { __read_end_io(bio, true, verity); } static void bio_post_read_processing(struct bio_post_read_ctx *ctx); static void f2fs_decrypt_work(struct bio_post_read_ctx *ctx) { fscrypt_decrypt_bio(ctx->bio); } static void f2fs_decompress_work(struct bio_post_read_ctx *ctx) { f2fs_decompress_bio(ctx->bio, ctx->enabled_steps & (1 << STEP_VERITY)); } #ifdef CONFIG_F2FS_FS_COMPRESSION static void f2fs_verify_pages(struct page **rpages, unsigned int cluster_size) { f2fs_decompress_end_io(rpages, cluster_size, false, true); } static void f2fs_verify_bio(struct bio *bio) { struct bio_vec *bv; struct bvec_iter_all iter_all; bio_for_each_segment_all(bv, bio, iter_all) { struct page *page = bv->bv_page; struct decompress_io_ctx *dic; dic = (struct decompress_io_ctx *)page_private(page); if (dic) { if (refcount_dec_not_one(&dic->ref)) continue; f2fs_verify_pages(dic->rpages, dic->cluster_size); f2fs_free_dic(dic); continue; } if (bio->bi_status || PageError(page)) goto clear_uptodate; if (fsverity_verify_page(page)) { SetPageUptodate(page); goto unlock; } clear_uptodate: ClearPageUptodate(page); ClearPageError(page); unlock: dec_page_count(F2FS_P_SB(page), __read_io_type(page)); unlock_page(page); } } #endif static void f2fs_verity_work(struct work_struct *work) { struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio; #ifdef CONFIG_F2FS_FS_COMPRESSION unsigned int enabled_steps = ctx->enabled_steps; #endif /* * fsverity_verify_bio() may call readpages() again, and while verity * will be disabled for this, decryption may still be needed, resulting * in another bio_post_read_ctx being allocated. So to prevent * deadlocks we need to release the current ctx to the mempool first. * This assumes that verity is the last post-read step. */ mempool_free(ctx, bio_post_read_ctx_pool); bio->bi_private = NULL; #ifdef CONFIG_F2FS_FS_COMPRESSION /* previous step is decompression */ if (enabled_steps & (1 << STEP_DECOMPRESS)) { f2fs_verify_bio(bio); f2fs_release_read_bio(bio); return; } #endif fsverity_verify_bio(bio); __f2fs_read_end_io(bio, false, false); } static void f2fs_post_read_work(struct work_struct *work) { struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); if (ctx->enabled_steps & (1 << STEP_DECRYPT)) f2fs_decrypt_work(ctx); if (ctx->enabled_steps & (1 << STEP_DECOMPRESS)) f2fs_decompress_work(ctx); if (ctx->enabled_steps & (1 << STEP_VERITY)) { INIT_WORK(&ctx->work, f2fs_verity_work); fsverity_enqueue_verify_work(&ctx->work); return; } __f2fs_read_end_io(ctx->bio, ctx->enabled_steps & (1 << STEP_DECOMPRESS), false); } static void f2fs_enqueue_post_read_work(struct f2fs_sb_info *sbi, struct work_struct *work) { queue_work(sbi->post_read_wq, work); } static void bio_post_read_processing(struct bio_post_read_ctx *ctx) { /* * We use different work queues for decryption and for verity because * verity may require reading metadata pages that need decryption, and * we shouldn't recurse to the same workqueue. */ if (ctx->enabled_steps & (1 << STEP_DECRYPT) || ctx->enabled_steps & (1 << STEP_DECOMPRESS)) { INIT_WORK(&ctx->work, f2fs_post_read_work); f2fs_enqueue_post_read_work(ctx->sbi, &ctx->work); return; } if (ctx->enabled_steps & (1 << STEP_VERITY)) { INIT_WORK(&ctx->work, f2fs_verity_work); fsverity_enqueue_verify_work(&ctx->work); return; } __f2fs_read_end_io(ctx->bio, false, false); } static bool f2fs_bio_post_read_required(struct bio *bio) { return bio->bi_private; } static void f2fs_read_end_io(struct bio *bio) { struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio)); if (time_to_inject(sbi, FAULT_READ_IO)) { f2fs_show_injection_info(sbi, FAULT_READ_IO); bio->bi_status = BLK_STS_IOERR; } if (f2fs_bio_post_read_required(bio)) { struct bio_post_read_ctx *ctx = bio->bi_private; bio_post_read_processing(ctx); return; } __f2fs_read_end_io(bio, false, false); } static void f2fs_write_end_io(struct bio *bio) { struct f2fs_sb_info *sbi = bio->bi_private; struct bio_vec *bvec; struct bvec_iter_all iter_all; if (time_to_inject(sbi, FAULT_WRITE_IO)) { f2fs_show_injection_info(sbi, FAULT_WRITE_IO); bio->bi_status = BLK_STS_IOERR; } bio_for_each_segment_all(bvec, bio, iter_all) { struct page *page = bvec->bv_page; enum count_type type = WB_DATA_TYPE(page); if (IS_DUMMY_WRITTEN_PAGE(page)) { set_page_private(page, (unsigned long)NULL); ClearPagePrivate(page); unlock_page(page); mempool_free(page, sbi->write_io_dummy); if (unlikely(bio->bi_status)) f2fs_stop_checkpoint(sbi, true); continue; } fscrypt_finalize_bounce_page(&page); #ifdef CONFIG_F2FS_FS_COMPRESSION if (f2fs_is_compressed_page(page)) { f2fs_compress_write_end_io(bio, page); continue; } #endif if (unlikely(bio->bi_status)) { mapping_set_error(page->mapping, -EIO); if (type == F2FS_WB_CP_DATA) f2fs_stop_checkpoint(sbi, true); } f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) && page->index != nid_of_node(page)); dec_page_count(sbi, type); if (f2fs_in_warm_node_list(sbi, page)) f2fs_del_fsync_node_entry(sbi, page); clear_cold_data(page); end_page_writeback(page); } if (!get_pages(sbi, F2FS_WB_CP_DATA) && wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); bio_put(bio); } struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, block_t blk_addr, struct bio *bio) { struct block_device *bdev = sbi->sb->s_bdev; int i; if (f2fs_is_multi_device(sbi)) { for (i = 0; i < sbi->s_ndevs; i++) { if (FDEV(i).start_blk <= blk_addr && FDEV(i).end_blk >= blk_addr) { blk_addr -= FDEV(i).start_blk; bdev = FDEV(i).bdev; break; } } } if (bio) { bio_set_dev(bio, bdev); bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); } return bdev; } int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) { int i; if (!f2fs_is_multi_device(sbi)) return 0; for (i = 0; i < sbi->s_ndevs; i++) if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) return i; return 0; } /* * Return true, if pre_bio's bdev is same as its target device. */ static bool __same_bdev(struct f2fs_sb_info *sbi, block_t blk_addr, struct bio *bio) { struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL); return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno; } static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) { struct f2fs_sb_info *sbi = fio->sbi; struct bio *bio; bio = f2fs_bio_alloc(sbi, npages, true); f2fs_target_device(sbi, fio->new_blkaddr, bio); if (is_read_io(fio->op)) { bio->bi_end_io = f2fs_read_end_io; bio->bi_private = NULL; } else { bio->bi_end_io = f2fs_write_end_io; bio->bi_private = sbi; bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, fio->type, fio->temp); } if (fio->io_wbc) wbc_init_bio(fio->io_wbc, bio); return bio; } static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, pgoff_t first_idx, const struct f2fs_io_info *fio, gfp_t gfp_mask) { /* * The f2fs garbage collector sets ->encrypted_page when it wants to * read/write raw data without encryption. */ if (!fio || !fio->encrypted_page) fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); } static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, pgoff_t next_idx, const struct f2fs_io_info *fio) { /* * The f2fs garbage collector sets ->encrypted_page when it wants to * read/write raw data without encryption. */ if (fio && fio->encrypted_page) return !bio_has_crypt_ctx(bio); return fscrypt_mergeable_bio(bio, inode, next_idx); } static inline void __submit_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type) { if (!is_read_io(bio_op(bio))) { unsigned int start; if (type != DATA && type != NODE) goto submit_io; if (f2fs_lfs_mode(sbi) && current->plug) blk_finish_plug(current->plug);