// SPDX-License-Identifier: GPL-2.0/* * fs/f2fs/data.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */#include <linux/fs.h>#include <linux/f2fs_fs.h>#include <linux/sched/mm.h>#include <linux/mpage.h>#include <linux/writeback.h>#include <linux/folio_batch.h>#include <linux/blkdev.h>#include <linux/bio.h>#include <linux/blk-crypto.h>#include <linux/swap.h>#include <linux/prefetch.h>#include <linux/uio.h>#include <linux/sched/signal.h>#include <linux/fiemap.h>#include <linux/iomap.h>#include <linux/fserror.h> #include "f2fs.h"#include "node.h"#include "segment.h"#include "iostat.h"#include <trace/events/f2fs.h> #define NUM_PREALLOC_POST_READ_CTXS 128 static struct kmem_cache *bio_post_read_ctx_cache;static struct kmem_cache *bio_entry_slab;static struct kmem_cache *ffs_entry_slab;static mempool_t *bio_post_read_ctx_pool;static struct bio_set f2fs_bioset; struct f2fs_folio_state { spinlock_t state_lock; unsigned int read_pages_pending;}; struct f2fs_bio { struct work_struct work; struct bio bio;}; #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE int __init f2fs_init_bioset(void){ return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, offsetof(struct f2fs_bio, bio), BIOSET_NEED_BVECS);} void f2fs_destroy_bioset(void){ bioset_exit(&f2fs_bioset);} bool f2fs_is_cp_guaranteed(const struct folio *folio){ struct address_space *mapping = folio->mapping; struct inode *inode; struct f2fs_sb_info *sbi; inode = mapping->host; sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi) || inode->i_ino == F2FS_NODE_INO(sbi) || S_ISDIR(inode->i_mode)) return true; if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) || folio_test_f2fs_gcing(folio)) return true; return false;} static enum count_type __read_io_type(struct folio *folio){ struct address_space *mapping = folio->mapping; if (mapping) { struct inode *inode = mapping->host; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi)) return F2FS_RD_META; if (inode->i_ino == F2FS_NODE_INO(sbi)) return F2FS_RD_NODE; } return F2FS_RD_DATA;} /* postprocessing steps for read bios */enum bio_post_read_step {#ifdef CONFIG_F2FS_FS_COMPRESSION STEP_DECOMPRESS = BIT(1),#else STEP_DECOMPRESS = 0, /* compile out the decompression-related code */#endif#ifdef CONFIG_FS_VERITY STEP_VERITY = BIT(2),#else STEP_VERITY = 0, /* compile out the verity-related code */#endif}; struct bio_post_read_ctx { struct bio *bio; struct f2fs_sb_info *sbi; struct fsverity_info *vi; struct work_struct work; unsigned int enabled_steps; /* * decompression_attempted keeps track of whether * f2fs_end_read_compressed_page() has been called on the pages in the * bio that belong to a compressed cluster yet. */ bool decompression_attempted; block_t fs_blkaddr;}; /* * Update and unlock a bio's pages, and free the bio. * * This marks pages up-to-date only if there was no error in the bio (I/O error, * decryption error, or verity error), as indicated by bio->bi_status. * * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk) * aren't marked up-to-date here, as decompression is done on a per-compression- * cluster basis rather than a per-bio basis. Instead, we only must do two * things for each compressed page here: call f2fs_end_read_compressed_page() * with failed=true if an error occurred before it would have normally gotten * called (i.e., I/O error or decryption error, but *not* verity error), and * release the bio's reference to the decompress_io_ctx of the page's cluster. */static void f2fs_finish_read_bio(struct bio *bio, bool in_task){ struct folio_iter fi; struct bio_post_read_ctx *ctx = bio->bi_private; unsigned long flags; bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; unsigned nr_pages = fi.length >> PAGE_SHIFT; bool finished = true; if (!folio_test_large(folio) && f2fs_is_compressed_page(folio)) { if (ctx && !ctx->decompression_attempted) f2fs_end_read_compressed_page(folio, true, 0, in_task); f2fs_put_folio_dic(folio, in_task); continue; } if (folio_test_large(folio)) { struct f2fs_folio_state *ffs = folio->private; spin_lock_irqsave(&ffs->state_lock, flags); ffs->read_pages_pending -= nr_pages; finished = !ffs->read_pages_pending; spin_unlock_irqrestore(&ffs->state_lock, flags); } while (nr_pages--) dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); if (bio->bi_status == BLK_STS_OK && F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && f2fs_sanity_check_node_footer(F2FS_F_SB(folio), folio, folio->index, NODE_TYPE_REGULAR, true)) bio->bi_status = BLK_STS_IOERR; if (finished) folio_end_read(folio, bio->bi_status == BLK_STS_OK); } if (ctx) mempool_free(ctx, bio_post_read_ctx_pool); bio_put(bio);} static void f2fs_verify_bio(struct work_struct *work){ struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio; bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); struct fsverity_info *vi = ctx->vi; /* * fsverity_verify_bio() may call readahead() again, and while verity * will be disabled for this, decryption and/or decompression may still * be needed, resulting in another bio_post_read_ctx being allocated. * So to prevent deadlocks we need to release the current ctx to the * mempool first. This assumes that verity is the last post-read step. */ mempool_free(ctx, bio_post_read_ctx_pool); bio->bi_private = NULL; /* * Verify the bio's pages with fs-verity. Exclude compressed pages, * as those were handled separately by f2fs_end_read_compressed_page(). */ if (may_have_compressed_pages) { struct folio_iter fi; bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; if (!f2fs_is_compressed_page(folio) && !fsverity_verify_folio(vi, folio)) { bio->bi_status = BLK_STS_IOERR; break; } } } else { fsverity_verify_bio(vi, bio); } f2fs_finish_read_bio(bio, true);} /* * If the bio's data needs to be verified with fs-verity, then enqueue the * verity work for the bio. Otherwise finish the bio now. * * Note that to avoid deadlocks, the verity work can't be done on the * decryption/decompression workqueue. This is because verifying the data pages * can involve reading verity metadata pages from the file, and these verity * metadata pages may be encrypted and/or compressed. */static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task){ struct bio_post_read_ctx *ctx = bio->bi_private; if (ctx && (ctx->enabled_steps & STEP_VERITY)) { INIT_WORK(&ctx->work, f2fs_verify_bio); fsverity_enqueue_verify_work(&ctx->work); } else { f2fs_finish_read_bio(bio, in_task); }} /* * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last * remaining page was read by @ctx->bio. * * Note that a bio may span clusters (even a mix of compressed and uncompressed * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates * that the bio includes at least one compressed page. The actual decompression * is done on a per-cluster basis, not a per-bio basis. */static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx, bool in_task){ struct folio_iter fi; bool all_compressed = true; block_t blkaddr = ctx->fs_blkaddr; bio_for_each_folio_all(fi, ctx->bio) { struct folio *folio = fi.folio; if (f2fs_is_compressed_page(folio)) f2fs_end_read_compressed_page(folio, false, blkaddr, in_task); else all_compressed = false; blkaddr++; } ctx->decompression_attempted = true; /* * Optimization: if all the bio's pages are compressed, then scheduling * the per-bio verity work is unnecessary, as verity will be fully * handled at the compression cluster level. */ if (all_compressed) ctx->enabled_steps &= ~STEP_VERITY;} static void f2fs_post_read_work(struct work_struct *work){ struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio; if (ctx->enabled_steps & STEP_DECOMPRESS) f2fs_handle_step_decompress(ctx, true); f2fs_verify_and_finish_bio(bio, true);} static void f2fs_read_end_io(struct bio *bio){ struct f2fs_sb_info *sbi = F2FS_F_SB(bio_first_folio_all(bio)); struct bio_post_read_ctx *ctx; bool intask = in_task() && !irqs_disabled(); iostat_update_and_unbind_ctx(bio); ctx = bio->bi_private; if (time_to_inject(sbi, FAULT_READ_IO)) bio->bi_status = BLK_STS_IOERR; if (bio->bi_status != BLK_STS_OK) { f2fs_finish_read_bio(bio, intask); return; } if (ctx && (ctx->enabled_steps & STEP_DECOMPRESS)) { if (!f2fs_low_mem_mode(sbi)) { /* Decompress inline. */ f2fs_handle_step_decompress(ctx, intask); } else { INIT_WORK(&ctx->work, f2fs_post_read_work); queue_work(ctx->sbi->wq, &ctx->work); return; } } f2fs_verify_and_finish_bio(bio, intask);} static void f2fs_write_end_bio(struct bio *bio){ struct f2fs_sb_info *sbi = bio->bi_private; struct folio_iter fi; if (time_to_inject(sbi, FAULT_WRITE_IO)) bio->bi_status = BLK_STS_IOERR; bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; enum count_type type; #ifdef CONFIG_F2FS_FS_COMPRESSION if (f2fs_is_compressed_page(folio)) { f2fs_compress_write_end_io(bio, folio); continue; }#endif type = WB_DATA_TYPE(folio, false); if (unlikely(bio->bi_status != BLK_STS_OK)) { mapping_set_error(folio->mapping, -EIO); if (type == F2FS_WB_CP_DATA) { f2fs_stop_checkpoint(sbi, true, STOP_CP_REASON_WRITE_FAIL); } } if (is_node_folio(folio)) { f2fs_sanity_check_node_footer(sbi, folio, folio->index, NODE_TYPE_REGULAR, true); f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); } if (f2fs_in_warm_node_list(folio)) f2fs_del_fsync_node_entry(sbi, folio); dec_page_count(sbi, type); /* * we should access sbi before folio_end_writeback() to * avoid racing w/ kill_f2fs_super() */ if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); folio_clear_f2fs_gcing(folio); folio_end_writeback(folio); } bio_put(bio);} static void f2fs_write_end_io_work(struct work_struct *work){ struct bio *bio = &container_of(work, struct f2fs_bio, work)->bio; f2fs_write_end_bio(bio);} static void f2fs_write_end_io(struct bio *bio){ struct f2fs_sb_info *sbi; iostat_update_and_unbind_ctx(bio); sbi = bio->bi_private; if (in_atomic() && bio->bi_iter.bi_size > sbi->max_atc_write_bio_size) { struct work_struct *w; w = &container_of(bio, struct f2fs_bio, bio)->work; INIT_WORK(w, f2fs_write_end_io_work); queue_work(sbi->wq, w); } else { f2fs_write_end_bio(bio); }} #ifdef CONFIG_BLK_DEV_ZONEDstatic void f2fs_zone_write_end_io(struct bio *bio){ struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private; bio->bi_private = io->bi_private; complete(&io->zone_wait); f2fs_write_end_io(bio);}#endif struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, block_t blk_addr, sector_t *sector){ struct block_device *bdev = sbi->sb->s_bdev; int i; if (f2fs_is_multi_device(sbi)) { for (i = 0; i < sbi->s_ndevs; i++) { if (FDEV(i).start_blk <= blk_addr && FDEV(i).end_blk >= blk_addr) { blk_addr -= FDEV(i).start_blk; bdev = FDEV(i).bdev; break; } } } if (sector) *sector = SECTOR_FROM_BLOCK(blk_addr); return bdev;} int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr){ int i; if (!f2fs_is_multi_device(sbi)) return 0; for (i = 0; i < sbi->s_ndevs; i++) if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) return i; return 0;} static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio){ unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0); unsigned int fua_flag, meta_flag, io_flag; blk_opf_t op_flags = 0; if (fio->op != REQ_OP_WRITE) return 0; if (fio->type == DATA) io_flag = fio->sbi->data_io_flag; else if (fio->type == NODE) io_flag = fio->sbi->node_io_flag; else return 0; fua_flag = io_flag & temp_mask; meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask; /* * data/node io flag bits per temp: * REQ_META | REQ_FUA | * 5 | 4 | 3 | 2 | 1 | 0 | * Cold | Warm | Hot | Cold | Warm | Hot | */ if (BIT(fio->temp) & meta_flag) op_flags |= REQ_META; if (BIT(fio->temp) & fua_flag) op_flags |= REQ_FUA; if (fio->type == DATA && F2FS_I(fio->folio->mapping->host)->ioprio_hint == F2FS_IOPRIO_WRITE) op_flags |= REQ_PRIO; return op_flags;} static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages){ struct f2fs_sb_info *sbi = fio->sbi; struct block_device *bdev; sector_t sector; struct bio *bio;Lines 1–500 of 4619. Load more
// SPDX-License-Identifier: GPL-2.0 /* * fs/f2fs/data.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/sched/mm.h> #include <linux/mpage.h> #include <linux/writeback.h> #include <linux/folio_batch.h> #include <linux/blkdev.h> #include <linux/bio.h> #include <linux/blk-crypto.h> #include <linux/swap.h> #include <linux/prefetch.h> #include <linux/uio.h> #include <linux/sched/signal.h> #include <linux/fiemap.h> #include <linux/iomap.h> #include <linux/fserror.h> #include "f2fs.h" #include "node.h" #include "segment.h" #include "iostat.h" #include <trace/events/f2fs.h> #define NUM_PREALLOC_POST_READ_CTXS 128 static struct kmem_cache *bio_post_read_ctx_cache; static struct kmem_cache *bio_entry_slab; static struct kmem_cache *ffs_entry_slab; static mempool_t *bio_post_read_ctx_pool; static struct bio_set f2fs_bioset; struct f2fs_folio_state { spinlock_t state_lock; unsigned int read_pages_pending; }; struct f2fs_bio { struct work_struct work; struct bio bio; }; #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE int __init f2fs_init_bioset(void) { return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, offsetof(struct f2fs_bio, bio), BIOSET_NEED_BVECS); } void f2fs_destroy_bioset(void) { bioset_exit(&f2fs_bioset); } bool f2fs_is_cp_guaranteed(const struct folio *folio) { struct address_space *mapping = folio->mapping; struct inode *inode; struct f2fs_sb_info *sbi; inode = mapping->host; sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi) || inode->i_ino == F2FS_NODE_INO(sbi) || S_ISDIR(inode->i_mode)) return true; if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) || folio_test_f2fs_gcing(folio)) return true; return false; } static enum count_type __read_io_type(struct folio *folio) { struct address_space *mapping = folio->mapping; if (mapping) { struct inode *inode = mapping->host; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi)) return F2FS_RD_META; if (inode->i_ino == F2FS_NODE_INO(sbi)) return F2FS_RD_NODE; } return F2FS_RD_DATA; } /* postprocessing steps for read bios */ enum bio_post_read_step { #ifdef CONFIG_F2FS_FS_COMPRESSION STEP_DECOMPRESS = BIT(1), #else STEP_DECOMPRESS = 0, /* compile out the decompression-related code */ #endif #ifdef CONFIG_FS_VERITY STEP_VERITY = BIT(2), #else STEP_VERITY = 0, /* compile out the verity-related code */ #endif }; struct bio_post_read_ctx { struct bio *bio; struct f2fs_sb_info *sbi; struct fsverity_info *vi; struct work_struct work; unsigned int enabled_steps; /* * decompression_attempted keeps track of whether * f2fs_end_read_compressed_page() has been called on the pages in the * bio that belong to a compressed cluster yet. */ bool decompression_attempted; block_t fs_blkaddr; }; /* * Update and unlock a bio's pages, and free the bio. * * This marks pages up-to-date only if there was no error in the bio (I/O error, * decryption error, or verity error), as indicated by bio->bi_status. * * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk) * aren't marked up-to-date here, as decompression is done on a per-compression- * cluster basis rather than a per-bio basis. Instead, we only must do two * things for each compressed page here: call f2fs_end_read_compressed_page() * with failed=true if an error occurred before it would have normally gotten * called (i.e., I/O error or decryption error, but *not* verity error), and * release the bio's reference to the decompress_io_ctx of the page's cluster. */ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) { struct folio_iter fi; struct bio_post_read_ctx *ctx = bio->bi_private; unsigned long flags; bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; unsigned nr_pages = fi.length >> PAGE_SHIFT; bool finished = true; if (!folio_test_large(folio) && f2fs_is_compressed_page(folio)) { if (ctx && !ctx->decompression_attempted) f2fs_end_read_compressed_page(folio, true, 0, in_task); f2fs_put_folio_dic(folio, in_task); continue; } if (folio_test_large(folio)) { struct f2fs_folio_state *ffs = folio->private; spin_lock_irqsave(&ffs->state_lock, flags); ffs->read_pages_pending -= nr_pages; finished = !ffs->read_pages_pending; spin_unlock_irqrestore(&ffs->state_lock, flags); } while (nr_pages--) dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); if (bio->bi_status == BLK_STS_OK && F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && f2fs_sanity_check_node_footer(F2FS_F_SB(folio), folio, folio->index, NODE_TYPE_REGULAR, true)) bio->bi_status = BLK_STS_IOERR; if (finished) folio_end_read(folio, bio->bi_status == BLK_STS_OK); } if (ctx) mempool_free(ctx, bio_post_read_ctx_pool); bio_put(bio); } static void f2fs_verify_bio(struct work_struct *work) { struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio; bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); struct fsverity_info *vi = ctx->vi; /* * fsverity_verify_bio() may call readahead() again, and while verity * will be disabled for this, decryption and/or decompression may still * be needed, resulting in another bio_post_read_ctx being allocated. * So to prevent deadlocks we need to release the current ctx to the * mempool first. This assumes that verity is the last post-read step. */ mempool_free(ctx, bio_post_read_ctx_pool); bio->bi_private = NULL; /* * Verify the bio's pages with fs-verity. Exclude compressed pages, * as those were handled separately by f2fs_end_read_compressed_page(). */ if (may_have_compressed_pages) { struct folio_iter fi; bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; if (!f2fs_is_compressed_page(folio) && !fsverity_verify_folio(vi, folio)) { bio->bi_status = BLK_STS_IOERR; break; } } } else { fsverity_verify_bio(vi, bio); } f2fs_finish_read_bio(bio, true); } /* * If the bio's data needs to be verified with fs-verity, then enqueue the * verity work for the bio. Otherwise finish the bio now. * * Note that to avoid deadlocks, the verity work can't be done on the * decryption/decompression workqueue. This is because verifying the data pages * can involve reading verity metadata pages from the file, and these verity * metadata pages may be encrypted and/or compressed. */ static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task) { struct bio_post_read_ctx *ctx = bio->bi_private; if (ctx && (ctx->enabled_steps & STEP_VERITY)) { INIT_WORK(&ctx->work, f2fs_verify_bio); fsverity_enqueue_verify_work(&ctx->work); } else { f2fs_finish_read_bio(bio, in_task); } } /* * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last * remaining page was read by @ctx->bio. * * Note that a bio may span clusters (even a mix of compressed and uncompressed * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates * that the bio includes at least one compressed page. The actual decompression * is done on a per-cluster basis, not a per-bio basis. */ static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx, bool in_task) { struct folio_iter fi; bool all_compressed = true; block_t blkaddr = ctx->fs_blkaddr; bio_for_each_folio_all(fi, ctx->bio) { struct folio *folio = fi.folio; if (f2fs_is_compressed_page(folio)) f2fs_end_read_compressed_page(folio, false, blkaddr, in_task); else all_compressed = false; blkaddr++; } ctx->decompression_attempted = true; /* * Optimization: if all the bio's pages are compressed, then scheduling * the per-bio verity work is unnecessary, as verity will be fully * handled at the compression cluster level. */ if (all_compressed) ctx->enabled_steps &= ~STEP_VERITY; } static void f2fs_post_read_work(struct work_struct *work) { struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio; if (ctx->enabled_steps & STEP_DECOMPRESS) f2fs_handle_step_decompress(ctx, true); f2fs_verify_and_finish_bio(bio, true); } static void f2fs_read_end_io(struct bio *bio) { struct f2fs_sb_info *sbi = F2FS_F_SB(bio_first_folio_all(bio)); struct bio_post_read_ctx *ctx; bool intask = in_task() && !irqs_disabled(); iostat_update_and_unbind_ctx(bio); ctx = bio->bi_private; if (time_to_inject(sbi, FAULT_READ_IO)) bio->bi_status = BLK_STS_IOERR; if (bio->bi_status != BLK_STS_OK) { f2fs_finish_read_bio(bio, intask); return; } if (ctx && (ctx->enabled_steps & STEP_DECOMPRESS)) { if (!f2fs_low_mem_mode(sbi)) { /* Decompress inline. */ f2fs_handle_step_decompress(ctx, intask); } else { INIT_WORK(&ctx->work, f2fs_post_read_work); queue_work(ctx->sbi->wq, &ctx->work); return; } } f2fs_verify_and_finish_bio(bio, intask); } static void f2fs_write_end_bio(struct bio *bio) { struct f2fs_sb_info *sbi = bio->bi_private; struct folio_iter fi; if (time_to_inject(sbi, FAULT_WRITE_IO)) bio->bi_status = BLK_STS_IOERR; bio_for_each_folio_all(fi, bio) { struct folio *folio = fi.folio; enum count_type type; #ifdef CONFIG_F2FS_FS_COMPRESSION if (f2fs_is_compressed_page(folio)) { f2fs_compress_write_end_io(bio, folio); continue; } #endif type = WB_DATA_TYPE(folio, false); if (unlikely(bio->bi_status != BLK_STS_OK)) { mapping_set_error(folio->mapping, -EIO); if (type == F2FS_WB_CP_DATA) { f2fs_stop_checkpoint(sbi, true, STOP_CP_REASON_WRITE_FAIL); } } if (is_node_folio(folio)) { f2fs_sanity_check_node_footer(sbi, folio, folio->index, NODE_TYPE_REGULAR, true); f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); } if (f2fs_in_warm_node_list(folio)) f2fs_del_fsync_node_entry(sbi, folio); dec_page_count(sbi, type); /* * we should access sbi before folio_end_writeback() to * avoid racing w/ kill_f2fs_super() */ if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); folio_clear_f2fs_gcing(folio); folio_end_writeback(folio); } bio_put(bio); } static void f2fs_write_end_io_work(struct work_struct *work) { struct bio *bio = &container_of(work, struct f2fs_bio, work)->bio; f2fs_write_end_bio(bio); } static void f2fs_write_end_io(struct bio *bio) { struct f2fs_sb_info *sbi; iostat_update_and_unbind_ctx(bio); sbi = bio->bi_private; if (in_atomic() && bio->bi_iter.bi_size > sbi->max_atc_write_bio_size) { struct work_struct *w; w = &container_of(bio, struct f2fs_bio, bio)->work; INIT_WORK(w, f2fs_write_end_io_work); queue_work(sbi->wq, w); } else { f2fs_write_end_bio(bio); } } #ifdef CONFIG_BLK_DEV_ZONED static void f2fs_zone_write_end_io(struct bio *bio) { struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private; bio->bi_private = io->bi_private; complete(&io->zone_wait); f2fs_write_end_io(bio); } #endif struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, block_t blk_addr, sector_t *sector) { struct block_device *bdev = sbi->sb->s_bdev; int i; if (f2fs_is_multi_device(sbi)) { for (i = 0; i < sbi->s_ndevs; i++) { if (FDEV(i).start_blk <= blk_addr && FDEV(i).end_blk >= blk_addr) { blk_addr -= FDEV(i).start_blk; bdev = FDEV(i).bdev; break; } } } if (sector) *sector = SECTOR_FROM_BLOCK(blk_addr); return bdev; } int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) { int i; if (!f2fs_is_multi_device(sbi)) return 0; for (i = 0; i < sbi->s_ndevs; i++) if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) return i; return 0; } static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio) { unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0); unsigned int fua_flag, meta_flag, io_flag; blk_opf_t op_flags = 0; if (fio->op != REQ_OP_WRITE) return 0; if (fio->type == DATA) io_flag = fio->sbi->data_io_flag; else if (fio->type == NODE) io_flag = fio->sbi->node_io_flag; else return 0; fua_flag = io_flag & temp_mask; meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask; /* * data/node io flag bits per temp: * REQ_META | REQ_FUA | * 5 | 4 | 3 | 2 | 1 | 0 | * Cold | Warm | Hot | Cold | Warm | Hot | */ if (BIT(fio->temp) & meta_flag) op_flags |= REQ_META; if (BIT(fio->temp) & fua_flag) op_flags |= REQ_FUA; if (fio->type == DATA && F2FS_I(fio->folio->mapping->host)->ioprio_hint == F2FS_IOPRIO_WRITE) op_flags |= REQ_PRIO; return op_flags; } static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) { struct f2fs_sb_info *sbi = fio->sbi; struct block_device *bdev; sector_t sector; struct bio *bio;