/* * fs/f2fs/data.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */#include <linux/fs.h>#include <linux/f2fs_fs.h>#include <linux/buffer_head.h>#include <linux/mpage.h>#include <linux/writeback.h>#include <linux/backing-dev.h>#include <linux/pagevec.h>#include <linux/blkdev.h>#include <linux/bio.h>#include <linux/prefetch.h>#include <linux/uio.h>#include <linux/mm.h>#include <linux/memcontrol.h>#include <linux/cleancache.h>#include <linux/sched/signal.h> #include "f2fs.h"#include "node.h"#include "segment.h"#include "trace.h"#include <trace/events/f2fs.h> static bool __is_cp_guaranteed(struct page *page){ struct address_space *mapping = page->mapping; struct inode *inode; struct f2fs_sb_info *sbi; if (!mapping) return false; inode = mapping->host; sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi) || inode->i_ino == F2FS_NODE_INO(sbi) || S_ISDIR(inode->i_mode) || is_cold_data(page)) return true; return false;} static void f2fs_read_end_io(struct bio *bio){ struct bio_vec *bvec; int i; #ifdef CONFIG_F2FS_FAULT_INJECTION if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) { f2fs_show_injection_info(FAULT_IO); bio->bi_status = BLK_STS_IOERR; }#endif if (f2fs_bio_encrypted(bio)) { if (bio->bi_status) { fscrypt_release_ctx(bio->bi_private); } else { fscrypt_decrypt_bio_pages(bio->bi_private, bio); return; } } bio_for_each_segment_all(bvec, bio, i) { struct page *page = bvec->bv_page; if (!bio->bi_status) { if (!PageUptodate(page)) SetPageUptodate(page); } else { ClearPageUptodate(page); SetPageError(page); } unlock_page(page); } bio_put(bio);} static void f2fs_write_end_io(struct bio *bio){ struct f2fs_sb_info *sbi = bio->bi_private; struct bio_vec *bvec; int i; bio_for_each_segment_all(bvec, bio, i) { struct page *page = bvec->bv_page; enum count_type type = WB_DATA_TYPE(page); if (IS_DUMMY_WRITTEN_PAGE(page)) { set_page_private(page, (unsigned long)NULL); ClearPagePrivate(page); unlock_page(page); mempool_free(page, sbi->write_io_dummy); if (unlikely(bio->bi_status)) f2fs_stop_checkpoint(sbi, true); continue; } fscrypt_pullback_bio_page(&page, true); if (unlikely(bio->bi_status)) { mapping_set_error(page->mapping, -EIO); f2fs_stop_checkpoint(sbi, true); } dec_page_count(sbi, type); clear_cold_data(page); end_page_writeback(page); } if (!get_pages(sbi, F2FS_WB_CP_DATA) && wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); bio_put(bio);} /* * Return true, if pre_bio's bdev is same as its target device. */struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, block_t blk_addr, struct bio *bio){ struct block_device *bdev = sbi->sb->s_bdev; int i; for (i = 0; i < sbi->s_ndevs; i++) { if (FDEV(i).start_blk <= blk_addr && FDEV(i).end_blk >= blk_addr) { blk_addr -= FDEV(i).start_blk; bdev = FDEV(i).bdev; break; } } if (bio) { bio->bi_bdev = bdev; bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); } return bdev;} int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr){ int i; for (i = 0; i < sbi->s_ndevs; i++) if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) return i; return 0;} static bool __same_bdev(struct f2fs_sb_info *sbi, block_t blk_addr, struct bio *bio){ return f2fs_target_device(sbi, blk_addr, NULL) == bio->bi_bdev;} /* * Low-level block read/write IO operations. */static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, int npages, bool is_read){ struct bio *bio; bio = f2fs_bio_alloc(npages); f2fs_target_device(sbi, blk_addr, bio); bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; bio->bi_private = is_read ? NULL : sbi; return bio;} static inline void __submit_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type){ if (!is_read_io(bio_op(bio))) { unsigned int start; if (f2fs_sb_mounted_blkzoned(sbi->sb) && current->plug && (type == DATA || type == NODE)) blk_finish_plug(current->plug); if (type != DATA && type != NODE) goto submit_io; start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS; start %= F2FS_IO_SIZE(sbi); if (start == 0) goto submit_io; /* fill dummy pages */ for (; start < F2FS_IO_SIZE(sbi); start++) { struct page *page = mempool_alloc(sbi->write_io_dummy, GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL); f2fs_bug_on(sbi, !page); SetPagePrivate(page); set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE); lock_page(page); if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) f2fs_bug_on(sbi, 1); } /* * In the NODE case, we lose next block address chain. So, we * need to do checkpoint in f2fs_sync_file. */ if (type == NODE) set_sbi_flag(sbi, SBI_NEED_CP); }submit_io: if (is_read_io(bio_op(bio))) trace_f2fs_submit_read_bio(sbi->sb, type, bio); else trace_f2fs_submit_write_bio(sbi->sb, type, bio); submit_bio(bio);} static void __submit_merged_bio(struct f2fs_bio_info *io){ struct f2fs_io_info *fio = &io->fio; if (!io->bio) return; bio_set_op_attrs(io->bio, fio->op, fio->op_flags); if (is_read_io(fio->op)) trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); else trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); __submit_bio(io->sbi, io->bio, fio->type); io->bio = NULL;} static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode, nid_t ino, pgoff_t idx){ struct bio_vec *bvec; struct page *target; int i; if (!io->bio) return false; if (!inode && !ino) return true; bio_for_each_segment_all(bvec, io->bio, i) { if (bvec->bv_page->mapping) target = bvec->bv_page; else target = fscrypt_control_page(bvec->bv_page); if (idx != target->index) continue; if (inode && inode == target->mapping->host) return true; if (ino && ino == ino_of_node(target)) return true; } return false;} static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode, nid_t ino, pgoff_t idx, enum page_type type){ enum page_type btype = PAGE_TYPE_OF_BIO(type); enum temp_type temp; struct f2fs_bio_info *io; bool ret = false; for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { io = sbi->write_io[btype] + temp; down_read(&io->io_rwsem); ret = __has_merged_page(io, inode, ino, idx); up_read(&io->io_rwsem); /* TODO: use HOT temp only for meta pages now. */ if (ret || btype == META) break; } return ret;} static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type, enum temp_type temp){ enum page_type btype = PAGE_TYPE_OF_BIO(type); struct f2fs_bio_info *io = sbi->write_io[btype] + temp; down_write(&io->io_rwsem); /* change META to META_FLUSH in the checkpoint procedure */ if (type >= META_FLUSH) { io->fio.type = META_FLUSH; io->fio.op = REQ_OP_WRITE; io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC; if (!test_opt(sbi, NOBARRIER)) io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA; } __submit_merged_bio(io); up_write(&io->io_rwsem);} static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, struct inode *inode, nid_t ino, pgoff_t idx, enum page_type type, bool force){ enum temp_type temp; if (!force && !has_merged_page(sbi, inode, ino, idx, type)) return; for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { __f2fs_submit_merged_write(sbi, type, temp); /* TODO: use HOT temp only for meta pages now. */ if (type >= META) break; }} void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type){ __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);} void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, struct inode *inode, nid_t ino, pgoff_t idx, enum page_type type){ __submit_merged_write_cond(sbi, inode, ino, idx, type, false);} void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi){ f2fs_submit_merged_write(sbi, DATA); f2fs_submit_merged_write(sbi, NODE); f2fs_submit_merged_write(sbi, META);} /* * Fill the locked page with data located in the block address. * A caller needs to unlock the page on failure. */int f2fs_submit_page_bio(struct f2fs_io_info *fio){ struct bio *bio; struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; trace_f2fs_submit_page_bio(page, fio); f2fs_trace_ios(fio, 0); /* Allocate a new bio */ bio = __bio_alloc(fio->sbi, fio->new_blkaddr, 1, is_read_io(fio->op)); if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { bio_put(bio); return -EFAULT; } bio_set_op_attrs(bio, fio->op, fio->op_flags); __submit_bio(fio->sbi, bio, fio->type); if (!is_read_io(fio->op)) inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page)); return 0;} int f2fs_submit_page_write(struct f2fs_io_info *fio){ struct f2fs_sb_info *sbi = fio->sbi; enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; struct page *bio_page; int err = 0; f2fs_bug_on(sbi, is_read_io(fio->op)); down_write(&io->io_rwsem);next: if (fio->in_list) { spin_lock(&io->io_lock); if (list_empty(&io->io_list)) { spin_unlock(&io->io_lock); goto out_fail; } fio = list_first_entry(&io->io_list, struct f2fs_io_info, list); list_del(&fio->list); spin_unlock(&io->io_lock); } if (fio->old_blkaddr != NEW_ADDR) verify_block_addr(sbi, fio->old_blkaddr); verify_block_addr(sbi, fio->new_blkaddr); bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; /* set submitted = 1 as a return value */ fio->submitted = 1; inc_page_count(sbi, WB_DATA_TYPE(bio_page)); if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 || (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) || !__same_bdev(sbi, fio->new_blkaddr, io->bio))) __submit_merged_bio(io);alloc_new: if (io->bio == NULL) { if ((fio->type == DATA || fio->type == NODE) && fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { err = -EAGAIN; dec_page_count(sbi, WB_DATA_TYPE(bio_page)); goto out_fail; } io->bio = __bio_alloc(sbi, fio->new_blkaddr, BIO_MAX_PAGES, false); io->fio = *fio; } if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { __submit_merged_bio(io); goto alloc_new; } io->last_block_in_bio = fio->new_blkaddr; f2fs_trace_ios(fio, 0); trace_f2fs_submit_page_write(fio->page, fio); if (fio->in_list) goto next;out_fail: up_write(&io->io_rwsem); return err;} static void __set_data_blkaddr(struct dnode_of_data *dn){ struct f2fs_node *rn = F2FS_NODE(dn->node_page); __le32 *addr_array; /* Get physical address of data block */ addr_array = blkaddr_in_node(rn); addr_array[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);} /* * Lock ordering for the change of data block address: * ->data_page * ->node_page * update block addresses in the node page */void set_data_blkaddr(struct dnode_of_data *dn){ f2fs_wait_on_page_writeback(dn->node_page, NODE, true); __set_data_blkaddr(dn); if (set_page_dirty(dn->node_page)) dn->node_changed = true;} void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr){ dn->data_blkaddr = blkaddr; set_data_blkaddr(dn); f2fs_update_extent_cache(dn);} /* dn->ofs_in_node will be returned with up-to-date last block pointer */int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count){ struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); int err; if (!count) return 0; if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) return -EPERM;Lines 1–500 of 2276. Load more
/* * fs/f2fs/data.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/buffer_head.h> #include <linux/mpage.h> #include <linux/writeback.h> #include <linux/backing-dev.h> #include <linux/pagevec.h> #include <linux/blkdev.h> #include <linux/bio.h> #include <linux/prefetch.h> #include <linux/uio.h> #include <linux/mm.h> #include <linux/memcontrol.h> #include <linux/cleancache.h> #include <linux/sched/signal.h> #include "f2fs.h" #include "node.h" #include "segment.h" #include "trace.h" #include <trace/events/f2fs.h> static bool __is_cp_guaranteed(struct page *page) { struct address_space *mapping = page->mapping; struct inode *inode; struct f2fs_sb_info *sbi; if (!mapping) return false; inode = mapping->host; sbi = F2FS_I_SB(inode); if (inode->i_ino == F2FS_META_INO(sbi) || inode->i_ino == F2FS_NODE_INO(sbi) || S_ISDIR(inode->i_mode) || is_cold_data(page)) return true; return false; } static void f2fs_read_end_io(struct bio *bio) { struct bio_vec *bvec; int i; #ifdef CONFIG_F2FS_FAULT_INJECTION if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) { f2fs_show_injection_info(FAULT_IO); bio->bi_status = BLK_STS_IOERR; } #endif if (f2fs_bio_encrypted(bio)) { if (bio->bi_status) { fscrypt_release_ctx(bio->bi_private); } else { fscrypt_decrypt_bio_pages(bio->bi_private, bio); return; } } bio_for_each_segment_all(bvec, bio, i) { struct page *page = bvec->bv_page; if (!bio->bi_status) { if (!PageUptodate(page)) SetPageUptodate(page); } else { ClearPageUptodate(page); SetPageError(page); } unlock_page(page); } bio_put(bio); } static void f2fs_write_end_io(struct bio *bio) { struct f2fs_sb_info *sbi = bio->bi_private; struct bio_vec *bvec; int i; bio_for_each_segment_all(bvec, bio, i) { struct page *page = bvec->bv_page; enum count_type type = WB_DATA_TYPE(page); if (IS_DUMMY_WRITTEN_PAGE(page)) { set_page_private(page, (unsigned long)NULL); ClearPagePrivate(page); unlock_page(page); mempool_free(page, sbi->write_io_dummy); if (unlikely(bio->bi_status)) f2fs_stop_checkpoint(sbi, true); continue; } fscrypt_pullback_bio_page(&page, true); if (unlikely(bio->bi_status)) { mapping_set_error(page->mapping, -EIO); f2fs_stop_checkpoint(sbi, true); } dec_page_count(sbi, type); clear_cold_data(page); end_page_writeback(page); } if (!get_pages(sbi, F2FS_WB_CP_DATA) && wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); bio_put(bio); } /* * Return true, if pre_bio's bdev is same as its target device. */ struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, block_t blk_addr, struct bio *bio) { struct block_device *bdev = sbi->sb->s_bdev; int i; for (i = 0; i < sbi->s_ndevs; i++) { if (FDEV(i).start_blk <= blk_addr && FDEV(i).end_blk >= blk_addr) { blk_addr -= FDEV(i).start_blk; bdev = FDEV(i).bdev; break; } } if (bio) { bio->bi_bdev = bdev; bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); } return bdev; } int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) { int i; for (i = 0; i < sbi->s_ndevs; i++) if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) return i; return 0; } static bool __same_bdev(struct f2fs_sb_info *sbi, block_t blk_addr, struct bio *bio) { return f2fs_target_device(sbi, blk_addr, NULL) == bio->bi_bdev; } /* * Low-level block read/write IO operations. */ static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, int npages, bool is_read) { struct bio *bio; bio = f2fs_bio_alloc(npages); f2fs_target_device(sbi, blk_addr, bio); bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; bio->bi_private = is_read ? NULL : sbi; return bio; } static inline void __submit_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type) { if (!is_read_io(bio_op(bio))) { unsigned int start; if (f2fs_sb_mounted_blkzoned(sbi->sb) && current->plug && (type == DATA || type == NODE)) blk_finish_plug(current->plug); if (type != DATA && type != NODE) goto submit_io; start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS; start %= F2FS_IO_SIZE(sbi); if (start == 0) goto submit_io; /* fill dummy pages */ for (; start < F2FS_IO_SIZE(sbi); start++) { struct page *page = mempool_alloc(sbi->write_io_dummy, GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL); f2fs_bug_on(sbi, !page); SetPagePrivate(page); set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE); lock_page(page); if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) f2fs_bug_on(sbi, 1); } /* * In the NODE case, we lose next block address chain. So, we * need to do checkpoint in f2fs_sync_file. */ if (type == NODE) set_sbi_flag(sbi, SBI_NEED_CP); } submit_io: if (is_read_io(bio_op(bio))) trace_f2fs_submit_read_bio(sbi->sb, type, bio); else trace_f2fs_submit_write_bio(sbi->sb, type, bio); submit_bio(bio); } static void __submit_merged_bio(struct f2fs_bio_info *io) { struct f2fs_io_info *fio = &io->fio; if (!io->bio) return; bio_set_op_attrs(io->bio, fio->op, fio->op_flags); if (is_read_io(fio->op)) trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); else trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); __submit_bio(io->sbi, io->bio, fio->type); io->bio = NULL; } static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode, nid_t ino, pgoff_t idx) { struct bio_vec *bvec; struct page *target; int i; if (!io->bio) return false; if (!inode && !ino) return true; bio_for_each_segment_all(bvec, io->bio, i) { if (bvec->bv_page->mapping) target = bvec->bv_page; else target = fscrypt_control_page(bvec->bv_page); if (idx != target->index) continue; if (inode && inode == target->mapping->host) return true; if (ino && ino == ino_of_node(target)) return true; } return false; } static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode, nid_t ino, pgoff_t idx, enum page_type type) { enum page_type btype = PAGE_TYPE_OF_BIO(type); enum temp_type temp; struct f2fs_bio_info *io; bool ret = false; for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { io = sbi->write_io[btype] + temp; down_read(&io->io_rwsem); ret = __has_merged_page(io, inode, ino, idx); up_read(&io->io_rwsem); /* TODO: use HOT temp only for meta pages now. */ if (ret || btype == META) break; } return ret; } static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type, enum temp_type temp) { enum page_type btype = PAGE_TYPE_OF_BIO(type); struct f2fs_bio_info *io = sbi->write_io[btype] + temp; down_write(&io->io_rwsem); /* change META to META_FLUSH in the checkpoint procedure */ if (type >= META_FLUSH) { io->fio.type = META_FLUSH; io->fio.op = REQ_OP_WRITE; io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC; if (!test_opt(sbi, NOBARRIER)) io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA; } __submit_merged_bio(io); up_write(&io->io_rwsem); } static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, struct inode *inode, nid_t ino, pgoff_t idx, enum page_type type, bool force) { enum temp_type temp; if (!force && !has_merged_page(sbi, inode, ino, idx, type)) return; for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { __f2fs_submit_merged_write(sbi, type, temp); /* TODO: use HOT temp only for meta pages now. */ if (type >= META) break; } } void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) { __submit_merged_write_cond(sbi, NULL, 0, 0, type, true); } void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, struct inode *inode, nid_t ino, pgoff_t idx, enum page_type type) { __submit_merged_write_cond(sbi, inode, ino, idx, type, false); } void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) { f2fs_submit_merged_write(sbi, DATA); f2fs_submit_merged_write(sbi, NODE); f2fs_submit_merged_write(sbi, META); } /* * Fill the locked page with data located in the block address. * A caller needs to unlock the page on failure. */ int f2fs_submit_page_bio(struct f2fs_io_info *fio) { struct bio *bio; struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; trace_f2fs_submit_page_bio(page, fio); f2fs_trace_ios(fio, 0); /* Allocate a new bio */ bio = __bio_alloc(fio->sbi, fio->new_blkaddr, 1, is_read_io(fio->op)); if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { bio_put(bio); return -EFAULT; } bio_set_op_attrs(bio, fio->op, fio->op_flags); __submit_bio(fio->sbi, bio, fio->type); if (!is_read_io(fio->op)) inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page)); return 0; } int f2fs_submit_page_write(struct f2fs_io_info *fio) { struct f2fs_sb_info *sbi = fio->sbi; enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; struct page *bio_page; int err = 0; f2fs_bug_on(sbi, is_read_io(fio->op)); down_write(&io->io_rwsem); next: if (fio->in_list) { spin_lock(&io->io_lock); if (list_empty(&io->io_list)) { spin_unlock(&io->io_lock); goto out_fail; } fio = list_first_entry(&io->io_list, struct f2fs_io_info, list); list_del(&fio->list); spin_unlock(&io->io_lock); } if (fio->old_blkaddr != NEW_ADDR) verify_block_addr(sbi, fio->old_blkaddr); verify_block_addr(sbi, fio->new_blkaddr); bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; /* set submitted = 1 as a return value */ fio->submitted = 1; inc_page_count(sbi, WB_DATA_TYPE(bio_page)); if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 || (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) || !__same_bdev(sbi, fio->new_blkaddr, io->bio))) __submit_merged_bio(io); alloc_new: if (io->bio == NULL) { if ((fio->type == DATA || fio->type == NODE) && fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { err = -EAGAIN; dec_page_count(sbi, WB_DATA_TYPE(bio_page)); goto out_fail; } io->bio = __bio_alloc(sbi, fio->new_blkaddr, BIO_MAX_PAGES, false); io->fio = *fio; } if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { __submit_merged_bio(io); goto alloc_new; } io->last_block_in_bio = fio->new_blkaddr; f2fs_trace_ios(fio, 0); trace_f2fs_submit_page_write(fio->page, fio); if (fio->in_list) goto next; out_fail: up_write(&io->io_rwsem); return err; } static void __set_data_blkaddr(struct dnode_of_data *dn) { struct f2fs_node *rn = F2FS_NODE(dn->node_page); __le32 *addr_array; /* Get physical address of data block */ addr_array = blkaddr_in_node(rn); addr_array[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); } /* * Lock ordering for the change of data block address: * ->data_page * ->node_page * update block addresses in the node page */ void set_data_blkaddr(struct dnode_of_data *dn) { f2fs_wait_on_page_writeback(dn->node_page, NODE, true); __set_data_blkaddr(dn); if (set_page_dirty(dn->node_page)) dn->node_changed = true; } void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) { dn->data_blkaddr = blkaddr; set_data_blkaddr(dn); f2fs_update_extent_cache(dn); } /* dn->ofs_in_node will be returned with up-to-date last block pointer */ int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) { struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); int err; if (!count) return 0; if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) return -EPERM;