// SPDX-License-Identifier: GPL-2.0/* * fs/f2fs/segment.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */#include <linux/fs.h>#include <linux/f2fs_fs.h>#include <linux/bio.h>#include <linux/blkdev.h>#include <linux/prefetch.h>#include <linux/kthread.h>#include <linux/swap.h>#include <linux/timer.h>#include <linux/freezer.h>#include <linux/sched/signal.h> #include "f2fs.h"#include "segment.h"#include "node.h"#include "gc.h"#include "trace.h"#include <trace/events/f2fs.h> #define __reverse_ffz(x) __reverse_ffs(~(x)) static struct kmem_cache *discard_entry_slab;static struct kmem_cache *discard_cmd_slab;static struct kmem_cache *sit_entry_set_slab;static struct kmem_cache *inmem_entry_slab; static unsigned long __reverse_ulong(unsigned char *str){ unsigned long tmp = 0; int shift = 24, idx = 0; #if BITS_PER_LONG == 64 shift = 56;#endif while (shift >= 0) { tmp |= (unsigned long)str[idx++] << shift; shift -= BITS_PER_BYTE; } return tmp;} /* * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since * MSB and LSB are reversed in a byte by f2fs_set_bit. */static inline unsigned long __reverse_ffs(unsigned long word){ int num = 0; #if BITS_PER_LONG == 64 if ((word & 0xffffffff00000000UL) == 0) num += 32; else word >>= 32;#endif if ((word & 0xffff0000) == 0) num += 16; else word >>= 16; if ((word & 0xff00) == 0) num += 8; else word >>= 8; if ((word & 0xf0) == 0) num += 4; else word >>= 4; if ((word & 0xc) == 0) num += 2; else word >>= 2; if ((word & 0x2) == 0) num += 1; return num;} /* * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because * f2fs_set_bit makes MSB and LSB reversed in a byte. * @size must be integral times of unsigned long. * Example: * MSB <--> LSB * f2fs_set_bit(0, bitmap) => 1000 0000 * f2fs_set_bit(7, bitmap) => 0000 0001 */static unsigned long __find_rev_next_bit(const unsigned long *addr, unsigned long size, unsigned long offset){ const unsigned long *p = addr + BIT_WORD(offset); unsigned long result = size; unsigned long tmp; if (offset >= size) return size; size -= (offset & ~(BITS_PER_LONG - 1)); offset %= BITS_PER_LONG; while (1) { if (*p == 0) goto pass; tmp = __reverse_ulong((unsigned char *)p); tmp &= ~0UL >> offset; if (size < BITS_PER_LONG) tmp &= (~0UL << (BITS_PER_LONG - size)); if (tmp) goto found;pass: if (size <= BITS_PER_LONG) break; size -= BITS_PER_LONG; offset = 0; p++; } return result;found: return result - size + __reverse_ffs(tmp);} static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, unsigned long size, unsigned long offset){ const unsigned long *p = addr + BIT_WORD(offset); unsigned long result = size; unsigned long tmp; if (offset >= size) return size; size -= (offset & ~(BITS_PER_LONG - 1)); offset %= BITS_PER_LONG; while (1) { if (*p == ~0UL) goto pass; tmp = __reverse_ulong((unsigned char *)p); if (offset) tmp |= ~0UL << (BITS_PER_LONG - offset); if (size < BITS_PER_LONG) tmp |= ~0UL >> size; if (tmp != ~0UL) goto found;pass: if (size <= BITS_PER_LONG) break; size -= BITS_PER_LONG; offset = 0; p++; } return result;found: return result - size + __reverse_ffz(tmp);} bool f2fs_need_SSR(struct f2fs_sb_info *sbi){ int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); if (f2fs_lfs_mode(sbi)) return false; if (sbi->gc_mode == GC_URGENT_HIGH) return true; if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) return true; return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));} void f2fs_register_inmem_page(struct inode *inode, struct page *page){ struct inmem_pages *new; f2fs_trace_pid(page); f2fs_set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); /* add atomic page indices to the list */ new->page = page; INIT_LIST_HEAD(&new->list); /* increase reference count with clean state */ get_page(page); mutex_lock(&F2FS_I(inode)->inmem_lock); list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages); inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); mutex_unlock(&F2FS_I(inode)->inmem_lock); trace_f2fs_register_inmem_page(page, INMEM);} static int __revoke_inmem_pages(struct inode *inode, struct list_head *head, bool drop, bool recover, bool trylock){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct inmem_pages *cur, *tmp; int err = 0; list_for_each_entry_safe(cur, tmp, head, list) { struct page *page = cur->page; if (drop) trace_f2fs_commit_inmem_page(page, INMEM_DROP); if (trylock) { /* * to avoid deadlock in between page lock and * inmem_lock. */ if (!trylock_page(page)) continue; } else { lock_page(page); } f2fs_wait_on_page_writeback(page, DATA, true, true); if (recover) { struct dnode_of_data dn; struct node_info ni; trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);retry: set_new_dnode(&dn, inode, NULL, NULL, 0); err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE); if (err) { if (err == -ENOMEM) { congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); cond_resched(); goto retry; } err = -EAGAIN; goto next; } err = f2fs_get_node_info(sbi, dn.nid, &ni); if (err) { f2fs_put_dnode(&dn); return err; } if (cur->old_addr == NEW_ADDR) { f2fs_invalidate_blocks(sbi, dn.data_blkaddr); f2fs_update_data_blkaddr(&dn, NEW_ADDR); } else f2fs_replace_block(sbi, &dn, dn.data_blkaddr, cur->old_addr, ni.version, true, true); f2fs_put_dnode(&dn); }next: /* we don't need to invalidate this in the sccessful status */ if (drop || recover) { ClearPageUptodate(page); clear_cold_data(page); } f2fs_clear_page_private(page); f2fs_put_page(page, 1); list_del(&cur->list); kmem_cache_free(inmem_entry_slab, cur); dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); } return err;} void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure){ struct list_head *head = &sbi->inode_list[ATOMIC_FILE]; struct inode *inode; struct f2fs_inode_info *fi; unsigned int count = sbi->atomic_files; unsigned int looped = 0;next: spin_lock(&sbi->inode_lock[ATOMIC_FILE]); if (list_empty(head)) { spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); return; } fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); inode = igrab(&fi->vfs_inode); if (inode) list_move_tail(&fi->inmem_ilist, head); spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); if (inode) { if (gc_failure) { if (!fi->i_gc_failures[GC_FAILURE_ATOMIC]) goto skip; } set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST); f2fs_drop_inmem_pages(inode);skip: iput(inode); } congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); cond_resched(); if (gc_failure) { if (++looped >= count) return; } goto next;} void f2fs_drop_inmem_pages(struct inode *inode){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); while (!list_empty(&fi->inmem_pages)) { mutex_lock(&fi->inmem_lock); __revoke_inmem_pages(inode, &fi->inmem_pages, true, false, true); mutex_unlock(&fi->inmem_lock); } fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0; spin_lock(&sbi->inode_lock[ATOMIC_FILE]); if (!list_empty(&fi->inmem_ilist)) list_del_init(&fi->inmem_ilist); if (f2fs_is_atomic_file(inode)) { clear_inode_flag(inode, FI_ATOMIC_FILE); sbi->atomic_files--; } spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);} void f2fs_drop_inmem_page(struct inode *inode, struct page *page){ struct f2fs_inode_info *fi = F2FS_I(inode); struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct list_head *head = &fi->inmem_pages; struct inmem_pages *cur = NULL; f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page)); mutex_lock(&fi->inmem_lock); list_for_each_entry(cur, head, list) { if (cur->page == page) break; } f2fs_bug_on(sbi, list_empty(head) || cur->page != page); list_del(&cur->list); mutex_unlock(&fi->inmem_lock); dec_page_count(sbi, F2FS_INMEM_PAGES); kmem_cache_free(inmem_entry_slab, cur); ClearPageUptodate(page); f2fs_clear_page_private(page); f2fs_put_page(page, 0); trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);} static int __f2fs_commit_inmem_pages(struct inode *inode){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); struct inmem_pages *cur, *tmp; struct f2fs_io_info fio = { .sbi = sbi, .ino = inode->i_ino, .type = DATA, .op = REQ_OP_WRITE, .op_flags = REQ_SYNC | REQ_PRIO, .io_type = FS_DATA_IO, }; struct list_head revoke_list; bool submit_bio = false; int err = 0; INIT_LIST_HEAD(&revoke_list); list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) { struct page *page = cur->page; lock_page(page); if (page->mapping == inode->i_mapping) { trace_f2fs_commit_inmem_page(page, INMEM); f2fs_wait_on_page_writeback(page, DATA, true, true); set_page_dirty(page); if (clear_page_dirty_for_io(page)) { inode_dec_dirty_pages(inode); f2fs_remove_dirty_inode(inode); }retry: fio.page = page; fio.old_blkaddr = NULL_ADDR; fio.encrypted_page = NULL; fio.need_lock = LOCK_DONE; err = f2fs_do_write_data_page(&fio); if (err) { if (err == -ENOMEM) { congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); cond_resched(); goto retry; } unlock_page(page); break; } /* record old blkaddr for revoking */ cur->old_addr = fio.old_blkaddr; submit_bio = true; } unlock_page(page); list_move_tail(&cur->list, &revoke_list); } if (submit_bio) f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA); if (err) { /* * try to revoke all committed pages, but still we could fail * due to no memory or other reason, if that happened, EAGAIN * will be returned, which means in such case, transaction is * already not integrity, caller should use journal to do the * recovery or rewrite & commit last transaction. For other * error number, revoking was done by filesystem itself. */ err = __revoke_inmem_pages(inode, &revoke_list, false, true, false); /* drop all uncommitted pages */ __revoke_inmem_pages(inode, &fi->inmem_pages, true, false, false); } else { __revoke_inmem_pages(inode, &revoke_list, false, false, false); } return err;} int f2fs_commit_inmem_pages(struct inode *inode){ struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); int err; f2fs_balance_fs(sbi, true); down_write(&fi->i_gc_rwsem[WRITE]); f2fs_lock_op(sbi); set_inode_flag(inode, FI_ATOMIC_COMMIT); mutex_lock(&fi->inmem_lock); err = __f2fs_commit_inmem_pages(inode); mutex_unlock(&fi->inmem_lock); clear_inode_flag(inode, FI_ATOMIC_COMMIT); f2fs_unlock_op(sbi); up_write(&fi->i_gc_rwsem[WRITE]); return err;} /* * This function balances dirty node and dentry pages. * In addition, it controls garbage collection. */void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need){ if (time_to_inject(sbi, FAULT_CHECKPOINT)) { f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); f2fs_stop_checkpoint(sbi, false); } /* balance_fs_bg is able to be pending */ if (need && excess_cached_nats(sbi)) f2fs_balance_fs_bg(sbi, false); Lines 1–500 of 4941. Load more
// SPDX-License-Identifier: GPL-2.0 /* * fs/f2fs/segment.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/prefetch.h> #include <linux/kthread.h> #include <linux/swap.h> #include <linux/timer.h> #include <linux/freezer.h> #include <linux/sched/signal.h> #include "f2fs.h" #include "segment.h" #include "node.h" #include "gc.h" #include "trace.h" #include <trace/events/f2fs.h> #define __reverse_ffz(x) __reverse_ffs(~(x)) static struct kmem_cache *discard_entry_slab; static struct kmem_cache *discard_cmd_slab; static struct kmem_cache *sit_entry_set_slab; static struct kmem_cache *inmem_entry_slab; static unsigned long __reverse_ulong(unsigned char *str) { unsigned long tmp = 0; int shift = 24, idx = 0; #if BITS_PER_LONG == 64 shift = 56; #endif while (shift >= 0) { tmp |= (unsigned long)str[idx++] << shift; shift -= BITS_PER_BYTE; } return tmp; } /* * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since * MSB and LSB are reversed in a byte by f2fs_set_bit. */ static inline unsigned long __reverse_ffs(unsigned long word) { int num = 0; #if BITS_PER_LONG == 64 if ((word & 0xffffffff00000000UL) == 0) num += 32; else word >>= 32; #endif if ((word & 0xffff0000) == 0) num += 16; else word >>= 16; if ((word & 0xff00) == 0) num += 8; else word >>= 8; if ((word & 0xf0) == 0) num += 4; else word >>= 4; if ((word & 0xc) == 0) num += 2; else word >>= 2; if ((word & 0x2) == 0) num += 1; return num; } /* * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because * f2fs_set_bit makes MSB and LSB reversed in a byte. * @size must be integral times of unsigned long. * Example: * MSB <--> LSB * f2fs_set_bit(0, bitmap) => 1000 0000 * f2fs_set_bit(7, bitmap) => 0000 0001 */ static unsigned long __find_rev_next_bit(const unsigned long *addr, unsigned long size, unsigned long offset) { const unsigned long *p = addr + BIT_WORD(offset); unsigned long result = size; unsigned long tmp; if (offset >= size) return size; size -= (offset & ~(BITS_PER_LONG - 1)); offset %= BITS_PER_LONG; while (1) { if (*p == 0) goto pass; tmp = __reverse_ulong((unsigned char *)p); tmp &= ~0UL >> offset; if (size < BITS_PER_LONG) tmp &= (~0UL << (BITS_PER_LONG - size)); if (tmp) goto found; pass: if (size <= BITS_PER_LONG) break; size -= BITS_PER_LONG; offset = 0; p++; } return result; found: return result - size + __reverse_ffs(tmp); } static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, unsigned long size, unsigned long offset) { const unsigned long *p = addr + BIT_WORD(offset); unsigned long result = size; unsigned long tmp; if (offset >= size) return size; size -= (offset & ~(BITS_PER_LONG - 1)); offset %= BITS_PER_LONG; while (1) { if (*p == ~0UL) goto pass; tmp = __reverse_ulong((unsigned char *)p); if (offset) tmp |= ~0UL << (BITS_PER_LONG - offset); if (size < BITS_PER_LONG) tmp |= ~0UL >> size; if (tmp != ~0UL) goto found; pass: if (size <= BITS_PER_LONG) break; size -= BITS_PER_LONG; offset = 0; p++; } return result; found: return result - size + __reverse_ffz(tmp); } bool f2fs_need_SSR(struct f2fs_sb_info *sbi) { int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); if (f2fs_lfs_mode(sbi)) return false; if (sbi->gc_mode == GC_URGENT_HIGH) return true; if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) return true; return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); } void f2fs_register_inmem_page(struct inode *inode, struct page *page) { struct inmem_pages *new; f2fs_trace_pid(page); f2fs_set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); /* add atomic page indices to the list */ new->page = page; INIT_LIST_HEAD(&new->list); /* increase reference count with clean state */ get_page(page); mutex_lock(&F2FS_I(inode)->inmem_lock); list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages); inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); mutex_unlock(&F2FS_I(inode)->inmem_lock); trace_f2fs_register_inmem_page(page, INMEM); } static int __revoke_inmem_pages(struct inode *inode, struct list_head *head, bool drop, bool recover, bool trylock) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct inmem_pages *cur, *tmp; int err = 0; list_for_each_entry_safe(cur, tmp, head, list) { struct page *page = cur->page; if (drop) trace_f2fs_commit_inmem_page(page, INMEM_DROP); if (trylock) { /* * to avoid deadlock in between page lock and * inmem_lock. */ if (!trylock_page(page)) continue; } else { lock_page(page); } f2fs_wait_on_page_writeback(page, DATA, true, true); if (recover) { struct dnode_of_data dn; struct node_info ni; trace_f2fs_commit_inmem_page(page, INMEM_REVOKE); retry: set_new_dnode(&dn, inode, NULL, NULL, 0); err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE); if (err) { if (err == -ENOMEM) { congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); cond_resched(); goto retry; } err = -EAGAIN; goto next; } err = f2fs_get_node_info(sbi, dn.nid, &ni); if (err) { f2fs_put_dnode(&dn); return err; } if (cur->old_addr == NEW_ADDR) { f2fs_invalidate_blocks(sbi, dn.data_blkaddr); f2fs_update_data_blkaddr(&dn, NEW_ADDR); } else f2fs_replace_block(sbi, &dn, dn.data_blkaddr, cur->old_addr, ni.version, true, true); f2fs_put_dnode(&dn); } next: /* we don't need to invalidate this in the sccessful status */ if (drop || recover) { ClearPageUptodate(page); clear_cold_data(page); } f2fs_clear_page_private(page); f2fs_put_page(page, 1); list_del(&cur->list); kmem_cache_free(inmem_entry_slab, cur); dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); } return err; } void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) { struct list_head *head = &sbi->inode_list[ATOMIC_FILE]; struct inode *inode; struct f2fs_inode_info *fi; unsigned int count = sbi->atomic_files; unsigned int looped = 0; next: spin_lock(&sbi->inode_lock[ATOMIC_FILE]); if (list_empty(head)) { spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); return; } fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); inode = igrab(&fi->vfs_inode); if (inode) list_move_tail(&fi->inmem_ilist, head); spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); if (inode) { if (gc_failure) { if (!fi->i_gc_failures[GC_FAILURE_ATOMIC]) goto skip; } set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST); f2fs_drop_inmem_pages(inode); skip: iput(inode); } congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); cond_resched(); if (gc_failure) { if (++looped >= count) return; } goto next; } void f2fs_drop_inmem_pages(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); while (!list_empty(&fi->inmem_pages)) { mutex_lock(&fi->inmem_lock); __revoke_inmem_pages(inode, &fi->inmem_pages, true, false, true); mutex_unlock(&fi->inmem_lock); } fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0; spin_lock(&sbi->inode_lock[ATOMIC_FILE]); if (!list_empty(&fi->inmem_ilist)) list_del_init(&fi->inmem_ilist); if (f2fs_is_atomic_file(inode)) { clear_inode_flag(inode, FI_ATOMIC_FILE); sbi->atomic_files--; } spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); } void f2fs_drop_inmem_page(struct inode *inode, struct page *page) { struct f2fs_inode_info *fi = F2FS_I(inode); struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct list_head *head = &fi->inmem_pages; struct inmem_pages *cur = NULL; f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page)); mutex_lock(&fi->inmem_lock); list_for_each_entry(cur, head, list) { if (cur->page == page) break; } f2fs_bug_on(sbi, list_empty(head) || cur->page != page); list_del(&cur->list); mutex_unlock(&fi->inmem_lock); dec_page_count(sbi, F2FS_INMEM_PAGES); kmem_cache_free(inmem_entry_slab, cur); ClearPageUptodate(page); f2fs_clear_page_private(page); f2fs_put_page(page, 0); trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE); } static int __f2fs_commit_inmem_pages(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); struct inmem_pages *cur, *tmp; struct f2fs_io_info fio = { .sbi = sbi, .ino = inode->i_ino, .type = DATA, .op = REQ_OP_WRITE, .op_flags = REQ_SYNC | REQ_PRIO, .io_type = FS_DATA_IO, }; struct list_head revoke_list; bool submit_bio = false; int err = 0; INIT_LIST_HEAD(&revoke_list); list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) { struct page *page = cur->page; lock_page(page); if (page->mapping == inode->i_mapping) { trace_f2fs_commit_inmem_page(page, INMEM); f2fs_wait_on_page_writeback(page, DATA, true, true); set_page_dirty(page); if (clear_page_dirty_for_io(page)) { inode_dec_dirty_pages(inode); f2fs_remove_dirty_inode(inode); } retry: fio.page = page; fio.old_blkaddr = NULL_ADDR; fio.encrypted_page = NULL; fio.need_lock = LOCK_DONE; err = f2fs_do_write_data_page(&fio); if (err) { if (err == -ENOMEM) { congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); cond_resched(); goto retry; } unlock_page(page); break; } /* record old blkaddr for revoking */ cur->old_addr = fio.old_blkaddr; submit_bio = true; } unlock_page(page); list_move_tail(&cur->list, &revoke_list); } if (submit_bio) f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA); if (err) { /* * try to revoke all committed pages, but still we could fail * due to no memory or other reason, if that happened, EAGAIN * will be returned, which means in such case, transaction is * already not integrity, caller should use journal to do the * recovery or rewrite & commit last transaction. For other * error number, revoking was done by filesystem itself. */ err = __revoke_inmem_pages(inode, &revoke_list, false, true, false); /* drop all uncommitted pages */ __revoke_inmem_pages(inode, &fi->inmem_pages, true, false, false); } else { __revoke_inmem_pages(inode, &revoke_list, false, false, false); } return err; } int f2fs_commit_inmem_pages(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); int err; f2fs_balance_fs(sbi, true); down_write(&fi->i_gc_rwsem[WRITE]); f2fs_lock_op(sbi); set_inode_flag(inode, FI_ATOMIC_COMMIT); mutex_lock(&fi->inmem_lock); err = __f2fs_commit_inmem_pages(inode); mutex_unlock(&fi->inmem_lock); clear_inode_flag(inode, FI_ATOMIC_COMMIT); f2fs_unlock_op(sbi); up_write(&fi->i_gc_rwsem[WRITE]); return err; } /* * This function balances dirty node and dentry pages. * In addition, it controls garbage collection. */ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) { if (time_to_inject(sbi, FAULT_CHECKPOINT)) { f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); f2fs_stop_checkpoint(sbi, false); } /* balance_fs_bg is able to be pending */ if (need && excess_cached_nats(sbi)) f2fs_balance_fs_bg(sbi, false);