// SPDX-License-Identifier: GPL-2.0/* * fs/f2fs/node.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */#include <linux/fs.h>#include <linux/f2fs_fs.h>#include <linux/mpage.h>#include <linux/sched/mm.h>#include <linux/blkdev.h>#include <linux/pagevec.h>#include <linux/swap.h> #include "f2fs.h"#include "node.h"#include "segment.h"#include "xattr.h"#include "iostat.h"#include <trace/events/f2fs.h> #define on_f2fs_build_free_nids(nm_i) mutex_is_locked(&(nm_i)->build_lock) static struct kmem_cache *nat_entry_slab;static struct kmem_cache *free_nid_slab;static struct kmem_cache *nat_entry_set_slab;static struct kmem_cache *fsync_node_entry_slab; /* * Check whether the given nid is within node id range. */int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid){ if (unlikely(nid < F2FS_ROOT_INO(sbi) || nid >= NM_I(sbi)->max_nid)) { set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_warn(sbi, "%s: out-of-range nid=%x, run fsck to fix.", __func__, nid); f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); return -EFSCORRUPTED; } return 0;} bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; struct sysinfo val; unsigned long avail_ram; unsigned long mem_size = 0; bool res = false; if (!nm_i) return true; si_meminfo(&val); /* only uses low memory */ avail_ram = val.totalram - val.totalhigh; /* * give 25%, 25%, 50%, 50%, 25%, 25% memory for each components respectively */ if (type == FREE_NIDS) { mem_size = (nm_i->nid_cnt[FREE_NID] * sizeof(struct free_nid)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == NAT_ENTRIES) { mem_size = (nm_i->nat_cnt[TOTAL_NAT] * sizeof(struct nat_entry)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); if (excess_cached_nats(sbi)) res = false; } else if (type == DIRTY_DENTS) { if (sbi->sb->s_bdi->wb.dirty_exceeded) return false; mem_size = get_pages(sbi, F2FS_DIRTY_DENTS); res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else if (type == INO_ENTRIES) { int i; for (i = 0; i < MAX_INO_ENTRY; i++) mem_size += sbi->im[i].ino_num * sizeof(struct ino_entry); mem_size >>= PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else if (type == READ_EXTENT_CACHE || type == AGE_EXTENT_CACHE) { enum extent_type etype = type == READ_EXTENT_CACHE ? EX_READ : EX_BLOCK_AGE; struct extent_tree_info *eti = &sbi->extent_tree[etype]; mem_size = (atomic_read(&eti->total_ext_tree) * sizeof(struct extent_tree) + atomic_read(&eti->total_ext_node) * sizeof(struct extent_node)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == DISCARD_CACHE) { mem_size = (atomic_read(&dcc->discard_cmd_cnt) * sizeof(struct discard_cmd)) >> PAGE_SHIFT; res = mem_size < (avail_ram * nm_i->ram_thresh / 100); } else if (type == COMPRESS_PAGE) {#ifdef CONFIG_F2FS_FS_COMPRESSION unsigned long free_ram = val.freeram; /* * free memory is lower than watermark or cached page count * exceed threshold, deny caching compress page. */ res = (free_ram > avail_ram * sbi->compress_watermark / 100) && (COMPRESS_MAPPING(sbi)->nrpages < free_ram * sbi->compress_percent / 100);#else res = false;#endif } else { if (!sbi->sb->s_bdi->wb.dirty_exceeded) return true; } return res;} static void clear_node_page_dirty(struct page *page){ if (PageDirty(page)) { f2fs_clear_page_cache_dirty_tag(page_folio(page)); clear_page_dirty_for_io(page); dec_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES); } ClearPageUptodate(page);} static struct page *get_current_nat_page(struct f2fs_sb_info *sbi, nid_t nid){ return f2fs_get_meta_page_retry(sbi, current_nat_addr(sbi, nid));} static struct page *get_next_nat_page(struct f2fs_sb_info *sbi, nid_t nid){ struct page *src_page; struct page *dst_page; pgoff_t dst_off; void *src_addr; void *dst_addr; struct f2fs_nm_info *nm_i = NM_I(sbi); dst_off = next_nat_addr(sbi, current_nat_addr(sbi, nid)); /* get current nat block page with lock */ src_page = get_current_nat_page(sbi, nid); if (IS_ERR(src_page)) return src_page; dst_page = f2fs_grab_meta_page(sbi, dst_off); f2fs_bug_on(sbi, PageDirty(src_page)); src_addr = page_address(src_page); dst_addr = page_address(dst_page); memcpy(dst_addr, src_addr, PAGE_SIZE); set_page_dirty(dst_page); f2fs_put_page(src_page, 1); set_to_next_nat(nm_i, nid); return dst_page;} static struct nat_entry *__alloc_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, bool no_fail){ struct nat_entry *new; new = f2fs_kmem_cache_alloc(nat_entry_slab, GFP_F2FS_ZERO, no_fail, sbi); if (new) { nat_set_nid(new, nid); nat_reset_flag(new); } return new;} static void __free_nat_entry(struct nat_entry *e){ kmem_cache_free(nat_entry_slab, e);} /* must be locked by nat_tree_lock */static struct nat_entry *__init_nat_entry(struct f2fs_nm_info *nm_i, struct nat_entry *ne, struct f2fs_nat_entry *raw_ne, bool no_fail){ if (no_fail) f2fs_radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne); else if (radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne)) return NULL; if (raw_ne) node_info_from_raw_nat(&ne->ni, raw_ne); spin_lock(&nm_i->nat_list_lock); list_add_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock); nm_i->nat_cnt[TOTAL_NAT]++; nm_i->nat_cnt[RECLAIMABLE_NAT]++; return ne;} static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n){ struct nat_entry *ne; ne = radix_tree_lookup(&nm_i->nat_root, n); /* for recent accessed nat entry, move it to tail of lru list */ if (ne && !get_nat_flag(ne, IS_DIRTY)) { spin_lock(&nm_i->nat_list_lock); if (!list_empty(&ne->list)) list_move_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock); } return ne;} static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t start, unsigned int nr, struct nat_entry **ep){ return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);} static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e){ radix_tree_delete(&nm_i->nat_root, nat_get_nid(e)); nm_i->nat_cnt[TOTAL_NAT]--; nm_i->nat_cnt[RECLAIMABLE_NAT]--; __free_nat_entry(e);} static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i, struct nat_entry *ne){ nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid); struct nat_entry_set *head; head = radix_tree_lookup(&nm_i->nat_set_root, set); if (!head) { head = f2fs_kmem_cache_alloc(nat_entry_set_slab, GFP_NOFS, true, NULL); INIT_LIST_HEAD(&head->entry_list); INIT_LIST_HEAD(&head->set_list); head->set = set; head->entry_cnt = 0; f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head); } return head;} static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i, struct nat_entry *ne){ struct nat_entry_set *head; bool new_ne = nat_get_blkaddr(ne) == NEW_ADDR; if (!new_ne) head = __grab_nat_entry_set(nm_i, ne); /* * update entry_cnt in below condition: * 1. update NEW_ADDR to valid block address; * 2. update old block address to new one; */ if (!new_ne && (get_nat_flag(ne, IS_PREALLOC) || !get_nat_flag(ne, IS_DIRTY))) head->entry_cnt++; set_nat_flag(ne, IS_PREALLOC, new_ne); if (get_nat_flag(ne, IS_DIRTY)) goto refresh_list; nm_i->nat_cnt[DIRTY_NAT]++; nm_i->nat_cnt[RECLAIMABLE_NAT]--; set_nat_flag(ne, IS_DIRTY, true);refresh_list: spin_lock(&nm_i->nat_list_lock); if (new_ne) list_del_init(&ne->list); else list_move_tail(&ne->list, &head->entry_list); spin_unlock(&nm_i->nat_list_lock);} static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i, struct nat_entry_set *set, struct nat_entry *ne){ spin_lock(&nm_i->nat_list_lock); list_move_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock); set_nat_flag(ne, IS_DIRTY, false); set->entry_cnt--; nm_i->nat_cnt[DIRTY_NAT]--; nm_i->nat_cnt[RECLAIMABLE_NAT]++;} static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i, nid_t start, unsigned int nr, struct nat_entry_set **ep){ return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep, start, nr);} bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct page *page){ return NODE_MAPPING(sbi) == page->mapping && IS_DNODE(page) && is_cold_node(page);} void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi){ spin_lock_init(&sbi->fsync_node_lock); INIT_LIST_HEAD(&sbi->fsync_node_list); sbi->fsync_seg_id = 0; sbi->fsync_node_num = 0;} static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi, struct page *page){ struct fsync_node_entry *fn; unsigned long flags; unsigned int seq_id; fn = f2fs_kmem_cache_alloc(fsync_node_entry_slab, GFP_NOFS, true, NULL); get_page(page); fn->page = page; INIT_LIST_HEAD(&fn->list); spin_lock_irqsave(&sbi->fsync_node_lock, flags); list_add_tail(&fn->list, &sbi->fsync_node_list); fn->seq_id = sbi->fsync_seg_id++; seq_id = fn->seq_id; sbi->fsync_node_num++; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); return seq_id;} void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct page *page){ struct fsync_node_entry *fn; unsigned long flags; spin_lock_irqsave(&sbi->fsync_node_lock, flags); list_for_each_entry(fn, &sbi->fsync_node_list, list) { if (fn->page == page) { list_del(&fn->list); sbi->fsync_node_num--; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); kmem_cache_free(fsync_node_entry_slab, fn); put_page(page); return; } } spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); f2fs_bug_on(sbi, 1);} void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi){ unsigned long flags; spin_lock_irqsave(&sbi->fsync_node_lock, flags); sbi->fsync_seg_id = 0; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);} int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need = false; f2fs_down_read(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, nid); if (e) { if (!get_nat_flag(e, IS_CHECKPOINTED) && !get_nat_flag(e, HAS_FSYNCED_INODE)) need = true; } f2fs_up_read(&nm_i->nat_tree_lock); return need;} bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool is_cp = true; f2fs_down_read(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, nid); if (e && !get_nat_flag(e, IS_CHECKPOINTED)) is_cp = false; f2fs_up_read(&nm_i->nat_tree_lock); return is_cp;} bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need_update = true; f2fs_down_read(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, ino); if (e && get_nat_flag(e, HAS_LAST_FSYNC) && (get_nat_flag(e, IS_CHECKPOINTED) || get_nat_flag(e, HAS_FSYNCED_INODE))) need_update = false; f2fs_up_read(&nm_i->nat_tree_lock); return need_update;} /* must be locked by nat_tree_lock */static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, struct f2fs_nat_entry *ne){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *new, *e; /* Let's mitigate lock contention of nat_tree_lock during checkpoint */ if (f2fs_rwsem_is_locked(&sbi->cp_global_sem)) return; new = __alloc_nat_entry(sbi, nid, false); if (!new) return; f2fs_down_write(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, nid); if (!e) e = __init_nat_entry(nm_i, new, ne, false); else f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) || nat_get_blkaddr(e) != le32_to_cpu(ne->block_addr) || nat_get_version(e) != ne->version); f2fs_up_write(&nm_i->nat_tree_lock); if (e != new) __free_nat_entry(new);} static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, block_t new_blkaddr, bool fsync_done){ struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true); f2fs_down_write(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, ni->nid); if (!e) { e = __init_nat_entry(nm_i, new, NULL, true); copy_node_info(&e->ni, ni); f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR); } else if (new_blkaddr == NEW_ADDR) { /* * when nid is reallocated, * previous nat entry can be remained in nat cache. * So, reinitialize it with new information. */ copy_node_info(&e->ni, ni); f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR); } /* let's free early to reduce memory consumption */ if (e != new) __free_nat_entry(new); /* sanity check */ f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr); f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR && new_blkaddr == NULL_ADDR); f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR && new_blkaddr == NEW_ADDR); f2fs_bug_on(sbi, __is_valid_data_blkaddr(nat_get_blkaddr(e)) && new_blkaddr == NEW_ADDR); /* increment version no as node is removed */ if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) { unsigned char version = nat_get_version(e); nat_set_version(e, inc_node_version(version)); } /* change address */ nat_set_blkaddr(e, new_blkaddr); if (!__is_valid_data_blkaddr(new_blkaddr))Lines 1–500 of 3485. Load more
// SPDX-License-Identifier: GPL-2.0 /* * fs/f2fs/node.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/mpage.h> #include <linux/sched/mm.h> #include <linux/blkdev.h> #include <linux/pagevec.h> #include <linux/swap.h> #include "f2fs.h" #include "node.h" #include "segment.h" #include "xattr.h" #include "iostat.h" #include <trace/events/f2fs.h> #define on_f2fs_build_free_nids(nm_i) mutex_is_locked(&(nm_i)->build_lock) static struct kmem_cache *nat_entry_slab; static struct kmem_cache *free_nid_slab; static struct kmem_cache *nat_entry_set_slab; static struct kmem_cache *fsync_node_entry_slab; /* * Check whether the given nid is within node id range. */ int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid) { if (unlikely(nid < F2FS_ROOT_INO(sbi) || nid >= NM_I(sbi)->max_nid)) { set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_warn(sbi, "%s: out-of-range nid=%x, run fsck to fix.", __func__, nid); f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); return -EFSCORRUPTED; } return 0; } bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) { struct f2fs_nm_info *nm_i = NM_I(sbi); struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; struct sysinfo val; unsigned long avail_ram; unsigned long mem_size = 0; bool res = false; if (!nm_i) return true; si_meminfo(&val); /* only uses low memory */ avail_ram = val.totalram - val.totalhigh; /* * give 25%, 25%, 50%, 50%, 25%, 25% memory for each components respectively */ if (type == FREE_NIDS) { mem_size = (nm_i->nid_cnt[FREE_NID] * sizeof(struct free_nid)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == NAT_ENTRIES) { mem_size = (nm_i->nat_cnt[TOTAL_NAT] * sizeof(struct nat_entry)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); if (excess_cached_nats(sbi)) res = false; } else if (type == DIRTY_DENTS) { if (sbi->sb->s_bdi->wb.dirty_exceeded) return false; mem_size = get_pages(sbi, F2FS_DIRTY_DENTS); res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else if (type == INO_ENTRIES) { int i; for (i = 0; i < MAX_INO_ENTRY; i++) mem_size += sbi->im[i].ino_num * sizeof(struct ino_entry); mem_size >>= PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); } else if (type == READ_EXTENT_CACHE || type == AGE_EXTENT_CACHE) { enum extent_type etype = type == READ_EXTENT_CACHE ? EX_READ : EX_BLOCK_AGE; struct extent_tree_info *eti = &sbi->extent_tree[etype]; mem_size = (atomic_read(&eti->total_ext_tree) * sizeof(struct extent_tree) + atomic_read(&eti->total_ext_node) * sizeof(struct extent_node)) >> PAGE_SHIFT; res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); } else if (type == DISCARD_CACHE) { mem_size = (atomic_read(&dcc->discard_cmd_cnt) * sizeof(struct discard_cmd)) >> PAGE_SHIFT; res = mem_size < (avail_ram * nm_i->ram_thresh / 100); } else if (type == COMPRESS_PAGE) { #ifdef CONFIG_F2FS_FS_COMPRESSION unsigned long free_ram = val.freeram; /* * free memory is lower than watermark or cached page count * exceed threshold, deny caching compress page. */ res = (free_ram > avail_ram * sbi->compress_watermark / 100) && (COMPRESS_MAPPING(sbi)->nrpages < free_ram * sbi->compress_percent / 100); #else res = false; #endif } else { if (!sbi->sb->s_bdi->wb.dirty_exceeded) return true; } return res; } static void clear_node_page_dirty(struct page *page) { if (PageDirty(page)) { f2fs_clear_page_cache_dirty_tag(page_folio(page)); clear_page_dirty_for_io(page); dec_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES); } ClearPageUptodate(page); } static struct page *get_current_nat_page(struct f2fs_sb_info *sbi, nid_t nid) { return f2fs_get_meta_page_retry(sbi, current_nat_addr(sbi, nid)); } static struct page *get_next_nat_page(struct f2fs_sb_info *sbi, nid_t nid) { struct page *src_page; struct page *dst_page; pgoff_t dst_off; void *src_addr; void *dst_addr; struct f2fs_nm_info *nm_i = NM_I(sbi); dst_off = next_nat_addr(sbi, current_nat_addr(sbi, nid)); /* get current nat block page with lock */ src_page = get_current_nat_page(sbi, nid); if (IS_ERR(src_page)) return src_page; dst_page = f2fs_grab_meta_page(sbi, dst_off); f2fs_bug_on(sbi, PageDirty(src_page)); src_addr = page_address(src_page); dst_addr = page_address(dst_page); memcpy(dst_addr, src_addr, PAGE_SIZE); set_page_dirty(dst_page); f2fs_put_page(src_page, 1); set_to_next_nat(nm_i, nid); return dst_page; } static struct nat_entry *__alloc_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, bool no_fail) { struct nat_entry *new; new = f2fs_kmem_cache_alloc(nat_entry_slab, GFP_F2FS_ZERO, no_fail, sbi); if (new) { nat_set_nid(new, nid); nat_reset_flag(new); } return new; } static void __free_nat_entry(struct nat_entry *e) { kmem_cache_free(nat_entry_slab, e); } /* must be locked by nat_tree_lock */ static struct nat_entry *__init_nat_entry(struct f2fs_nm_info *nm_i, struct nat_entry *ne, struct f2fs_nat_entry *raw_ne, bool no_fail) { if (no_fail) f2fs_radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne); else if (radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne)) return NULL; if (raw_ne) node_info_from_raw_nat(&ne->ni, raw_ne); spin_lock(&nm_i->nat_list_lock); list_add_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock); nm_i->nat_cnt[TOTAL_NAT]++; nm_i->nat_cnt[RECLAIMABLE_NAT]++; return ne; } static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n) { struct nat_entry *ne; ne = radix_tree_lookup(&nm_i->nat_root, n); /* for recent accessed nat entry, move it to tail of lru list */ if (ne && !get_nat_flag(ne, IS_DIRTY)) { spin_lock(&nm_i->nat_list_lock); if (!list_empty(&ne->list)) list_move_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock); } return ne; } static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t start, unsigned int nr, struct nat_entry **ep) { return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr); } static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e) { radix_tree_delete(&nm_i->nat_root, nat_get_nid(e)); nm_i->nat_cnt[TOTAL_NAT]--; nm_i->nat_cnt[RECLAIMABLE_NAT]--; __free_nat_entry(e); } static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i, struct nat_entry *ne) { nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid); struct nat_entry_set *head; head = radix_tree_lookup(&nm_i->nat_set_root, set); if (!head) { head = f2fs_kmem_cache_alloc(nat_entry_set_slab, GFP_NOFS, true, NULL); INIT_LIST_HEAD(&head->entry_list); INIT_LIST_HEAD(&head->set_list); head->set = set; head->entry_cnt = 0; f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head); } return head; } static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i, struct nat_entry *ne) { struct nat_entry_set *head; bool new_ne = nat_get_blkaddr(ne) == NEW_ADDR; if (!new_ne) head = __grab_nat_entry_set(nm_i, ne); /* * update entry_cnt in below condition: * 1. update NEW_ADDR to valid block address; * 2. update old block address to new one; */ if (!new_ne && (get_nat_flag(ne, IS_PREALLOC) || !get_nat_flag(ne, IS_DIRTY))) head->entry_cnt++; set_nat_flag(ne, IS_PREALLOC, new_ne); if (get_nat_flag(ne, IS_DIRTY)) goto refresh_list; nm_i->nat_cnt[DIRTY_NAT]++; nm_i->nat_cnt[RECLAIMABLE_NAT]--; set_nat_flag(ne, IS_DIRTY, true); refresh_list: spin_lock(&nm_i->nat_list_lock); if (new_ne) list_del_init(&ne->list); else list_move_tail(&ne->list, &head->entry_list); spin_unlock(&nm_i->nat_list_lock); } static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i, struct nat_entry_set *set, struct nat_entry *ne) { spin_lock(&nm_i->nat_list_lock); list_move_tail(&ne->list, &nm_i->nat_entries); spin_unlock(&nm_i->nat_list_lock); set_nat_flag(ne, IS_DIRTY, false); set->entry_cnt--; nm_i->nat_cnt[DIRTY_NAT]--; nm_i->nat_cnt[RECLAIMABLE_NAT]++; } static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i, nid_t start, unsigned int nr, struct nat_entry_set **ep) { return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep, start, nr); } bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct page *page) { return NODE_MAPPING(sbi) == page->mapping && IS_DNODE(page) && is_cold_node(page); } void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi) { spin_lock_init(&sbi->fsync_node_lock); INIT_LIST_HEAD(&sbi->fsync_node_list); sbi->fsync_seg_id = 0; sbi->fsync_node_num = 0; } static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi, struct page *page) { struct fsync_node_entry *fn; unsigned long flags; unsigned int seq_id; fn = f2fs_kmem_cache_alloc(fsync_node_entry_slab, GFP_NOFS, true, NULL); get_page(page); fn->page = page; INIT_LIST_HEAD(&fn->list); spin_lock_irqsave(&sbi->fsync_node_lock, flags); list_add_tail(&fn->list, &sbi->fsync_node_list); fn->seq_id = sbi->fsync_seg_id++; seq_id = fn->seq_id; sbi->fsync_node_num++; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); return seq_id; } void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct page *page) { struct fsync_node_entry *fn; unsigned long flags; spin_lock_irqsave(&sbi->fsync_node_lock, flags); list_for_each_entry(fn, &sbi->fsync_node_list, list) { if (fn->page == page) { list_del(&fn->list); sbi->fsync_node_num--; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); kmem_cache_free(fsync_node_entry_slab, fn); put_page(page); return; } } spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); f2fs_bug_on(sbi, 1); } void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi) { unsigned long flags; spin_lock_irqsave(&sbi->fsync_node_lock, flags); sbi->fsync_seg_id = 0; spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); } int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid) { struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need = false; f2fs_down_read(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, nid); if (e) { if (!get_nat_flag(e, IS_CHECKPOINTED) && !get_nat_flag(e, HAS_FSYNCED_INODE)) need = true; } f2fs_up_read(&nm_i->nat_tree_lock); return need; } bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid) { struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool is_cp = true; f2fs_down_read(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, nid); if (e && !get_nat_flag(e, IS_CHECKPOINTED)) is_cp = false; f2fs_up_read(&nm_i->nat_tree_lock); return is_cp; } bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino) { struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need_update = true; f2fs_down_read(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, ino); if (e && get_nat_flag(e, HAS_LAST_FSYNC) && (get_nat_flag(e, IS_CHECKPOINTED) || get_nat_flag(e, HAS_FSYNCED_INODE))) need_update = false; f2fs_up_read(&nm_i->nat_tree_lock); return need_update; } /* must be locked by nat_tree_lock */ static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, struct f2fs_nat_entry *ne) { struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *new, *e; /* Let's mitigate lock contention of nat_tree_lock during checkpoint */ if (f2fs_rwsem_is_locked(&sbi->cp_global_sem)) return; new = __alloc_nat_entry(sbi, nid, false); if (!new) return; f2fs_down_write(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, nid); if (!e) e = __init_nat_entry(nm_i, new, ne, false); else f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) || nat_get_blkaddr(e) != le32_to_cpu(ne->block_addr) || nat_get_version(e) != ne->version); f2fs_up_write(&nm_i->nat_tree_lock); if (e != new) __free_nat_entry(new); } static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, block_t new_blkaddr, bool fsync_done) { struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true); f2fs_down_write(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, ni->nid); if (!e) { e = __init_nat_entry(nm_i, new, NULL, true); copy_node_info(&e->ni, ni); f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR); } else if (new_blkaddr == NEW_ADDR) { /* * when nid is reallocated, * previous nat entry can be remained in nat cache. * So, reinitialize it with new information. */ copy_node_info(&e->ni, ni); f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR); } /* let's free early to reduce memory consumption */ if (e != new) __free_nat_entry(new); /* sanity check */ f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr); f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR && new_blkaddr == NULL_ADDR); f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR && new_blkaddr == NEW_ADDR); f2fs_bug_on(sbi, __is_valid_data_blkaddr(nat_get_blkaddr(e)) && new_blkaddr == NEW_ADDR); /* increment version no as node is removed */ if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) { unsigned char version = nat_get_version(e); nat_set_version(e, inc_node_version(version)); } /* change address */ nat_set_blkaddr(e, new_blkaddr); if (!__is_valid_data_blkaddr(new_blkaddr))