// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2007 Oracle. All rights reserved. */ #include <linux/slab.h>#include <linux/blkdev.h>#include <linux/writeback.h>#include <linux/sched/mm.h>#include "messages.h"#include "misc.h"#include "ctree.h"#include "transaction.h"#include "btrfs_inode.h"#include "extent_io.h"#include "disk-io.h"#include "compression.h"#include "delalloc-space.h"#include "qgroup.h"#include "subpage.h"#include "file.h"#include "block-group.h" static struct kmem_cache *btrfs_ordered_extent_cache; static u64 entry_end(struct btrfs_ordered_extent *entry){ if (entry->file_offset + entry->num_bytes < entry->file_offset) return (u64)-1; return entry->file_offset + entry->num_bytes;} /* returns NULL if the insertion worked, or it returns the node it did find * in the tree */static struct rb_node *tree_insert(struct rb_root *root, u64 file_offset, struct rb_node *node){ struct rb_node **p = &root->rb_node; struct rb_node *parent = NULL; struct btrfs_ordered_extent *entry; while (*p) { parent = *p; entry = rb_entry(parent, struct btrfs_ordered_extent, rb_node); if (file_offset < entry->file_offset) p = &(*p)->rb_left; else if (file_offset >= entry_end(entry)) p = &(*p)->rb_right; else return parent; } rb_link_node(node, parent, p); rb_insert_color(node, root); return NULL;} /* * look for a given offset in the tree, and if it can't be found return the * first lesser offset */static struct rb_node *__tree_search(struct rb_root *root, u64 file_offset, struct rb_node **prev_ret){ struct rb_node *n = root->rb_node; struct rb_node *prev = NULL; struct rb_node *test; struct btrfs_ordered_extent *entry; struct btrfs_ordered_extent *prev_entry = NULL; while (n) { entry = rb_entry(n, struct btrfs_ordered_extent, rb_node); prev = n; prev_entry = entry; if (file_offset < entry->file_offset) n = n->rb_left; else if (file_offset >= entry_end(entry)) n = n->rb_right; else return n; } if (!prev_ret) return NULL; while (prev && file_offset >= entry_end(prev_entry)) { test = rb_next(prev); if (!test) break; prev_entry = rb_entry(test, struct btrfs_ordered_extent, rb_node); if (file_offset < entry_end(prev_entry)) break; prev = test; } if (prev) prev_entry = rb_entry(prev, struct btrfs_ordered_extent, rb_node); while (prev && file_offset < entry_end(prev_entry)) { test = rb_prev(prev); if (!test) break; prev_entry = rb_entry(test, struct btrfs_ordered_extent, rb_node); prev = test; } *prev_ret = prev; return NULL;} static int btrfs_range_overlaps(struct btrfs_ordered_extent *entry, u64 file_offset, u64 len){ if (file_offset + len <= entry->file_offset || entry->file_offset + entry->num_bytes <= file_offset) return 0; return 1;} /* * look find the first ordered struct that has this offset, otherwise * the first one less than this offset */static inline struct rb_node *ordered_tree_search(struct btrfs_inode *inode, u64 file_offset){ struct rb_node *prev = NULL; struct rb_node *ret; struct btrfs_ordered_extent *entry; if (inode->ordered_tree_last) { entry = rb_entry(inode->ordered_tree_last, struct btrfs_ordered_extent, rb_node); if (in_range(file_offset, entry->file_offset, entry->num_bytes)) return inode->ordered_tree_last; } ret = __tree_search(&inode->ordered_tree, file_offset, &prev); if (!ret) ret = prev; if (ret) inode->ordered_tree_last = ret; return ret;} static struct btrfs_ordered_extent *alloc_ordered_extent( struct btrfs_inode *inode, u64 file_offset, u64 num_bytes, u64 ram_bytes, u64 disk_bytenr, u64 disk_num_bytes, u64 offset, unsigned long flags, int compress_type){ struct btrfs_ordered_extent *entry; int ret; u64 qgroup_rsv = 0; const bool is_nocow = (flags & ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))); /* Only one type flag can be set. */ ASSERT(has_single_bit_set(flags & BTRFS_ORDERED_EXCLUSIVE_FLAGS), "flags=0x%lx", flags); /* DIRECT cannot be set with COMPRESSED nor ENCODED. */ if (test_bit(BTRFS_ORDERED_DIRECT, &flags)) { ASSERT(!test_bit(BTRFS_ORDERED_COMPRESSED, &flags)); ASSERT(!test_bit(BTRFS_ORDERED_ENCODED, &flags)); } /* ENCODED must be set with COMPRESSED. */ if (test_bit(BTRFS_ORDERED_ENCODED, &flags)) ASSERT(test_bit(BTRFS_ORDERED_COMPRESSED, &flags)); /* * For a NOCOW write we can free the qgroup reserve right now. For a COW * one we transfer the reserved space from the inode's iotree into the * ordered extent by calling btrfs_qgroup_release_data() and tracking * the qgroup reserved amount in the ordered extent, so that later after * completing the ordered extent, when running the data delayed ref it * creates, we free the reserved data with btrfs_qgroup_free_refroot(). */ if (is_nocow) ret = btrfs_qgroup_free_data(inode, NULL, file_offset, num_bytes, &qgroup_rsv); else ret = btrfs_qgroup_release_data(inode, file_offset, num_bytes, &qgroup_rsv); if (ret < 0) return ERR_PTR(ret); entry = kmem_cache_zalloc(btrfs_ordered_extent_cache, GFP_NOFS); if (!entry) { entry = ERR_PTR(-ENOMEM); goto out; } entry->file_offset = file_offset; entry->num_bytes = num_bytes; entry->ram_bytes = ram_bytes; entry->disk_bytenr = disk_bytenr; entry->disk_num_bytes = disk_num_bytes; entry->offset = offset; entry->bytes_left = num_bytes; if (WARN_ON_ONCE(!igrab(&inode->vfs_inode))) { kmem_cache_free(btrfs_ordered_extent_cache, entry); entry = ERR_PTR(-ESTALE); goto out; } entry->inode = inode; entry->compress_type = compress_type; entry->truncated_len = (u64)-1; entry->qgroup_rsv = qgroup_rsv; entry->flags = flags; refcount_set(&entry->refs, 1); init_waitqueue_head(&entry->wait); INIT_LIST_HEAD(&entry->csum_list); INIT_LIST_HEAD(&entry->log_list); INIT_LIST_HEAD(&entry->root_extent_list); INIT_LIST_HEAD(&entry->work_list); INIT_LIST_HEAD(&entry->bioc_list); init_completion(&entry->completion); /* * We don't need the count_max_extents here, we can assume that all of * that work has been done at higher layers, so this is truly the * smallest the extent is going to get. */ spin_lock(&inode->lock); btrfs_mod_outstanding_extents(inode, 1); spin_unlock(&inode->lock); out: if (IS_ERR(entry) && !is_nocow) btrfs_qgroup_free_refroot(inode->root->fs_info, btrfs_root_id(inode->root), qgroup_rsv, BTRFS_QGROUP_RSV_DATA); return entry;} static void insert_ordered_extent(struct btrfs_ordered_extent *entry){ struct btrfs_inode *inode = entry->inode; struct btrfs_root *root = inode->root; struct btrfs_fs_info *fs_info = root->fs_info; struct rb_node *node; trace_btrfs_ordered_extent_add(inode, entry); percpu_counter_add_batch(&fs_info->ordered_bytes, entry->num_bytes, fs_info->delalloc_batch); /* One ref for the tree. */ refcount_inc(&entry->refs); spin_lock(&inode->ordered_tree_lock); node = tree_insert(&inode->ordered_tree, entry->file_offset, &entry->rb_node); if (unlikely(node)) { struct btrfs_ordered_extent *exist = rb_entry(node, struct btrfs_ordered_extent, rb_node); btrfs_panic(fs_info, -EEXIST,"overlapping ordered extents, existing oe file_offset %llu num_bytes %llu flags 0x%lx, new oe file_offset %llu num_bytes %llu flags 0x%lx", exist->file_offset, exist->num_bytes, exist->flags, entry->file_offset, entry->num_bytes, entry->flags); } spin_unlock(&inode->ordered_tree_lock); spin_lock(&root->ordered_extent_lock); list_add_tail(&entry->root_extent_list, &root->ordered_extents); root->nr_ordered_extents++; if (root->nr_ordered_extents == 1) { spin_lock(&fs_info->ordered_root_lock); BUG_ON(!list_empty(&root->ordered_root)); list_add_tail(&root->ordered_root, &fs_info->ordered_roots); spin_unlock(&fs_info->ordered_root_lock); } spin_unlock(&root->ordered_extent_lock);} /* * Add an ordered extent to the per-inode tree. * * @inode: Inode that this extent is for. * @file_offset: Logical offset in file where the extent starts. * @num_bytes: Logical length of extent in file. * @ram_bytes: Full length of unencoded data. * @disk_bytenr: Offset of extent on disk. * @disk_num_bytes: Size of extent on disk. * @offset: Offset into unencoded data where file data starts. * @flags: Flags specifying type of extent (1U << BTRFS_ORDERED_*). * @compress_type: Compression algorithm used for data. * * Most of these parameters correspond to &struct btrfs_file_extent_item. The * tree is given a single reference on the ordered extent that was inserted, and * the returned pointer is given a second reference. * * Return: the new ordered extent or error pointer. */struct btrfs_ordered_extent *btrfs_alloc_ordered_extent( struct btrfs_inode *inode, u64 file_offset, const struct btrfs_file_extent *file_extent, unsigned long flags){ struct btrfs_ordered_extent *entry; ASSERT((flags & ~BTRFS_ORDERED_TYPE_FLAGS) == 0, "flags=0x%lx", flags); /* * For regular writes, we just use the members in @file_extent. * * For NOCOW, we don't really care about the numbers except @start and * file_extent->num_bytes, as we won't insert a file extent item at all. * * For PREALLOC, we do not use ordered extent members, but * btrfs_mark_extent_written() handles everything. * * So here we always pass 0 as offset for NOCOW/PREALLOC ordered extents, * or btrfs_split_ordered_extent() cannot handle it correctly. */ if (flags & ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))) entry = alloc_ordered_extent(inode, file_offset, file_extent->num_bytes, file_extent->num_bytes, file_extent->disk_bytenr + file_extent->offset, file_extent->num_bytes, 0, flags, file_extent->compression); else entry = alloc_ordered_extent(inode, file_offset, file_extent->num_bytes, file_extent->ram_bytes, file_extent->disk_bytenr, file_extent->disk_num_bytes, file_extent->offset, flags, file_extent->compression); if (!IS_ERR(entry)) insert_ordered_extent(entry); return entry;} /* * Add a struct btrfs_ordered_sum into the list of checksums to be inserted * when an ordered extent is finished. If the list covers more than one * ordered extent, it is split across multiples. */void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry, struct btrfs_ordered_sum *sum){ struct btrfs_inode *inode = entry->inode; spin_lock(&inode->ordered_tree_lock); list_add_tail(&sum->list, &entry->csum_list); spin_unlock(&inode->ordered_tree_lock);} void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered){ if (!test_and_set_bit(BTRFS_ORDERED_IOERR, &ordered->flags)) mapping_set_error(ordered->inode->vfs_inode.i_mapping, -EIO);} void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, u64 truncate_len){ struct btrfs_inode *inode = ordered->inode; ASSERT(truncate_len <= ordered->num_bytes); spin_lock(&inode->ordered_tree_lock); set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); ordered->truncated_len = min(ordered->truncated_len, truncate_len); spin_unlock(&inode->ordered_tree_lock);} static void finish_ordered_fn(struct btrfs_work *work){ struct btrfs_ordered_extent *ordered_extent; ordered_extent = container_of(work, struct btrfs_ordered_extent, work); btrfs_finish_ordered_io(ordered_extent);} static bool can_finish_ordered_extent(struct btrfs_ordered_extent *ordered, u64 file_offset, u64 len, bool uptodate){ struct btrfs_inode *inode = ordered->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; lockdep_assert_held(&inode->ordered_tree_lock); /* Now we're fine to update the accounting. */ if (WARN_ON_ONCE(len > ordered->bytes_left)) { btrfs_crit(fs_info,"bad ordered extent accounting, root=%llu ino=%llu OE offset=%llu OE len=%llu to_dec=%llu left=%llu", btrfs_root_id(inode->root), btrfs_ino(inode), ordered->file_offset, ordered->num_bytes, len, ordered->bytes_left); ordered->bytes_left = 0; } else { ordered->bytes_left -= len; } if (!uptodate) btrfs_mark_ordered_extent_error(ordered); if (ordered->bytes_left) return false; /* * All the IO of the ordered extent is finished, we need to queue * the finish_func to be executed. */ set_bit(BTRFS_ORDERED_IO_DONE, &ordered->flags); cond_wake_up(&ordered->wait); refcount_inc(&ordered->refs); trace_btrfs_ordered_extent_mark_finished(inode, ordered); return true;} static void btrfs_queue_ordered_fn(struct btrfs_ordered_extent *ordered){ struct btrfs_inode *inode = ordered->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_workqueue *wq = btrfs_is_free_space_inode(inode) ? fs_info->endio_freespace_worker : fs_info->endio_write_workers; btrfs_init_work(&ordered->work, finish_ordered_fn, NULL); btrfs_queue_work(wq, &ordered->work);} void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered, u64 file_offset, u64 len, bool uptodate){ struct btrfs_inode *inode = ordered->inode; bool ret; trace_btrfs_finish_ordered_extent(inode, file_offset, len, uptodate); spin_lock(&inode->ordered_tree_lock); ret = can_finish_ordered_extent(ordered, file_offset, len, uptodate); spin_unlock(&inode->ordered_tree_lock); /* * If this is a COW write it means we created new extent maps for the * range and they point to unwritten locations if we got an error either * before submitting a bio or during IO. * * We have marked the ordered extent with BTRFS_ORDERED_IOERR, and we * are queuing its completion below. During completion, at * btrfs_finish_one_ordered(), we will drop the extent maps for the * unwritten extents. * * However because completion runs in a work queue we can end up having * a fast fsync running before that. In the case of direct IO, once we * unlock the inode the fsync might start, and we queue the completion * before unlocking the inode. In the case of buffered IO when writeback * finishes (end_bbio_data_write()) we queue the completion, so if the * writeback was triggered by a fast fsync, the fsync might start * logging before ordered extent completion runs in the work queue. * * The fast fsync will log file extent items based on the extent maps it * finds, so if by the time it collects extent maps the ordered extent * completion didn't happen yet, it will log file extent items that * point to unwritten extents, resulting in a corruption if a crash * happens and the log tree is replayed. Note that a fast fsync does not * wait for completion of ordered extents in order to reduce latency. * * Set a flag in the inode so that the next fast fsync will wait for * ordered extents to complete before starting to log. */ if (!uptodate && !test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) set_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags); if (ret) btrfs_queue_ordered_fn(ordered);} /* * Mark all ordered extents io inside the specified range finished. * * @folio: The involved folio for the operation. * For uncompressed buffered IO, the folio status also needs to be * updated to indicate whether the pending ordered io is finished. * Can be NULL for direct IO and compressed write. * For these cases, callers are ensured they won't execute the * endio function twice. * * This function is called for endio, thus the range must have ordered * extent(s) covering it. */void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode, u64 file_offset, u64 num_bytes, bool uptodate){ struct rb_node *node; struct btrfs_ordered_extent *entry = NULL; u64 cur = file_offset; const u64 end = file_offset + num_bytes; trace_btrfs_writepage_end_io_hook(inode, file_offset, end - 1, uptodate); spin_lock(&inode->ordered_tree_lock);Lines 1–500 of 1371. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2007 Oracle. All rights reserved. */ #include <linux/slab.h> #include <linux/blkdev.h> #include <linux/writeback.h> #include <linux/sched/mm.h> #include "messages.h" #include "misc.h" #include "ctree.h" #include "transaction.h" #include "btrfs_inode.h" #include "extent_io.h" #include "disk-io.h" #include "compression.h" #include "delalloc-space.h" #include "qgroup.h" #include "subpage.h" #include "file.h" #include "block-group.h" static struct kmem_cache *btrfs_ordered_extent_cache; static u64 entry_end(struct btrfs_ordered_extent *entry) { if (entry->file_offset + entry->num_bytes < entry->file_offset) return (u64)-1; return entry->file_offset + entry->num_bytes; } /* returns NULL if the insertion worked, or it returns the node it did find * in the tree */ static struct rb_node *tree_insert(struct rb_root *root, u64 file_offset, struct rb_node *node) { struct rb_node **p = &root->rb_node; struct rb_node *parent = NULL; struct btrfs_ordered_extent *entry; while (*p) { parent = *p; entry = rb_entry(parent, struct btrfs_ordered_extent, rb_node); if (file_offset < entry->file_offset) p = &(*p)->rb_left; else if (file_offset >= entry_end(entry)) p = &(*p)->rb_right; else return parent; } rb_link_node(node, parent, p); rb_insert_color(node, root); return NULL; } /* * look for a given offset in the tree, and if it can't be found return the * first lesser offset */ static struct rb_node *__tree_search(struct rb_root *root, u64 file_offset, struct rb_node **prev_ret) { struct rb_node *n = root->rb_node; struct rb_node *prev = NULL; struct rb_node *test; struct btrfs_ordered_extent *entry; struct btrfs_ordered_extent *prev_entry = NULL; while (n) { entry = rb_entry(n, struct btrfs_ordered_extent, rb_node); prev = n; prev_entry = entry; if (file_offset < entry->file_offset) n = n->rb_left; else if (file_offset >= entry_end(entry)) n = n->rb_right; else return n; } if (!prev_ret) return NULL; while (prev && file_offset >= entry_end(prev_entry)) { test = rb_next(prev); if (!test) break; prev_entry = rb_entry(test, struct btrfs_ordered_extent, rb_node); if (file_offset < entry_end(prev_entry)) break; prev = test; } if (prev) prev_entry = rb_entry(prev, struct btrfs_ordered_extent, rb_node); while (prev && file_offset < entry_end(prev_entry)) { test = rb_prev(prev); if (!test) break; prev_entry = rb_entry(test, struct btrfs_ordered_extent, rb_node); prev = test; } *prev_ret = prev; return NULL; } static int btrfs_range_overlaps(struct btrfs_ordered_extent *entry, u64 file_offset, u64 len) { if (file_offset + len <= entry->file_offset || entry->file_offset + entry->num_bytes <= file_offset) return 0; return 1; } /* * look find the first ordered struct that has this offset, otherwise * the first one less than this offset */ static inline struct rb_node *ordered_tree_search(struct btrfs_inode *inode, u64 file_offset) { struct rb_node *prev = NULL; struct rb_node *ret; struct btrfs_ordered_extent *entry; if (inode->ordered_tree_last) { entry = rb_entry(inode->ordered_tree_last, struct btrfs_ordered_extent, rb_node); if (in_range(file_offset, entry->file_offset, entry->num_bytes)) return inode->ordered_tree_last; } ret = __tree_search(&inode->ordered_tree, file_offset, &prev); if (!ret) ret = prev; if (ret) inode->ordered_tree_last = ret; return ret; } static struct btrfs_ordered_extent *alloc_ordered_extent( struct btrfs_inode *inode, u64 file_offset, u64 num_bytes, u64 ram_bytes, u64 disk_bytenr, u64 disk_num_bytes, u64 offset, unsigned long flags, int compress_type) { struct btrfs_ordered_extent *entry; int ret; u64 qgroup_rsv = 0; const bool is_nocow = (flags & ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))); /* Only one type flag can be set. */ ASSERT(has_single_bit_set(flags & BTRFS_ORDERED_EXCLUSIVE_FLAGS), "flags=0x%lx", flags); /* DIRECT cannot be set with COMPRESSED nor ENCODED. */ if (test_bit(BTRFS_ORDERED_DIRECT, &flags)) { ASSERT(!test_bit(BTRFS_ORDERED_COMPRESSED, &flags)); ASSERT(!test_bit(BTRFS_ORDERED_ENCODED, &flags)); } /* ENCODED must be set with COMPRESSED. */ if (test_bit(BTRFS_ORDERED_ENCODED, &flags)) ASSERT(test_bit(BTRFS_ORDERED_COMPRESSED, &flags)); /* * For a NOCOW write we can free the qgroup reserve right now. For a COW * one we transfer the reserved space from the inode's iotree into the * ordered extent by calling btrfs_qgroup_release_data() and tracking * the qgroup reserved amount in the ordered extent, so that later after * completing the ordered extent, when running the data delayed ref it * creates, we free the reserved data with btrfs_qgroup_free_refroot(). */ if (is_nocow) ret = btrfs_qgroup_free_data(inode, NULL, file_offset, num_bytes, &qgroup_rsv); else ret = btrfs_qgroup_release_data(inode, file_offset, num_bytes, &qgroup_rsv); if (ret < 0) return ERR_PTR(ret); entry = kmem_cache_zalloc(btrfs_ordered_extent_cache, GFP_NOFS); if (!entry) { entry = ERR_PTR(-ENOMEM); goto out; } entry->file_offset = file_offset; entry->num_bytes = num_bytes; entry->ram_bytes = ram_bytes; entry->disk_bytenr = disk_bytenr; entry->disk_num_bytes = disk_num_bytes; entry->offset = offset; entry->bytes_left = num_bytes; if (WARN_ON_ONCE(!igrab(&inode->vfs_inode))) { kmem_cache_free(btrfs_ordered_extent_cache, entry); entry = ERR_PTR(-ESTALE); goto out; } entry->inode = inode; entry->compress_type = compress_type; entry->truncated_len = (u64)-1; entry->qgroup_rsv = qgroup_rsv; entry->flags = flags; refcount_set(&entry->refs, 1); init_waitqueue_head(&entry->wait); INIT_LIST_HEAD(&entry->csum_list); INIT_LIST_HEAD(&entry->log_list); INIT_LIST_HEAD(&entry->root_extent_list); INIT_LIST_HEAD(&entry->work_list); INIT_LIST_HEAD(&entry->bioc_list); init_completion(&entry->completion); /* * We don't need the count_max_extents here, we can assume that all of * that work has been done at higher layers, so this is truly the * smallest the extent is going to get. */ spin_lock(&inode->lock); btrfs_mod_outstanding_extents(inode, 1); spin_unlock(&inode->lock); out: if (IS_ERR(entry) && !is_nocow) btrfs_qgroup_free_refroot(inode->root->fs_info, btrfs_root_id(inode->root), qgroup_rsv, BTRFS_QGROUP_RSV_DATA); return entry; } static void insert_ordered_extent(struct btrfs_ordered_extent *entry) { struct btrfs_inode *inode = entry->inode; struct btrfs_root *root = inode->root; struct btrfs_fs_info *fs_info = root->fs_info; struct rb_node *node; trace_btrfs_ordered_extent_add(inode, entry); percpu_counter_add_batch(&fs_info->ordered_bytes, entry->num_bytes, fs_info->delalloc_batch); /* One ref for the tree. */ refcount_inc(&entry->refs); spin_lock(&inode->ordered_tree_lock); node = tree_insert(&inode->ordered_tree, entry->file_offset, &entry->rb_node); if (unlikely(node)) { struct btrfs_ordered_extent *exist = rb_entry(node, struct btrfs_ordered_extent, rb_node); btrfs_panic(fs_info, -EEXIST, "overlapping ordered extents, existing oe file_offset %llu num_bytes %llu flags 0x%lx, new oe file_offset %llu num_bytes %llu flags 0x%lx", exist->file_offset, exist->num_bytes, exist->flags, entry->file_offset, entry->num_bytes, entry->flags); } spin_unlock(&inode->ordered_tree_lock); spin_lock(&root->ordered_extent_lock); list_add_tail(&entry->root_extent_list, &root->ordered_extents); root->nr_ordered_extents++; if (root->nr_ordered_extents == 1) { spin_lock(&fs_info->ordered_root_lock); BUG_ON(!list_empty(&root->ordered_root)); list_add_tail(&root->ordered_root, &fs_info->ordered_roots); spin_unlock(&fs_info->ordered_root_lock); } spin_unlock(&root->ordered_extent_lock); } /* * Add an ordered extent to the per-inode tree. * * @inode: Inode that this extent is for. * @file_offset: Logical offset in file where the extent starts. * @num_bytes: Logical length of extent in file. * @ram_bytes: Full length of unencoded data. * @disk_bytenr: Offset of extent on disk. * @disk_num_bytes: Size of extent on disk. * @offset: Offset into unencoded data where file data starts. * @flags: Flags specifying type of extent (1U << BTRFS_ORDERED_*). * @compress_type: Compression algorithm used for data. * * Most of these parameters correspond to &struct btrfs_file_extent_item. The * tree is given a single reference on the ordered extent that was inserted, and * the returned pointer is given a second reference. * * Return: the new ordered extent or error pointer. */ struct btrfs_ordered_extent *btrfs_alloc_ordered_extent( struct btrfs_inode *inode, u64 file_offset, const struct btrfs_file_extent *file_extent, unsigned long flags) { struct btrfs_ordered_extent *entry; ASSERT((flags & ~BTRFS_ORDERED_TYPE_FLAGS) == 0, "flags=0x%lx", flags); /* * For regular writes, we just use the members in @file_extent. * * For NOCOW, we don't really care about the numbers except @start and * file_extent->num_bytes, as we won't insert a file extent item at all. * * For PREALLOC, we do not use ordered extent members, but * btrfs_mark_extent_written() handles everything. * * So here we always pass 0 as offset for NOCOW/PREALLOC ordered extents, * or btrfs_split_ordered_extent() cannot handle it correctly. */ if (flags & ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))) entry = alloc_ordered_extent(inode, file_offset, file_extent->num_bytes, file_extent->num_bytes, file_extent->disk_bytenr + file_extent->offset, file_extent->num_bytes, 0, flags, file_extent->compression); else entry = alloc_ordered_extent(inode, file_offset, file_extent->num_bytes, file_extent->ram_bytes, file_extent->disk_bytenr, file_extent->disk_num_bytes, file_extent->offset, flags, file_extent->compression); if (!IS_ERR(entry)) insert_ordered_extent(entry); return entry; } /* * Add a struct btrfs_ordered_sum into the list of checksums to be inserted * when an ordered extent is finished. If the list covers more than one * ordered extent, it is split across multiples. */ void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry, struct btrfs_ordered_sum *sum) { struct btrfs_inode *inode = entry->inode; spin_lock(&inode->ordered_tree_lock); list_add_tail(&sum->list, &entry->csum_list); spin_unlock(&inode->ordered_tree_lock); } void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered) { if (!test_and_set_bit(BTRFS_ORDERED_IOERR, &ordered->flags)) mapping_set_error(ordered->inode->vfs_inode.i_mapping, -EIO); } void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, u64 truncate_len) { struct btrfs_inode *inode = ordered->inode; ASSERT(truncate_len <= ordered->num_bytes); spin_lock(&inode->ordered_tree_lock); set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); ordered->truncated_len = min(ordered->truncated_len, truncate_len); spin_unlock(&inode->ordered_tree_lock); } static void finish_ordered_fn(struct btrfs_work *work) { struct btrfs_ordered_extent *ordered_extent; ordered_extent = container_of(work, struct btrfs_ordered_extent, work); btrfs_finish_ordered_io(ordered_extent); } static bool can_finish_ordered_extent(struct btrfs_ordered_extent *ordered, u64 file_offset, u64 len, bool uptodate) { struct btrfs_inode *inode = ordered->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; lockdep_assert_held(&inode->ordered_tree_lock); /* Now we're fine to update the accounting. */ if (WARN_ON_ONCE(len > ordered->bytes_left)) { btrfs_crit(fs_info, "bad ordered extent accounting, root=%llu ino=%llu OE offset=%llu OE len=%llu to_dec=%llu left=%llu", btrfs_root_id(inode->root), btrfs_ino(inode), ordered->file_offset, ordered->num_bytes, len, ordered->bytes_left); ordered->bytes_left = 0; } else { ordered->bytes_left -= len; } if (!uptodate) btrfs_mark_ordered_extent_error(ordered); if (ordered->bytes_left) return false; /* * All the IO of the ordered extent is finished, we need to queue * the finish_func to be executed. */ set_bit(BTRFS_ORDERED_IO_DONE, &ordered->flags); cond_wake_up(&ordered->wait); refcount_inc(&ordered->refs); trace_btrfs_ordered_extent_mark_finished(inode, ordered); return true; } static void btrfs_queue_ordered_fn(struct btrfs_ordered_extent *ordered) { struct btrfs_inode *inode = ordered->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_workqueue *wq = btrfs_is_free_space_inode(inode) ? fs_info->endio_freespace_worker : fs_info->endio_write_workers; btrfs_init_work(&ordered->work, finish_ordered_fn, NULL); btrfs_queue_work(wq, &ordered->work); } void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered, u64 file_offset, u64 len, bool uptodate) { struct btrfs_inode *inode = ordered->inode; bool ret; trace_btrfs_finish_ordered_extent(inode, file_offset, len, uptodate); spin_lock(&inode->ordered_tree_lock); ret = can_finish_ordered_extent(ordered, file_offset, len, uptodate); spin_unlock(&inode->ordered_tree_lock); /* * If this is a COW write it means we created new extent maps for the * range and they point to unwritten locations if we got an error either * before submitting a bio or during IO. * * We have marked the ordered extent with BTRFS_ORDERED_IOERR, and we * are queuing its completion below. During completion, at * btrfs_finish_one_ordered(), we will drop the extent maps for the * unwritten extents. * * However because completion runs in a work queue we can end up having * a fast fsync running before that. In the case of direct IO, once we * unlock the inode the fsync might start, and we queue the completion * before unlocking the inode. In the case of buffered IO when writeback * finishes (end_bbio_data_write()) we queue the completion, so if the * writeback was triggered by a fast fsync, the fsync might start * logging before ordered extent completion runs in the work queue. * * The fast fsync will log file extent items based on the extent maps it * finds, so if by the time it collects extent maps the ordered extent * completion didn't happen yet, it will log file extent items that * point to unwritten extents, resulting in a corruption if a crash * happens and the log tree is replayed. Note that a fast fsync does not * wait for completion of ordered extents in order to reduce latency. * * Set a flag in the inode so that the next fast fsync will wait for * ordered extents to complete before starting to log. */ if (!uptodate && !test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) set_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags); if (ret) btrfs_queue_ordered_fn(ordered); } /* * Mark all ordered extents io inside the specified range finished. * * @folio: The involved folio for the operation. * For uncompressed buffered IO, the folio status also needs to be * updated to indicate whether the pending ordered io is finished. * Can be NULL for direct IO and compressed write. * For these cases, callers are ensured they won't execute the * endio function twice. * * This function is called for endio, thus the range must have ordered * extent(s) covering it. */ void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode, u64 file_offset, u64 num_bytes, bool uptodate) { struct rb_node *node; struct btrfs_ordered_extent *entry = NULL; u64 cur = file_offset; const u64 end = file_offset + num_bytes; trace_btrfs_writepage_end_io_hook(inode, file_offset, end - 1, uptodate); spin_lock(&inode->ordered_tree_lock);