// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2007 Oracle.  All rights reserved. */ #include <linux/slab.h>#include <linux/blkdev.h>#include <linux/writeback.h>#include <linux/sched/mm.h>#include "messages.h"#include "misc.h"#include "ctree.h"#include "transaction.h"#include "btrfs_inode.h"#include "extent_io.h"#include "disk-io.h"#include "compression.h"#include "delalloc-space.h"#include "qgroup.h"#include "subpage.h"#include "file.h"#include "super.h" static struct kmem_cache *btrfs_ordered_extent_cache; static u64 entry_end(struct btrfs_ordered_extent *entry){	if (entry->file_offset + entry->num_bytes < entry->file_offset)		return (u64)-1;	return entry->file_offset + entry->num_bytes;} /* returns NULL if the insertion worked, or it returns the node it did find * in the tree */static struct rb_node *tree_insert(struct rb_root *root, u64 file_offset,				   struct rb_node *node){	struct rb_node **p = &root->rb_node;	struct rb_node *parent = NULL;	struct btrfs_ordered_extent *entry; 	while (*p) {		parent = *p;		entry = rb_entry(parent, struct btrfs_ordered_extent, rb_node); 		if (file_offset < entry->file_offset)			p = &(*p)->rb_left;		else if (file_offset >= entry_end(entry))			p = &(*p)->rb_right;		else			return parent;	} 	rb_link_node(node, parent, p);	rb_insert_color(node, root);	return NULL;} /* * look for a given offset in the tree, and if it can't be found return the * first lesser offset */static struct rb_node *__tree_search(struct rb_root *root, u64 file_offset,				     struct rb_node **prev_ret){	struct rb_node *n = root->rb_node;	struct rb_node *prev = NULL;	struct rb_node *test;	struct btrfs_ordered_extent *entry;	struct btrfs_ordered_extent *prev_entry = NULL; 	while (n) {		entry = rb_entry(n, struct btrfs_ordered_extent, rb_node);		prev = n;		prev_entry = entry; 		if (file_offset < entry->file_offset)			n = n->rb_left;		else if (file_offset >= entry_end(entry))			n = n->rb_right;		else			return n;	}	if (!prev_ret)		return NULL; 	while (prev && file_offset >= entry_end(prev_entry)) {		test = rb_next(prev);		if (!test)			break;		prev_entry = rb_entry(test, struct btrfs_ordered_extent,				      rb_node);		if (file_offset < entry_end(prev_entry))			break; 		prev = test;	}	if (prev)		prev_entry = rb_entry(prev, struct btrfs_ordered_extent,				      rb_node);	while (prev && file_offset < entry_end(prev_entry)) {		test = rb_prev(prev);		if (!test)			break;		prev_entry = rb_entry(test, struct btrfs_ordered_extent,				      rb_node);		prev = test;	}	*prev_ret = prev;	return NULL;} static int range_overlaps(struct btrfs_ordered_extent *entry, u64 file_offset,			  u64 len){	if (file_offset + len <= entry->file_offset ||	    entry->file_offset + entry->num_bytes <= file_offset)		return 0;	return 1;} /* * look find the first ordered struct that has this offset, otherwise * the first one less than this offset */static inline struct rb_node *tree_search(struct btrfs_ordered_inode_tree *tree,					  u64 file_offset){	struct rb_root *root = &tree->tree;	struct rb_node *prev = NULL;	struct rb_node *ret;	struct btrfs_ordered_extent *entry; 	if (tree->last) {		entry = rb_entry(tree->last, struct btrfs_ordered_extent,				 rb_node);		if (in_range(file_offset, entry->file_offset, entry->num_bytes))			return tree->last;	}	ret = __tree_search(root, file_offset, &prev);	if (!ret)		ret = prev;	if (ret)		tree->last = ret;	return ret;} /* * Add an ordered extent to the per-inode tree. * * @inode:           Inode that this extent is for. * @file_offset:     Logical offset in file where the extent starts. * @num_bytes:       Logical length of extent in file. * @ram_bytes:       Full length of unencoded data. * @disk_bytenr:     Offset of extent on disk. * @disk_num_bytes:  Size of extent on disk. * @offset:          Offset into unencoded data where file data starts. * @flags:           Flags specifying type of extent (1 << BTRFS_ORDERED_*). * @compress_type:   Compression algorithm used for data. * * Most of these parameters correspond to &struct btrfs_file_extent_item. The * tree is given a single reference on the ordered extent that was inserted. * * Return: 0 or -ENOMEM. */int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset,			     u64 num_bytes, u64 ram_bytes, u64 disk_bytenr,			     u64 disk_num_bytes, u64 offset, unsigned flags,			     int compress_type){	struct btrfs_root *root = inode->root;	struct btrfs_fs_info *fs_info = root->fs_info;	struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree;	struct rb_node *node;	struct btrfs_ordered_extent *entry;	int ret; 	if (flags &	    ((1 << BTRFS_ORDERED_NOCOW) | (1 << BTRFS_ORDERED_PREALLOC))) {		/* For nocow write, we can release the qgroup rsv right now */		ret = btrfs_qgroup_free_data(inode, NULL, file_offset, num_bytes);		if (ret < 0)			return ret;		ret = 0;	} else {		/*		 * The ordered extent has reserved qgroup space, release now		 * and pass the reserved number for qgroup_record to free.		 */		ret = btrfs_qgroup_release_data(inode, file_offset, num_bytes);		if (ret < 0)			return ret;	}	entry = kmem_cache_zalloc(btrfs_ordered_extent_cache, GFP_NOFS);	if (!entry)		return -ENOMEM; 	entry->file_offset = file_offset;	entry->num_bytes = num_bytes;	entry->ram_bytes = ram_bytes;	entry->disk_bytenr = disk_bytenr;	entry->disk_num_bytes = disk_num_bytes;	entry->offset = offset;	entry->bytes_left = num_bytes;	entry->inode = igrab(&inode->vfs_inode);	entry->compress_type = compress_type;	entry->truncated_len = (u64)-1;	entry->qgroup_rsv = ret;	entry->physical = (u64)-1; 	ASSERT((flags & ~BTRFS_ORDERED_TYPE_FLAGS) == 0);	entry->flags = flags; 	percpu_counter_add_batch(&fs_info->ordered_bytes, num_bytes,				 fs_info->delalloc_batch); 	/* one ref for the tree */	refcount_set(&entry->refs, 1);	init_waitqueue_head(&entry->wait);	INIT_LIST_HEAD(&entry->list);	INIT_LIST_HEAD(&entry->log_list);	INIT_LIST_HEAD(&entry->root_extent_list);	INIT_LIST_HEAD(&entry->work_list);	init_completion(&entry->completion); 	trace_btrfs_ordered_extent_add(inode, entry); 	spin_lock_irq(&tree->lock);	node = tree_insert(&tree->tree, file_offset,			   &entry->rb_node);	if (node)		btrfs_panic(fs_info, -EEXIST,				"inconsistency in ordered tree at offset %llu",				file_offset);	spin_unlock_irq(&tree->lock); 	spin_lock(&root->ordered_extent_lock);	list_add_tail(&entry->root_extent_list,		      &root->ordered_extents);	root->nr_ordered_extents++;	if (root->nr_ordered_extents == 1) {		spin_lock(&fs_info->ordered_root_lock);		BUG_ON(!list_empty(&root->ordered_root));		list_add_tail(&root->ordered_root, &fs_info->ordered_roots);		spin_unlock(&fs_info->ordered_root_lock);	}	spin_unlock(&root->ordered_extent_lock); 	/*	 * We don't need the count_max_extents here, we can assume that all of	 * that work has been done at higher layers, so this is truly the	 * smallest the extent is going to get.	 */	spin_lock(&inode->lock);	btrfs_mod_outstanding_extents(inode, 1);	spin_unlock(&inode->lock); 	return 0;} /* * Add a struct btrfs_ordered_sum into the list of checksums to be inserted * when an ordered extent is finished.  If the list covers more than one * ordered extent, it is split across multiples. */void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry,			   struct btrfs_ordered_sum *sum){	struct btrfs_ordered_inode_tree *tree; 	tree = &BTRFS_I(entry->inode)->ordered_tree;	spin_lock_irq(&tree->lock);	list_add_tail(&sum->list, &entry->list);	spin_unlock_irq(&tree->lock);} static void finish_ordered_fn(struct btrfs_work *work){	struct btrfs_ordered_extent *ordered_extent; 	ordered_extent = container_of(work, struct btrfs_ordered_extent, work);	btrfs_finish_ordered_io(ordered_extent);} /* * Mark all ordered extents io inside the specified range finished. * * @page:	 The involved page for the operation. *		 For uncompressed buffered IO, the page status also needs to be *		 updated to indicate whether the pending ordered io is finished. *		 Can be NULL for direct IO and compressed write. *		 For these cases, callers are ensured they won't execute the *		 endio function twice. * * This function is called for endio, thus the range must have ordered * extent(s) covering it. */void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode,				    struct page *page, u64 file_offset,				    u64 num_bytes, bool uptodate){	struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree;	struct btrfs_fs_info *fs_info = inode->root->fs_info;	struct btrfs_workqueue *wq;	struct rb_node *node;	struct btrfs_ordered_extent *entry = NULL;	unsigned long flags;	u64 cur = file_offset; 	if (btrfs_is_free_space_inode(inode))		wq = fs_info->endio_freespace_worker;	else		wq = fs_info->endio_write_workers; 	if (page)		ASSERT(page->mapping && page_offset(page) <= file_offset &&		       file_offset + num_bytes <= page_offset(page) + PAGE_SIZE); 	spin_lock_irqsave(&tree->lock, flags);	while (cur < file_offset + num_bytes) {		u64 entry_end;		u64 end;		u32 len; 		node = tree_search(tree, cur);		/* No ordered extents at all */		if (!node)			break; 		entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);		entry_end = entry->file_offset + entry->num_bytes;		/*		 * |<-- OE --->|  |		 *		  cur		 * Go to next OE.		 */		if (cur >= entry_end) {			node = rb_next(node);			/* No more ordered extents, exit */			if (!node)				break;			entry = rb_entry(node, struct btrfs_ordered_extent,					 rb_node); 			/* Go to next ordered extent and continue */			cur = entry->file_offset;			continue;		}		/*		 * |	|<--- OE --->|		 * cur		 * Go to the start of OE.		 */		if (cur < entry->file_offset) {			cur = entry->file_offset;			continue;		} 		/*		 * Now we are definitely inside one ordered extent.		 *		 * |<--- OE --->|		 *	|		 *	cur		 */		end = min(entry->file_offset + entry->num_bytes,			  file_offset + num_bytes) - 1;		ASSERT(end + 1 - cur < U32_MAX);		len = end + 1 - cur; 		if (page) {			/*			 * Ordered (Private2) bit indicates whether we still			 * have pending io unfinished for the ordered extent.			 *			 * If there's no such bit, we need to skip to next range.			 */			if (!btrfs_page_test_ordered(fs_info, page, cur, len)) {				cur += len;				continue;			}			btrfs_page_clear_ordered(fs_info, page, cur, len);		} 		/* Now we're fine to update the accounting */		if (unlikely(len > entry->bytes_left)) {			WARN_ON(1);			btrfs_crit(fs_info,"bad ordered extent accounting, root=%llu ino=%llu OE offset=%llu OE len=%llu to_dec=%u left=%llu",				   inode->root->root_key.objectid,				   btrfs_ino(inode),				   entry->file_offset,				   entry->num_bytes,				   len, entry->bytes_left);			entry->bytes_left = 0;		} else {			entry->bytes_left -= len;		} 		if (!uptodate)			set_bit(BTRFS_ORDERED_IOERR, &entry->flags); 		/*		 * All the IO of the ordered extent is finished, we need to queue		 * the finish_func to be executed.		 */		if (entry->bytes_left == 0) {			set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags);			cond_wake_up(&entry->wait);			refcount_inc(&entry->refs);			trace_btrfs_ordered_extent_mark_finished(inode, entry);			spin_unlock_irqrestore(&tree->lock, flags);			btrfs_init_work(&entry->work, finish_ordered_fn, NULL, NULL);			btrfs_queue_work(wq, &entry->work);			spin_lock_irqsave(&tree->lock, flags);		}		cur += len;	}	spin_unlock_irqrestore(&tree->lock, flags);} /* * Finish IO for one ordered extent across a given range.  The range can only * contain one ordered extent. * * @cached:	 The cached ordered extent. If not NULL, we can skip the tree *               search and use the ordered extent directly. * 		 Will be also used to store the finished ordered extent. * @file_offset: File offset for the finished IO * @io_size:	 Length of the finish IO range * * Return true if the ordered extent is finished in the range, and update * @cached. * Return false otherwise. * * NOTE: The range can NOT cross multiple ordered extents. * Thus caller should ensure the range doesn't cross ordered extents. */bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode,				    struct btrfs_ordered_extent **cached,				    u64 file_offset, u64 io_size){	struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree;	struct rb_node *node;	struct btrfs_ordered_extent *entry = NULL;	unsigned long flags;	bool finished = false; 	spin_lock_irqsave(&tree->lock, flags);	if (cached && *cached) {		entry = *cached;		goto have_entry;	} 	node = tree_search(tree, file_offset);	if (!node)		goto out; 	entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);have_entry:	if (!in_range(file_offset, entry->file_offset, entry->num_bytes))		goto out; 	if (io_size > entry->bytes_left)		btrfs_crit(inode->root->fs_info,			   "bad ordered accounting left %llu size %llu",		       entry->bytes_left, io_size); 	entry->bytes_left -= io_size; 	if (entry->bytes_left == 0) {		/*		 * Ensure only one caller can set the flag and finished_ret		 * accordingly		 */		finished = !test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags);		/* test_and_set_bit implies a barrier */		cond_wake_up_nomb(&entry->wait);	}out:	if (finished && cached && entry) {		*cached = entry;		refcount_inc(&entry->refs);		trace_btrfs_ordered_extent_dec_test_pending(inode, entry);	}	spin_unlock_irqrestore(&tree->lock, flags);	return finished;} /* * used to drop a reference on an ordered extent.  This will free * the extent if the last reference is dropped */void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry){	struct list_head *cur;	struct btrfs_ordered_sum *sum; 	trace_btrfs_ordered_extent_put(BTRFS_I(entry->inode), entry);

Lines 1–500 of 1174. Load more