// SPDX-License-Identifier: GPL-2.0 /* * fs/ext4/fast_commit.c * * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com> * * Ext4 fast commits routines. */#include "ext4.h"#include "ext4_jbd2.h"#include "ext4_extents.h"#include "mballoc.h" /* * Ext4 Fast Commits * ----------------- * * Ext4 fast commits implement fine grained journalling for Ext4. * * Fast commits are organized as a log of tag-length-value (TLV) structs. (See * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by * TLV during the recovery phase. For the scenarios for which we currently * don't have replay code, fast commit falls back to full commits. * Fast commits record delta in one of the following three categories. * * (A) Directory entry updates: * * - EXT4_FC_TAG_UNLINK		- records directory entry unlink * - EXT4_FC_TAG_LINK		- records directory entry link * - EXT4_FC_TAG_CREAT		- records inode and directory entry creation * * (B) File specific data range updates: * * - EXT4_FC_TAG_ADD_RANGE	- records addition of new blocks to an inode * - EXT4_FC_TAG_DEL_RANGE	- records deletion of blocks from an inode * * (C) Inode metadata (mtime / ctime etc): * * - EXT4_FC_TAG_INODE		- record the inode that should be replayed *				  during recovery. Note that iblocks field is *				  not replayed and instead derived during *				  replay. * Commit Operation * ---------------- * With fast commits, we maintain all the directory entry operations in the * order in which they are issued in an in-memory queue. This queue is flushed * to disk during the commit operation. We also maintain a list of inodes * that need to be committed during a fast commit in another in memory queue of * inodes. During the commit operation, we commit in the following order: * * [1] Lock inodes for any further data updates by setting COMMITTING state * [2] Submit data buffers of all the inodes * [3] Wait for [2] to complete * [4] Commit all the directory entry updates in the fast commit space * [5] Commit all the changed inode structures * [6] Write tail tag (this tag ensures the atomicity, please read the following *     section for more details). * [7] Wait for [4], [5] and [6] to complete. * * All the inode updates must call ext4_fc_start_update() before starting an * update. If such an ongoing update is present, fast commit waits for it to * complete. The completion of such an update is marked by * ext4_fc_stop_update(). * * Fast Commit Ineligibility * ------------------------- * * Not all operations are supported by fast commits today (e.g extended * attributes). Fast commit ineligibility is marked by calling * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back * to full commit. * * Atomicity of commits * -------------------- * In order to guarantee atomicity during the commit operation, fast commit * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail * tag contains CRC of the contents and TID of the transaction after which * this fast commit should be applied. Recovery code replays fast commit * logs only if there's at least 1 valid tail present. For every fast commit * operation, there is 1 tail. This means, we may end up with multiple tails * in the fast commit space. Here's an example: * * - Create a new file A and remove existing file B * - fsync() * - Append contents to file A * - Truncate file A * - fsync() * * The fast commit space at the end of above operations would look like this: *      [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL] *             |<---  Fast Commit 1   --->|<---      Fast Commit 2     ---->| * * Replay code should thus check for all the valid tails in the FC area. * * Fast Commit Replay Idempotence * ------------------------------ * * Fast commits tags are idempotent in nature provided the recovery code follows * certain rules. The guiding principle that the commit path follows while * committing is that it stores the result of a particular operation instead of * storing the procedure. * * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a' * was associated with inode 10. During fast commit, instead of storing this * operation as a procedure "rename a to b", we store the resulting file system * state as a "series" of outcomes: * * - Link dirent b to inode 10 * - Unlink dirent a * - Inode <10> with valid refcount * * Now when recovery code runs, it needs "enforce" this state on the file * system. This is what guarantees idempotence of fast commit replay. * * Let's take an example of a procedure that is not idempotent and see how fast * commits make it idempotent. Consider following sequence of operations: * *     rm A;    mv B A;    read A *  (x)     (y)        (z) * * (x), (y) and (z) are the points at which we can crash. If we store this * sequence of operations as is then the replay is not idempotent. Let's say * while in replay, we crash at (z). During the second replay, file A (which was * actually created as a result of "mv B A" operation) would get deleted. Thus, * file named A would be absent when we try to read A. So, this sequence of * operations is not idempotent. However, as mentioned above, instead of storing * the procedure fast commits store the outcome of each procedure. Thus the fast * commit log for above procedure would be as follows: * * (Let's assume dirent A was linked to inode 10 and dirent B was linked to * inode 11 before the replay) * *    [Unlink A]   [Link A to inode 11]   [Unlink B]   [Inode 11] * (w)          (x)                    (y)          (z) * * If we crash at (z), we will have file A linked to inode 11. During the second * replay, we will remove file A (inode 11). But we will create it back and make * it point to inode 11. We won't find B, so we'll just skip that step. At this * point, the refcount for inode 11 is not reliable, but that gets fixed by the * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled * similarly. Thus, by converting a non-idempotent procedure into a series of * idempotent outcomes, fast commits ensured idempotence during the replay. * * TODOs * ----- * * 0) Fast commit replay path hardening: Fast commit replay code should use *    journal handles to make sure all the updates it does during the replay *    path are atomic. With that if we crash during fast commit replay, after *    trying to do recovery again, we will find a file system where fast commit *    area is invalid (because new full commit would be found). In order to deal *    with that, fast commit replay code should ensure that the "FC_REPLAY" *    superblock state is persisted before starting the replay, so that after *    the crash, fast commit recovery code can look at that flag and perform *    fast commit recovery even if that area is invalidated by later full *    commits. * * 1) Fast commit's commit path locks the entire file system during fast *    commit. This has significant performance penalty. Instead of that, we *    should use ext4_fc_start/stop_update functions to start inode level *    updates from ext4_journal_start/stop. Once we do that we can drop file *    system locking during commit path. * * 2) Handle more ineligible cases. */ #include <trace/events/ext4.h>static struct kmem_cache *ext4_fc_dentry_cachep; static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate){	BUFFER_TRACE(bh, "");	if (uptodate) {		ext4_debug("%s: Block %lld up-to-date",			   __func__, bh->b_blocknr);		set_buffer_uptodate(bh);	} else {		ext4_debug("%s: Block %lld not up-to-date",			   __func__, bh->b_blocknr);		clear_buffer_uptodate(bh);	} 	unlock_buffer(bh);} static inline void ext4_fc_reset_inode(struct inode *inode){	struct ext4_inode_info *ei = EXT4_I(inode); 	ei->i_fc_lblk_start = 0;	ei->i_fc_lblk_len = 0;} void ext4_fc_init_inode(struct inode *inode){	struct ext4_inode_info *ei = EXT4_I(inode); 	ext4_fc_reset_inode(inode);	ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);	INIT_LIST_HEAD(&ei->i_fc_list);	INIT_LIST_HEAD(&ei->i_fc_dilist);	init_waitqueue_head(&ei->i_fc_wait);	atomic_set(&ei->i_fc_updates, 0);} /* This function must be called with sbi->s_fc_lock held. */static void ext4_fc_wait_committing_inode(struct inode *inode)__releases(&EXT4_SB(inode->i_sb)->s_fc_lock){	wait_queue_head_t *wq;	struct ext4_inode_info *ei = EXT4_I(inode); #if (BITS_PER_LONG < 64)	DEFINE_WAIT_BIT(wait, &ei->i_state_flags,			EXT4_STATE_FC_COMMITTING);	wq = bit_waitqueue(&ei->i_state_flags,				EXT4_STATE_FC_COMMITTING);#else	DEFINE_WAIT_BIT(wait, &ei->i_flags,			EXT4_STATE_FC_COMMITTING);	wq = bit_waitqueue(&ei->i_flags,				EXT4_STATE_FC_COMMITTING);#endif	lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock);	prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);	spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);	schedule();	finish_wait(wq, &wait.wq_entry);} static bool ext4_fc_disabled(struct super_block *sb){	return (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||		(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY));} /* * Inform Ext4's fast about start of an inode update * * This function is called by the high level call VFS callbacks before * performing any inode update. This function blocks if there's an ongoing * fast commit on the inode in question. */void ext4_fc_start_update(struct inode *inode){	struct ext4_inode_info *ei = EXT4_I(inode); 	if (ext4_fc_disabled(inode->i_sb))		return; restart:	spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);	if (list_empty(&ei->i_fc_list))		goto out; 	if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {		ext4_fc_wait_committing_inode(inode);		goto restart;	}out:	atomic_inc(&ei->i_fc_updates);	spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);} /* * Stop inode update and wake up waiting fast commits if any. */void ext4_fc_stop_update(struct inode *inode){	struct ext4_inode_info *ei = EXT4_I(inode); 	if (ext4_fc_disabled(inode->i_sb))		return; 	if (atomic_dec_and_test(&ei->i_fc_updates))		wake_up_all(&ei->i_fc_wait);} /* * Remove inode from fast commit list. If the inode is being committed * we wait until inode commit is done. */void ext4_fc_del(struct inode *inode){	struct ext4_inode_info *ei = EXT4_I(inode);	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);	struct ext4_fc_dentry_update *fc_dentry; 	if (ext4_fc_disabled(inode->i_sb))		return; restart:	spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);	if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) {		spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);		return;	} 	if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {		ext4_fc_wait_committing_inode(inode);		goto restart;	} 	if (!list_empty(&ei->i_fc_list))		list_del_init(&ei->i_fc_list); 	/*	 * Since this inode is getting removed, let's also remove all FC	 * dentry create references, since it is not needed to log it anyways.	 */	if (list_empty(&ei->i_fc_dilist)) {		spin_unlock(&sbi->s_fc_lock);		return;	} 	fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist);	WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT);	list_del_init(&fc_dentry->fcd_list);	list_del_init(&fc_dentry->fcd_dilist); 	WARN_ON(!list_empty(&ei->i_fc_dilist));	spin_unlock(&sbi->s_fc_lock); 	if (fc_dentry->fcd_name.name &&		fc_dentry->fcd_name.len > DNAME_INLINE_LEN)		kfree(fc_dentry->fcd_name.name);	kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); 	return;} /* * Mark file system as fast commit ineligible, and record latest * ineligible transaction tid. This means until the recorded * transaction, commit operation would result in a full jbd2 commit. */void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle){	struct ext4_sb_info *sbi = EXT4_SB(sb);	tid_t tid; 	if (ext4_fc_disabled(sb))		return; 	ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);	if (handle && !IS_ERR(handle))		tid = handle->h_transaction->t_tid;	else {		read_lock(&sbi->s_journal->j_state_lock);		tid = sbi->s_journal->j_running_transaction ?				sbi->s_journal->j_running_transaction->t_tid : 0;		read_unlock(&sbi->s_journal->j_state_lock);	}	spin_lock(&sbi->s_fc_lock);	if (tid_gt(tid, sbi->s_fc_ineligible_tid))		sbi->s_fc_ineligible_tid = tid;	spin_unlock(&sbi->s_fc_lock);	WARN_ON(reason >= EXT4_FC_REASON_MAX);	sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;} /* * Generic fast commit tracking function. If this is the first time this we are * called after a full commit, we initialize fast commit fields and then call * __fc_track_fn() with update = 0. If we have already been called after a full * commit, we pass update = 1. Based on that, the track function can determine * if it needs to track a field for the first time or if it needs to just * update the previously tracked value. * * If enqueue is set, this function enqueues the inode in fast commit list. */static int ext4_fc_track_template(	handle_t *handle, struct inode *inode,	int (*__fc_track_fn)(struct inode *, void *, bool),	void *args, int enqueue){	bool update = false;	struct ext4_inode_info *ei = EXT4_I(inode);	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);	tid_t tid = 0;	int ret; 	tid = handle->h_transaction->t_tid;	mutex_lock(&ei->i_fc_lock);	if (tid == ei->i_sync_tid) {		update = true;	} else {		ext4_fc_reset_inode(inode);		ei->i_sync_tid = tid;	}	ret = __fc_track_fn(inode, args, update);	mutex_unlock(&ei->i_fc_lock); 	if (!enqueue)		return ret; 	spin_lock(&sbi->s_fc_lock);	if (list_empty(&EXT4_I(inode)->i_fc_list))		list_add_tail(&EXT4_I(inode)->i_fc_list,				(sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||				 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?				&sbi->s_fc_q[FC_Q_STAGING] :				&sbi->s_fc_q[FC_Q_MAIN]);	spin_unlock(&sbi->s_fc_lock); 	return ret;} struct __track_dentry_update_args {	struct dentry *dentry;	int op;}; /* __track_fn for directory entry updates. Called with ei->i_fc_lock. */static int __track_dentry_update(struct inode *inode, void *arg, bool update){	struct ext4_fc_dentry_update *node;	struct ext4_inode_info *ei = EXT4_I(inode);	struct __track_dentry_update_args *dentry_update =		(struct __track_dentry_update_args *)arg;	struct dentry *dentry = dentry_update->dentry;	struct inode *dir = dentry->d_parent->d_inode;	struct super_block *sb = inode->i_sb;	struct ext4_sb_info *sbi = EXT4_SB(sb); 	mutex_unlock(&ei->i_fc_lock); 	if (IS_ENCRYPTED(dir)) {		ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME,					NULL);		mutex_lock(&ei->i_fc_lock);		return -EOPNOTSUPP;	} 	node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);	if (!node) {		ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL);		mutex_lock(&ei->i_fc_lock);		return -ENOMEM;	} 	node->fcd_op = dentry_update->op;	node->fcd_parent = dir->i_ino;	node->fcd_ino = inode->i_ino;	if (dentry->d_name.len > DNAME_INLINE_LEN) {		node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS);		if (!node->fcd_name.name) {			kmem_cache_free(ext4_fc_dentry_cachep, node);			ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL);			mutex_lock(&ei->i_fc_lock);			return -ENOMEM;		}		memcpy((u8 *)node->fcd_name.name, dentry->d_name.name,			dentry->d_name.len);	} else {		memcpy(node->fcd_iname, dentry->d_name.name,			dentry->d_name.len);		node->fcd_name.name = node->fcd_iname;	}	node->fcd_name.len = dentry->d_name.len;	INIT_LIST_HEAD(&node->fcd_dilist);	spin_lock(&sbi->s_fc_lock);	if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||		sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)		list_add_tail(&node->fcd_list,				&sbi->s_fc_dentry_q[FC_Q_STAGING]);	else		list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]); 	/*	 * This helps us keep a track of all fc_dentry updates which is part of	 * this ext4 inode. So in case the inode is getting unlinked, before	 * even we get a chance to fsync, we could remove all fc_dentry	 * references while evicting the inode in ext4_fc_del().	 * Also with this, we don't need to loop over all the inodes in	 * sbi->s_fc_q to get the corresponding inode in	 * ext4_fc_commit_dentry_updates().	 */	if (dentry_update->op == EXT4_FC_TAG_CREAT) {		WARN_ON(!list_empty(&ei->i_fc_dilist));		list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist);	}	spin_unlock(&sbi->s_fc_lock);	mutex_lock(&ei->i_fc_lock); 	return 0;} void __ext4_fc_track_unlink(handle_t *handle,		struct inode *inode, struct dentry *dentry){	struct __track_dentry_update_args args;	int ret; 	args.dentry = dentry;	args.op = EXT4_FC_TAG_UNLINK; 	ret = ext4_fc_track_template(handle, inode, __track_dentry_update,					(void *)&args, 0);

Lines 1–500 of 2310. Load more