// SPDX-License-Identifier: GPL-2.0-only/* * fs/fs-writeback.c * * Copyright (C) 2002, Linus Torvalds. * * Contains all the functions related to writing back and waiting * upon dirty inodes against superblocks, and writing back dirty * pages against inodes.  ie: data writeback.  Writeout of the * inode itself is not handled here. * * 10Apr2002	Andrew Morton *		Split out of fs/inode.c *		Additions for address_space-based writeback */ #include <linux/sched/sysctl.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/spinlock.h>#include <linux/slab.h>#include <linux/sched.h>#include <linux/fs.h>#include <linux/mm.h>#include <linux/pagemap.h>#include <linux/kthread.h>#include <linux/writeback.h>#include <linux/blkdev.h>#include <linux/backing-dev.h>#include <linux/tracepoint.h>#include <linux/device.h>#include <linux/memcontrol.h>#include "internal.h" /* * Passed into wb_writeback(), essentially a subset of writeback_control */struct wb_writeback_work {	long nr_pages;	struct super_block *sb;	enum writeback_sync_modes sync_mode;	unsigned int tagged_writepages:1;	unsigned int for_kupdate:1;	unsigned int range_cyclic:1;	unsigned int for_background:1;	unsigned int for_sync:1;	/* sync(2) WB_SYNC_ALL writeback */	unsigned int auto_free:1;	/* free on completion */	enum wb_reason reason;		/* why was writeback initiated? */ 	struct list_head list;		/* pending work list */	struct wb_completion *done;	/* set if the caller waits */}; /* * If an inode is constantly having its pages dirtied, but then the * updates stop dirtytime_expire_interval seconds in the past, it's * possible for the worst case time between when an inode has its * timestamps updated and when they finally get written out to be two * dirtytime_expire_intervals.  We set the default to 12 hours (in * seconds), which means most of the time inodes will have their * timestamps written to disk after 12 hours, but in the worst case a * few inodes might not their timestamps updated for 24 hours. */static unsigned int dirtytime_expire_interval = 12 * 60 * 60; static inline struct inode *wb_inode(struct list_head *head){	return list_entry(head, struct inode, i_io_list);} /* * Include the creation of the trace points after defining the * wb_writeback_work structure and inline functions so that the definition * remains local to this file. */#define CREATE_TRACE_POINTS#include <trace/events/writeback.h> EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage); static bool wb_io_lists_populated(struct bdi_writeback *wb){	if (wb_has_dirty_io(wb)) {		return false;	} else {		set_bit(WB_has_dirty_io, &wb->state);		WARN_ON_ONCE(!wb->avg_write_bandwidth);		atomic_long_add(wb->avg_write_bandwidth,				&wb->bdi->tot_write_bandwidth);		return true;	}} static void wb_io_lists_depopulated(struct bdi_writeback *wb){	if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) &&	    list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) {		clear_bit(WB_has_dirty_io, &wb->state);		WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth,					&wb->bdi->tot_write_bandwidth) < 0);	}} /** * inode_io_list_move_locked - move an inode onto a bdi_writeback IO list * @inode: inode to be moved * @wb: target bdi_writeback * @head: one of @wb->b_{dirty|io|more_io|dirty_time} * * Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io. * Returns %true if @inode is the first occupant of the !dirty_time IO * lists; otherwise, %false. */static bool inode_io_list_move_locked(struct inode *inode,				      struct bdi_writeback *wb,				      struct list_head *head){	assert_spin_locked(&wb->list_lock);	assert_spin_locked(&inode->i_lock);	WARN_ON_ONCE(inode_state_read(inode) & I_FREEING); 	list_move(&inode->i_io_list, head); 	/* dirty_time doesn't count as dirty_io until expiration */	if (head != &wb->b_dirty_time)		return wb_io_lists_populated(wb); 	wb_io_lists_depopulated(wb);	return false;} static void wb_wakeup(struct bdi_writeback *wb){	spin_lock_irq(&wb->work_lock);	if (test_bit(WB_registered, &wb->state))		mod_delayed_work(bdi_wq, &wb->dwork, 0);	spin_unlock_irq(&wb->work_lock);} /* * This function is used when the first inode for this wb is marked dirty. It * wakes-up the corresponding bdi thread which should then take care of the * periodic background write-out of dirty inodes. Since the write-out would * starts only 'dirty_writeback_interval' centisecs from now anyway, we just * set up a timer which wakes the bdi thread up later. * * Note, we wouldn't bother setting up the timer, but this function is on the * fast-path (used by '__mark_inode_dirty()'), so we save few context switches * by delaying the wake-up. * * We have to be careful not to postpone flush work if it is scheduled for * earlier. Thus we use queue_delayed_work(). */static void wb_wakeup_delayed(struct bdi_writeback *wb){	unsigned long timeout; 	timeout = msecs_to_jiffies(dirty_writeback_interval * 10);	spin_lock_irq(&wb->work_lock);	if (test_bit(WB_registered, &wb->state))		queue_delayed_work(bdi_wq, &wb->dwork, timeout);	spin_unlock_irq(&wb->work_lock);} static void finish_writeback_work(struct wb_writeback_work *work){	struct wb_completion *done = work->done; 	if (work->auto_free)		kfree(work);	if (done) {		wait_queue_head_t *waitq = done->waitq; 		/* @done can't be accessed after the following dec */		if (atomic_dec_and_test(&done->cnt))			wake_up_all(waitq);	}} static void wb_queue_work(struct bdi_writeback *wb,			  struct wb_writeback_work *work){	trace_writeback_queue(wb, work); 	if (work->done)		atomic_inc(&work->done->cnt); 	spin_lock_irq(&wb->work_lock); 	if (test_bit(WB_registered, &wb->state)) {		list_add_tail(&work->list, &wb->work_list);		mod_delayed_work(bdi_wq, &wb->dwork, 0);	} else		finish_writeback_work(work); 	spin_unlock_irq(&wb->work_lock);} static bool wb_wait_for_completion_cb(struct wb_completion *done){	unsigned long timeout = sysctl_hung_task_timeout_secs;	unsigned long waited_secs = (jiffies - done->wait_start) / HZ; 	done->progress_stamp = jiffies;	if (timeout && (waited_secs > timeout))		pr_info("INFO: The task %s:%d has been waiting for writeback "			"completion for more than %lu seconds.",			current->comm, current->pid, waited_secs); 	return !atomic_read(&done->cnt);} /** * wb_wait_for_completion - wait for completion of bdi_writeback_works * @done: target wb_completion * * Wait for one or more work items issued to @bdi with their ->done field * set to @done, which should have been initialized with * DEFINE_WB_COMPLETION().  This function returns after all such work items * are completed.  Work items which are waited upon aren't freed * automatically on completion. */void wb_wait_for_completion(struct wb_completion *done){	done->wait_start = jiffies;	atomic_dec(&done->cnt);		/* put down the initial count */	wait_event(*done->waitq, wb_wait_for_completion_cb(done));} #ifdef CONFIG_CGROUP_WRITEBACK /* * Parameters for foreign inode detection, see wbc_detach_inode() to see * how they're used. * * These paramters are inherently heuristical as the detection target * itself is fuzzy.  All we want to do is detaching an inode from the * current owner if it's being written to by some other cgroups too much. * * The current cgroup writeback is built on the assumption that multiple * cgroups writing to the same inode concurrently is very rare and a mode * of operation which isn't well supported.  As such, the goal is not * taking too long when a different cgroup takes over an inode while * avoiding too aggressive flip-flops from occasional foreign writes. * * We record, very roughly, 2s worth of IO time history and if more than * half of that is foreign, trigger the switch.  The recording is quantized * to 16 slots.  To avoid tiny writes from swinging the decision too much, * writes smaller than 1/8 of avg size are ignored. */#define WB_FRN_TIME_SHIFT	13	/* 1s = 2^13, upto 8 secs w/ 16bit */#define WB_FRN_TIME_AVG_SHIFT	3	/* avg = avg * 7/8 + new * 1/8 */#define WB_FRN_TIME_CUT_DIV	8	/* ignore rounds < avg / 8 */#define WB_FRN_TIME_PERIOD	(2 * (1 << WB_FRN_TIME_SHIFT))	/* 2s */ #define WB_FRN_HIST_SLOTS	16	/* inode->i_wb_frn_history is 16bit */#define WB_FRN_HIST_UNIT	(WB_FRN_TIME_PERIOD / WB_FRN_HIST_SLOTS)					/* each slot's duration is 2s / 16 */#define WB_FRN_HIST_THR_SLOTS	(WB_FRN_HIST_SLOTS / 2)					/* if foreign slots >= 8, switch */#define WB_FRN_HIST_MAX_SLOTS	(WB_FRN_HIST_THR_SLOTS / 2 + 1)					/* one round can affect upto 5 slots */#define WB_FRN_MAX_IN_FLIGHT	1024	/* don't queue too many concurrently */ /* * Maximum inodes per isw.  A specific value has been chosen to make * struct inode_switch_wbs_context fit into 1024 bytes kmalloc. */#define WB_MAX_INODES_PER_ISW  ((1024UL - sizeof(struct inode_switch_wbs_context)) \                                / sizeof(struct inode *)) static atomic_t isw_nr_in_flight = ATOMIC_INIT(0);static struct workqueue_struct *isw_wq; void __inode_attach_wb(struct inode *inode, struct folio *folio){	struct backing_dev_info *bdi = inode_to_bdi(inode);	struct bdi_writeback *wb = NULL; 	if (inode_cgwb_enabled(inode)) {		struct cgroup_subsys_state *memcg_css; 		/* must pin memcg_css, see wb_get_create() */		if (folio)			memcg_css = get_mem_cgroup_css_from_folio(folio);		else			memcg_css = task_get_css(current, memory_cgrp_id);		wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);		css_put(memcg_css);	} 	if (!wb)		wb = &bdi->wb; 	/*	 * There may be multiple instances of this function racing to	 * update the same inode.  Use cmpxchg() to tell the winner.	 */	if (unlikely(cmpxchg(&inode->i_wb, NULL, wb)))		wb_put(wb);} /** * inode_cgwb_move_to_attached - put the inode onto wb->b_attached list * @inode: inode of interest with i_lock held * @wb: target bdi_writeback * * Remove the inode from wb's io lists and if necessarily put onto b_attached * list.  Only inodes attached to cgwb's are kept on this list. */static void inode_cgwb_move_to_attached(struct inode *inode,					struct bdi_writeback *wb){	assert_spin_locked(&wb->list_lock);	assert_spin_locked(&inode->i_lock);	WARN_ON_ONCE(inode_state_read(inode) & I_FREEING); 	inode_state_clear(inode, I_SYNC_QUEUED);	if (wb != &wb->bdi->wb)		list_move(&inode->i_io_list, &wb->b_attached);	else		list_del_init(&inode->i_io_list);	wb_io_lists_depopulated(wb);} /** * locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it * @inode: inode of interest with i_lock held * * Returns @inode's wb with its list_lock held.  @inode->i_lock must be * held on entry and is released on return.  The returned wb is guaranteed * to stay @inode's associated wb until its list_lock is released. */static struct bdi_writeback *locked_inode_to_wb_and_lock_list(struct inode *inode)	__releases(&inode->i_lock)	__acquires(&wb->list_lock){	while (true) {		struct bdi_writeback *wb = inode_to_wb(inode); 		/*		 * inode_to_wb() association is protected by both		 * @inode->i_lock and @wb->list_lock but list_lock nests		 * outside i_lock.  Drop i_lock and verify that the		 * association hasn't changed after acquiring list_lock.		 */		wb_get(wb);		spin_unlock(&inode->i_lock);		spin_lock(&wb->list_lock); 		/* i_wb may have changed inbetween, can't use inode_to_wb() */		if (likely(wb == inode->i_wb)) {			wb_put(wb);	/* @inode already has ref */			return wb;		} 		spin_unlock(&wb->list_lock);		wb_put(wb);		cpu_relax();		spin_lock(&inode->i_lock);	}} /** * inode_to_wb_and_lock_list - determine an inode's wb and lock it * @inode: inode of interest * * Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held * on entry. */static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)	__acquires(&wb->list_lock){	spin_lock(&inode->i_lock);	return locked_inode_to_wb_and_lock_list(inode);} struct inode_switch_wbs_context {	/* List of queued switching contexts for the wb */	struct llist_node	list; 	/*	 * Multiple inodes can be switched at once.  The switching procedure	 * consists of two parts, separated by a RCU grace period.  To make	 * sure that the second part is executed for each inode gone through	 * the first part, all inode pointers are placed into a NULL-terminated	 * array embedded into struct inode_switch_wbs_context.  Otherwise	 * an inode could be left in a non-consistent state.	 */	struct inode		*inodes[];}; static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi){	down_write(&bdi->wb_switch_rwsem);} static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi){	up_write(&bdi->wb_switch_rwsem);} static bool inode_do_switch_wbs(struct inode *inode,				struct bdi_writeback *old_wb,				struct bdi_writeback *new_wb){	struct address_space *mapping = inode->i_mapping;	XA_STATE(xas, &mapping->i_pages, 0);	struct folio *folio;	bool switched = false; 	spin_lock(&inode->i_lock);	xa_lock_irq(&mapping->i_pages); 	/*	 * Once I_FREEING or I_WILL_FREE are visible under i_lock, the eviction	 * path owns the inode and we shouldn't modify ->i_io_list.	 */	if (unlikely(inode_state_read(inode) & (I_FREEING | I_WILL_FREE)))		goto skip_switch; 	trace_inode_switch_wbs(inode, old_wb, new_wb); 	/*	 * Count and transfer stats.  Note that PAGECACHE_TAG_DIRTY points	 * to possibly dirty folios while PAGECACHE_TAG_WRITEBACK points to	 * folios actually under writeback.	 */	xas_for_each_marked(&xas, folio, ULONG_MAX, PAGECACHE_TAG_DIRTY) {		if (folio_test_dirty(folio)) {			long nr = folio_nr_pages(folio);			wb_stat_mod(old_wb, WB_RECLAIMABLE, -nr);			wb_stat_mod(new_wb, WB_RECLAIMABLE, nr);			if (folio_test_dropbehind(folio)) {				wb_stat_mod(old_wb, WB_DONTCACHE_DIRTY, -nr);				wb_stat_mod(new_wb, WB_DONTCACHE_DIRTY, nr);			}		}	} 	xas_set(&xas, 0);	xas_for_each_marked(&xas, folio, ULONG_MAX, PAGECACHE_TAG_WRITEBACK) {		long nr = folio_nr_pages(folio);		WARN_ON_ONCE(!folio_test_writeback(folio));		wb_stat_mod(old_wb, WB_WRITEBACK, -nr);		wb_stat_mod(new_wb, WB_WRITEBACK, nr);	} 	if (mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) {		atomic_dec(&old_wb->writeback_inodes);		atomic_inc(&new_wb->writeback_inodes);	} 	wb_get(new_wb); 	/*	 * Transfer to @new_wb's IO list if necessary.  If the @inode is dirty,	 * the specific list @inode was on is ignored and the @inode is put on	 * ->b_dirty which is always correct including from ->b_dirty_time.	 * If the @inode was clean, it means it was on the b_attached list, so	 * move it onto the b_attached list of @new_wb.	 */	if (!list_empty(&inode->i_io_list)) {		inode->i_wb = new_wb; 		if (inode_state_read(inode) & I_DIRTY_ALL) {			/*			 * We need to keep b_dirty list sorted by			 * dirtied_time_when. However properly sorting the			 * inode in the list gets too expensive when switching			 * many inodes. So just attach inode at the end of the			 * dirty list and clobber the dirtied_time_when.			 */			inode->dirtied_time_when = jiffies;			inode_io_list_move_locked(inode, new_wb,						  &new_wb->b_dirty);		} else {			inode_cgwb_move_to_attached(inode, new_wb);		}	} else {		inode->i_wb = new_wb;	} 	/* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */	inode->i_wb_frn_winner = 0;	inode->i_wb_frn_avg_time = 0;	inode->i_wb_frn_history = 0;	switched = true;skip_switch:	/*	 * Paired with an acquire fence in unlocked_inode_to_wb_begin() and	 * ensures that the new wb is visible if they see !I_WB_SWITCH.	 */	smp_wmb();	inode_state_clear(inode, I_WB_SWITCH); 	xa_unlock_irq(&mapping->i_pages);	spin_unlock(&inode->i_lock); 

Lines 1–500 of 3082. Load more