// SPDX-License-Identifier: GPL-2.0-only #include <linux/wait.h>#include <linux/rbtree.h>#include <linux/backing-dev.h>#include <linux/kthread.h>#include <linux/freezer.h>#include <linux/fs.h>#include <linux/pagemap.h>#include <linux/mm.h>#include <linux/sched/mm.h>#include <linux/sched.h>#include <linux/module.h>#include <linux/writeback.h>#include <linux/device.h>#include <trace/events/writeback.h> struct backing_dev_info noop_backing_dev_info;EXPORT_SYMBOL_GPL(noop_backing_dev_info); static struct class *bdi_class;static const char *bdi_unknown_name = "(unknown)"; /* * bdi_lock protects bdi_tree and updates to bdi_list. bdi_list has RCU * reader side locking. */DEFINE_SPINLOCK(bdi_lock);static u64 bdi_id_cursor;static struct rb_root bdi_tree = RB_ROOT;LIST_HEAD(bdi_list); /* bdi_wq serves all asynchronous writeback tasks */struct workqueue_struct *bdi_wq; #define K(x) ((x) << (PAGE_SHIFT - 10)) #ifdef CONFIG_DEBUG_FS#include <linux/debugfs.h>#include <linux/seq_file.h> static struct dentry *bdi_debug_root; static void bdi_debug_init(void){	bdi_debug_root = debugfs_create_dir("bdi", NULL);} static int bdi_debug_stats_show(struct seq_file *m, void *v){	struct backing_dev_info *bdi = m->private;	struct bdi_writeback *wb = &bdi->wb;	unsigned long background_thresh;	unsigned long dirty_thresh;	unsigned long wb_thresh;	unsigned long nr_dirty, nr_io, nr_more_io, nr_dirty_time;	struct inode *inode; 	nr_dirty = nr_io = nr_more_io = nr_dirty_time = 0;	spin_lock(&wb->list_lock);	list_for_each_entry(inode, &wb->b_dirty, i_io_list)		nr_dirty++;	list_for_each_entry(inode, &wb->b_io, i_io_list)		nr_io++;	list_for_each_entry(inode, &wb->b_more_io, i_io_list)		nr_more_io++;	list_for_each_entry(inode, &wb->b_dirty_time, i_io_list)		if (inode->i_state & I_DIRTY_TIME)			nr_dirty_time++;	spin_unlock(&wb->list_lock); 	global_dirty_limits(&background_thresh, &dirty_thresh);	wb_thresh = wb_calc_thresh(wb, dirty_thresh); 	seq_printf(m,		   "BdiWriteback:       %10lu kB\n"		   "BdiReclaimable:     %10lu kB\n"		   "BdiDirtyThresh:     %10lu kB\n"		   "DirtyThresh:        %10lu kB\n"		   "BackgroundThresh:   %10lu kB\n"		   "BdiDirtied:         %10lu kB\n"		   "BdiWritten:         %10lu kB\n"		   "BdiWriteBandwidth:  %10lu kBps\n"		   "b_dirty:            %10lu\n"		   "b_io:               %10lu\n"		   "b_more_io:          %10lu\n"		   "b_dirty_time:       %10lu\n"		   "bdi_list:           %10u\n"		   "state:              %10lx\n",		   (unsigned long) K(wb_stat(wb, WB_WRITEBACK)),		   (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),		   K(wb_thresh),		   K(dirty_thresh),		   K(background_thresh),		   (unsigned long) K(wb_stat(wb, WB_DIRTIED)),		   (unsigned long) K(wb_stat(wb, WB_WRITTEN)),		   (unsigned long) K(wb->write_bandwidth),		   nr_dirty,		   nr_io,		   nr_more_io,		   nr_dirty_time,		   !list_empty(&bdi->bdi_list), bdi->wb.state); 	return 0;}DEFINE_SHOW_ATTRIBUTE(bdi_debug_stats); static void bdi_debug_register(struct backing_dev_info *bdi, const char *name){	bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root); 	debugfs_create_file("stats", 0444, bdi->debug_dir, bdi,			    &bdi_debug_stats_fops);} static void bdi_debug_unregister(struct backing_dev_info *bdi){	debugfs_remove_recursive(bdi->debug_dir);}#elsestatic inline void bdi_debug_init(void){}static inline void bdi_debug_register(struct backing_dev_info *bdi,				      const char *name){}static inline void bdi_debug_unregister(struct backing_dev_info *bdi){}#endif static ssize_t read_ahead_kb_store(struct device *dev,				  struct device_attribute *attr,				  const char *buf, size_t count){	struct backing_dev_info *bdi = dev_get_drvdata(dev);	unsigned long read_ahead_kb;	ssize_t ret; 	ret = kstrtoul(buf, 10, &read_ahead_kb);	if (ret < 0)		return ret; 	bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10); 	return count;} #define BDI_SHOW(name, expr)						\static ssize_t name##_show(struct device *dev,				\			   struct device_attribute *attr, char *buf)	\{									\	struct backing_dev_info *bdi = dev_get_drvdata(dev);		\									\	return sysfs_emit(buf, "%lld\n", (long long)expr);		\}									\static DEVICE_ATTR_RW(name); BDI_SHOW(read_ahead_kb, K(bdi->ra_pages)) static ssize_t min_ratio_store(struct device *dev,		struct device_attribute *attr, const char *buf, size_t count){	struct backing_dev_info *bdi = dev_get_drvdata(dev);	unsigned int ratio;	ssize_t ret; 	ret = kstrtouint(buf, 10, &ratio);	if (ret < 0)		return ret; 	ret = bdi_set_min_ratio(bdi, ratio);	if (!ret)		ret = count; 	return ret;}BDI_SHOW(min_ratio, bdi->min_ratio) static ssize_t max_ratio_store(struct device *dev,		struct device_attribute *attr, const char *buf, size_t count){	struct backing_dev_info *bdi = dev_get_drvdata(dev);	unsigned int ratio;	ssize_t ret; 	ret = kstrtouint(buf, 10, &ratio);	if (ret < 0)		return ret; 	ret = bdi_set_max_ratio(bdi, ratio);	if (!ret)		ret = count; 	return ret;}BDI_SHOW(max_ratio, bdi->max_ratio) static ssize_t stable_pages_required_show(struct device *dev,					  struct device_attribute *attr,					  char *buf){	dev_warn_once(dev,		"the stable_pages_required attribute has been removed. Use the stable_writes queue attribute instead.\n");	return sysfs_emit(buf, "%d\n", 0);}static DEVICE_ATTR_RO(stable_pages_required); static struct attribute *bdi_dev_attrs[] = {	&dev_attr_read_ahead_kb.attr,	&dev_attr_min_ratio.attr,	&dev_attr_max_ratio.attr,	&dev_attr_stable_pages_required.attr,	NULL,};ATTRIBUTE_GROUPS(bdi_dev); static __init int bdi_class_init(void){	bdi_class = class_create(THIS_MODULE, "bdi");	if (IS_ERR(bdi_class))		return PTR_ERR(bdi_class); 	bdi_class->dev_groups = bdi_dev_groups;	bdi_debug_init(); 	return 0;}postcore_initcall(bdi_class_init); static int bdi_init(struct backing_dev_info *bdi); static int __init default_bdi_init(void){	int err; 	bdi_wq = alloc_workqueue("writeback", WQ_MEM_RECLAIM | WQ_UNBOUND |				 WQ_SYSFS, 0);	if (!bdi_wq)		return -ENOMEM; 	err = bdi_init(&noop_backing_dev_info); 	return err;}subsys_initcall(default_bdi_init); /* * This function is used when the first inode for this wb is marked dirty. It * wakes-up the corresponding bdi thread which should then take care of the * periodic background write-out of dirty inodes. Since the write-out would * starts only 'dirty_writeback_interval' centisecs from now anyway, we just * set up a timer which wakes the bdi thread up later. * * Note, we wouldn't bother setting up the timer, but this function is on the * fast-path (used by '__mark_inode_dirty()'), so we save few context switches * by delaying the wake-up. * * We have to be careful not to postpone flush work if it is scheduled for * earlier. Thus we use queue_delayed_work(). */void wb_wakeup_delayed(struct bdi_writeback *wb){	unsigned long timeout; 	timeout = msecs_to_jiffies(dirty_writeback_interval * 10);	spin_lock_bh(&wb->work_lock);	if (test_bit(WB_registered, &wb->state))		queue_delayed_work(bdi_wq, &wb->dwork, timeout);	spin_unlock_bh(&wb->work_lock);} /* * Initial write bandwidth: 100 MB/s */#define INIT_BW		(100 << (20 - PAGE_SHIFT)) static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,		   gfp_t gfp){	int i, err; 	memset(wb, 0, sizeof(*wb)); 	if (wb != &bdi->wb)		bdi_get(bdi);	wb->bdi = bdi;	wb->last_old_flush = jiffies;	INIT_LIST_HEAD(&wb->b_dirty);	INIT_LIST_HEAD(&wb->b_io);	INIT_LIST_HEAD(&wb->b_more_io);	INIT_LIST_HEAD(&wb->b_dirty_time);	spin_lock_init(&wb->list_lock); 	wb->bw_time_stamp = jiffies;	wb->balanced_dirty_ratelimit = INIT_BW;	wb->dirty_ratelimit = INIT_BW;	wb->write_bandwidth = INIT_BW;	wb->avg_write_bandwidth = INIT_BW; 	spin_lock_init(&wb->work_lock);	INIT_LIST_HEAD(&wb->work_list);	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);	wb->dirty_sleep = jiffies; 	err = fprop_local_init_percpu(&wb->completions, gfp);	if (err)		goto out_put_bdi; 	for (i = 0; i < NR_WB_STAT_ITEMS; i++) {		err = percpu_counter_init(&wb->stat[i], 0, gfp);		if (err)			goto out_destroy_stat;	} 	return 0; out_destroy_stat:	while (i--)		percpu_counter_destroy(&wb->stat[i]);	fprop_local_destroy_percpu(&wb->completions);out_put_bdi:	if (wb != &bdi->wb)		bdi_put(bdi);	return err;} static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb); /* * Remove bdi from the global list and shutdown any threads we have running */static void wb_shutdown(struct bdi_writeback *wb){	/* Make sure nobody queues further work */	spin_lock_bh(&wb->work_lock);	if (!test_and_clear_bit(WB_registered, &wb->state)) {		spin_unlock_bh(&wb->work_lock);		return;	}	spin_unlock_bh(&wb->work_lock); 	cgwb_remove_from_bdi_list(wb);	/*	 * Drain work list and shutdown the delayed_work.  !WB_registered	 * tells wb_workfn() that @wb is dying and its work_list needs to	 * be drained no matter what.	 */	mod_delayed_work(bdi_wq, &wb->dwork, 0);	flush_delayed_work(&wb->dwork);	WARN_ON(!list_empty(&wb->work_list));} static void wb_exit(struct bdi_writeback *wb){	int i; 	WARN_ON(delayed_work_pending(&wb->dwork)); 	for (i = 0; i < NR_WB_STAT_ITEMS; i++)		percpu_counter_destroy(&wb->stat[i]); 	fprop_local_destroy_percpu(&wb->completions);	if (wb != &wb->bdi->wb)		bdi_put(wb->bdi);} #ifdef CONFIG_CGROUP_WRITEBACK #include <linux/memcontrol.h> /* * cgwb_lock protects bdi->cgwb_tree, blkcg->cgwb_list, and memcg->cgwb_list. * bdi->cgwb_tree is also RCU protected. */static DEFINE_SPINLOCK(cgwb_lock);static struct workqueue_struct *cgwb_release_wq; static void cgwb_release_workfn(struct work_struct *work){	struct bdi_writeback *wb = container_of(work, struct bdi_writeback,						release_work);	struct blkcg *blkcg = css_to_blkcg(wb->blkcg_css); 	mutex_lock(&wb->bdi->cgwb_release_mutex);	wb_shutdown(wb); 	css_put(wb->memcg_css);	css_put(wb->blkcg_css);	mutex_unlock(&wb->bdi->cgwb_release_mutex); 	/* triggers blkg destruction if no online users left */	blkcg_unpin_online(blkcg); 	fprop_local_destroy_percpu(&wb->memcg_completions);	percpu_ref_exit(&wb->refcnt);	wb_exit(wb);	kfree_rcu(wb, rcu);} static void cgwb_release(struct percpu_ref *refcnt){	struct bdi_writeback *wb = container_of(refcnt, struct bdi_writeback,						refcnt);	queue_work(cgwb_release_wq, &wb->release_work);} static void cgwb_kill(struct bdi_writeback *wb){	lockdep_assert_held(&cgwb_lock); 	WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id));	list_del(&wb->memcg_node);	list_del(&wb->blkcg_node);	percpu_ref_kill(&wb->refcnt);} static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb){	spin_lock_irq(&cgwb_lock);	list_del_rcu(&wb->bdi_node);	spin_unlock_irq(&cgwb_lock);} static int cgwb_create(struct backing_dev_info *bdi,		       struct cgroup_subsys_state *memcg_css, gfp_t gfp){	struct mem_cgroup *memcg;	struct cgroup_subsys_state *blkcg_css;	struct blkcg *blkcg;	struct list_head *memcg_cgwb_list, *blkcg_cgwb_list;	struct bdi_writeback *wb;	unsigned long flags;	int ret = 0; 	memcg = mem_cgroup_from_css(memcg_css);	blkcg_css = cgroup_get_e_css(memcg_css->cgroup, &io_cgrp_subsys);	blkcg = css_to_blkcg(blkcg_css);	memcg_cgwb_list = &memcg->cgwb_list;	blkcg_cgwb_list = &blkcg->cgwb_list; 	/* look up again under lock and discard on blkcg mismatch */	spin_lock_irqsave(&cgwb_lock, flags);	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);	if (wb && wb->blkcg_css != blkcg_css) {		cgwb_kill(wb);		wb = NULL;	}	spin_unlock_irqrestore(&cgwb_lock, flags);	if (wb)		goto out_put; 	/* need to create a new one */	wb = kmalloc(sizeof(*wb), gfp);	if (!wb) {		ret = -ENOMEM;		goto out_put;	} 	ret = wb_init(wb, bdi, gfp);	if (ret)		goto err_free; 	ret = percpu_ref_init(&wb->refcnt, cgwb_release, 0, gfp);	if (ret)		goto err_wb_exit; 	ret = fprop_local_init_percpu(&wb->memcg_completions, gfp);	if (ret)		goto err_ref_exit; 	wb->memcg_css = memcg_css;	wb->blkcg_css = blkcg_css;	INIT_WORK(&wb->release_work, cgwb_release_workfn);	set_bit(WB_registered, &wb->state); 	/*	 * The root wb determines the registered state of the whole bdi and	 * memcg_cgwb_list and blkcg_cgwb_list's next pointers indicate	 * whether they're still online.  Don't link @wb if any is dead.	 * See wb_memcg_offline() and wb_blkcg_offline().	 */	ret = -ENODEV;	spin_lock_irqsave(&cgwb_lock, flags);	if (test_bit(WB_registered, &bdi->wb.state) &&	    blkcg_cgwb_list->next && memcg_cgwb_list->next) {		/* we might have raced another instance of this function */		ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb);		if (!ret) {			list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list);			list_add(&wb->memcg_node, memcg_cgwb_list);			list_add(&wb->blkcg_node, blkcg_cgwb_list);			blkcg_pin_online(blkcg);			css_get(memcg_css);			css_get(blkcg_css);		}	}	spin_unlock_irqrestore(&cgwb_lock, flags);	if (ret) {

Lines 1–500 of 1015. Load more