2262 lines · 59.2 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0/* * Common Block IO controller cgroup interface * * Based on ideas and code from CFQ, CFS and BFQ: * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk> * * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it> *		      Paolo Valente <paolo.valente@unimore.it> * * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com> * 	              Nauman Rafique <nauman@google.com> * * For policy-specific per-blkcg data: * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it> *                    Arianna Avanzini <avanzini.arianna@gmail.com> */#include <linux/ioprio.h>#include <linux/kdev_t.h>#include <linux/module.h>#include <linux/sched/signal.h>#include <linux/err.h>#include <linux/blkdev.h>#include <linux/backing-dev.h>#include <linux/slab.h>#include <linux/delay.h>#include <linux/wait_bit.h>#include <linux/atomic.h>#include <linux/ctype.h>#include <linux/resume_user_mode.h>#include <linux/psi.h>#include <linux/part_stat.h>#include "blk.h"#include "blk-cgroup.h"#include "blk-ioprio.h"#include "blk-throttle.h" static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu); /* * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation. * blkcg_pol_register_mutex nests outside of it and synchronizes entire * policy [un]register operations including cgroup file additions / * removals.  Putting cgroup file registration outside blkcg_pol_mutex * allows grabbing it from cgroup callbacks. */static DEFINE_MUTEX(blkcg_pol_register_mutex);static DEFINE_MUTEX(blkcg_pol_mutex); struct blkcg blkcg_root;EXPORT_SYMBOL_GPL(blkcg_root); struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;EXPORT_SYMBOL_GPL(blkcg_root_css); /* number of blkcgs with a non-zero congestion_count */atomic_t blkcg_nr_congested __read_mostly = ATOMIC_INIT(0); static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS]; static LIST_HEAD(all_blkcgs);		/* protected by blkcg_pol_mutex */ bool blkcg_debug_stats = false; static DEFINE_RAW_SPINLOCK(blkg_stat_lock); #define BLKG_DESTROY_BATCH_SIZE  64 /* * Lockless lists for tracking IO stats update * * New IO stats are stored in the percpu iostat_cpu within blkcg_gq (blkg). * There are multiple blkg's (one for each block device) attached to each * blkcg. The rstat code keeps track of which cpu has IO stats updated, * but it doesn't know which blkg has the updated stats. If there are many * block devices in a system, the cost of iterating all the blkg's to flush * out the IO stats can be high. To reduce such overhead, a set of percpu * lockless lists (lhead) per blkcg are used to track the set of recently * updated iostat_cpu's since the last flush. An iostat_cpu will be put * onto the lockless list on the update side [blk_cgroup_bio_start()] if * not there yet and then removed when being flushed [blkcg_rstat_flush()]. * References to blkg are gotten and then put back in the process to * protect against blkg removal. * * Return: 0 if successful or -ENOMEM if allocation fails. */static int init_blkcg_llists(struct blkcg *blkcg){	int cpu; 	blkcg->lhead = alloc_percpu_gfp(struct llist_head, GFP_KERNEL);	if (!blkcg->lhead)		return -ENOMEM; 	for_each_possible_cpu(cpu)		init_llist_head(per_cpu_ptr(blkcg->lhead, cpu));	return 0;} /** * blkcg_css - find the current css * * Find the css associated with either the kthread or the current task. * This may return a dying css, so it is up to the caller to use tryget logic * to confirm it is alive and well. */static struct cgroup_subsys_state *blkcg_css(void){	struct cgroup_subsys_state *css; 	css = kthread_blkcg();	if (css)		return css;	return task_css(current, io_cgrp_id);} static void blkg_free_workfn(struct work_struct *work){	struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,					     free_work);	struct request_queue *q = blkg->q;	int i; 	/*	 * pd_free_fn() can also be called from blkcg_deactivate_policy(),	 * in order to make sure pd_free_fn() is called in order, the deletion	 * of the list blkg->q_node is delayed to here from blkg_destroy(), and	 * blkcg_mutex is used to synchronize blkg_free_workfn() and	 * blkcg_deactivate_policy().	 */	mutex_lock(&q->blkcg_mutex);	for (i = 0; i < BLKCG_MAX_POLS; i++)		if (blkg->pd[i])			blkcg_policy[i]->pd_free_fn(blkg->pd[i]);	if (blkg->parent)		blkg_put(blkg->parent);	spin_lock_irq(&q->queue_lock);	list_del_init(&blkg->q_node);	spin_unlock_irq(&q->queue_lock);	mutex_unlock(&q->blkcg_mutex); 	/*	 * Release blkcg css ref only after blkg is removed from q->blkg_list,	 * so concurrent iterators won't see a blkg with a freed blkcg.	 */	css_put(&blkg->blkcg->css);	blk_put_queue(q);	free_percpu(blkg->iostat_cpu);	percpu_ref_exit(&blkg->refcnt);	kfree(blkg);} /** * blkg_free - free a blkg * @blkg: blkg to free * * Free @blkg which may be partially allocated. */static void blkg_free(struct blkcg_gq *blkg){	if (!blkg)		return; 	/*	 * Both ->pd_free_fn() and request queue's release handler may	 * sleep, so free us by scheduling one work func	 */	INIT_WORK(&blkg->free_work, blkg_free_workfn);	schedule_work(&blkg->free_work);} static void __blkg_release(struct rcu_head *rcu){	struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); #ifdef CONFIG_BLK_CGROUP_PUNT_BIO	WARN_ON(!bio_list_empty(&blkg->async_bios));#endif 	blkg_free(blkg);} /* * A group is RCU protected, but having an rcu lock does not mean that one * can access all the fields of blkg and assume these are valid.  For * example, don't try to follow throtl_data and request queue links. * * Having a reference to blkg under an rcu allows accesses to only values * local to groups like group stats and group rate limits. */static void blkg_release(struct percpu_ref *ref){	struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);	struct blkcg *blkcg = blkg->blkcg;	int cpu; 	/*	 * Flush all the non-empty percpu lockless lists before releasing	 * us, given these stat belongs to us.	 *	 * blkg_stat_lock is for serializing blkg stat update	 */	for_each_possible_cpu(cpu)		__blkcg_rstat_flush(blkcg, cpu); 	call_rcu(&blkg->rcu_head, __blkg_release);} #ifdef CONFIG_BLK_CGROUP_PUNT_BIOstatic struct workqueue_struct *blkcg_punt_bio_wq; static void blkg_async_bio_workfn(struct work_struct *work){	struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,					     async_bio_work);	struct bio_list bios = BIO_EMPTY_LIST;	struct bio *bio;	struct blk_plug plug;	bool need_plug = false; 	/* as long as there are pending bios, @blkg can't go away */	spin_lock(&blkg->async_bio_lock);	bio_list_merge_init(&bios, &blkg->async_bios);	spin_unlock(&blkg->async_bio_lock); 	/* start plug only when bio_list contains at least 2 bios */	if (bios.head && bios.head->bi_next) {		need_plug = true;		blk_start_plug(&plug);	}	while ((bio = bio_list_pop(&bios)))		submit_bio(bio);	if (need_plug)		blk_finish_plug(&plug);} /* * When a shared kthread issues a bio for a cgroup, doing so synchronously can * lead to priority inversions as the kthread can be trapped waiting for that * cgroup.  Use this helper instead of submit_bio to punt the actual issuing to * a dedicated per-blkcg work item to avoid such priority inversions. */void blkcg_punt_bio_submit(struct bio *bio){	struct blkcg_gq *blkg = bio->bi_blkg; 	if (blkg->parent) {		spin_lock(&blkg->async_bio_lock);		bio_list_add(&blkg->async_bios, bio);		spin_unlock(&blkg->async_bio_lock);		queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);	} else {		/* never bounce for the root cgroup */		submit_bio(bio);	}}EXPORT_SYMBOL_GPL(blkcg_punt_bio_submit); static int __init blkcg_punt_bio_init(void){	blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",					    WQ_MEM_RECLAIM | WQ_FREEZABLE |					    WQ_UNBOUND | WQ_SYSFS, 0);	if (!blkcg_punt_bio_wq)		return -ENOMEM;	return 0;}subsys_initcall(blkcg_punt_bio_init);#endif /* CONFIG_BLK_CGROUP_PUNT_BIO */ /** * bio_blkcg_css - return the blkcg CSS associated with a bio * @bio: target bio * * This returns the CSS for the blkcg associated with a bio, or %NULL if not * associated. Callers are expected to either handle %NULL or know association * has been done prior to calling this. */struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio){	if (!bio || !bio->bi_blkg)		return NULL;	return &bio->bi_blkg->blkcg->css;}EXPORT_SYMBOL_GPL(bio_blkcg_css); /** * blkcg_parent - get the parent of a blkcg * @blkcg: blkcg of interest * * Return the parent blkcg of @blkcg.  Can be called anytime. */static inline struct blkcg *blkcg_parent(struct blkcg *blkcg){	return css_to_blkcg(blkcg->css.parent);} /** * blkg_alloc - allocate a blkg * @blkcg: block cgroup the new blkg is associated with * @disk: gendisk the new blkg is associated with * @gfp_mask: allocation mask to use * * Allocate a new blkg associating @blkcg and @disk. */static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,				   gfp_t gfp_mask){	struct blkcg_gq *blkg;	int i, cpu; 	/* alloc and init base part */	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, disk->queue->node);	if (!blkg)		return NULL;	if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))		goto out_free_blkg;	blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);	if (!blkg->iostat_cpu)		goto out_exit_refcnt;	if (!blk_get_queue(disk->queue))		goto out_free_iostat;	/* blkg holds a reference to blkcg */	if (!css_tryget_online(&blkcg->css))		goto out_put_queue; 	blkg->q = disk->queue;	INIT_LIST_HEAD(&blkg->q_node);	blkg->blkcg = blkcg;	blkg->iostat.blkg = blkg;#ifdef CONFIG_BLK_CGROUP_PUNT_BIO	spin_lock_init(&blkg->async_bio_lock);	bio_list_init(&blkg->async_bios);	INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);#endif 	u64_stats_init(&blkg->iostat.sync);	for_each_possible_cpu(cpu) {		u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);		per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;	} 	for (i = 0; i < BLKCG_MAX_POLS; i++) {		struct blkcg_policy *pol = blkcg_policy[i];		struct blkg_policy_data *pd; 		if (!blkcg_policy_enabled(disk->queue, pol))			continue; 		/* alloc per-policy data and attach it to blkg */		pd = pol->pd_alloc_fn(disk, blkcg, gfp_mask);		if (!pd)			goto out_free_pds;		blkg->pd[i] = pd;		pd->blkg = blkg;		pd->plid = i;		pd->online = false;	} 	return blkg; out_free_pds:	while (--i >= 0)		if (blkg->pd[i])			blkcg_policy[i]->pd_free_fn(blkg->pd[i]);	css_put(&blkcg->css);out_put_queue:	blk_put_queue(disk->queue);out_free_iostat:	free_percpu(blkg->iostat_cpu);out_exit_refcnt:	percpu_ref_exit(&blkg->refcnt);out_free_blkg:	kfree(blkg);	return NULL;} /* * If @new_blkg is %NULL, this function tries to allocate a new one as * necessary using %GFP_NOWAIT.  @new_blkg is always consumed on return. */static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,				    struct blkcg_gq *new_blkg){	struct blkcg_gq *blkg;	int i, ret; 	lockdep_assert_held(&disk->queue->queue_lock); 	/* request_queue is dying, do not create/recreate a blkg */	if (blk_queue_dying(disk->queue)) {		ret = -ENODEV;		goto err_free_blkg;	} 	/* allocate */	if (!new_blkg) {		new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT);		if (unlikely(!new_blkg)) {			ret = -ENOMEM;			goto err_free_blkg;		}	}	blkg = new_blkg; 	/* link parent */	if (blkcg_parent(blkcg)) {		blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue);		if (WARN_ON_ONCE(!blkg->parent)) {			ret = -ENODEV;			goto err_free_blkg;		}		blkg_get(blkg->parent);	} 	/* invoke per-policy init */	for (i = 0; i < BLKCG_MAX_POLS; i++) {		struct blkcg_policy *pol = blkcg_policy[i]; 		if (blkg->pd[i] && pol->pd_init_fn)			pol->pd_init_fn(blkg->pd[i]);	} 	/* insert */	spin_lock(&blkcg->lock);	ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg);	if (likely(!ret)) {		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);		list_add(&blkg->q_node, &disk->queue->blkg_list); 		for (i = 0; i < BLKCG_MAX_POLS; i++) {			struct blkcg_policy *pol = blkcg_policy[i]; 			if (blkg->pd[i]) {				if (pol->pd_online_fn)					pol->pd_online_fn(blkg->pd[i]);				blkg->pd[i]->online = true;			}		}		blkg->online = true;	}	spin_unlock(&blkcg->lock); 	if (!ret)		return blkg; 	/* @blkg failed fully initialized, use the usual release path */	percpu_ref_kill(&blkg->refcnt);	return ERR_PTR(ret); err_free_blkg:	if (new_blkg)		blkg_free(new_blkg);	return ERR_PTR(ret);} /** * blkg_lookup_create - lookup blkg, try to create one if not there * @blkcg: blkcg of interest * @disk: gendisk of interest * * Lookup blkg for the @blkcg - @disk pair.  If it doesn't exist, try to * create one.  blkg creation is performed recursively from blkcg_root such * that all non-root blkg's have access to the parent blkg.  This function * should be called under RCU read lock and takes @disk->queue->queue_lock. * * Returns the blkg or the closest blkg if blkg_create() fails as it walks * down from root. */static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,		struct gendisk *disk){	struct request_queue *q = disk->queue;	struct blkcg_gq *blkg; 	rcu_read_lock();	blkg = blkg_lookup(blkcg, q);	if (blkg) {		if (blkcg != &blkcg_root &&		    blkg != rcu_dereference(blkcg->blkg_hint))			rcu_assign_pointer(blkcg->blkg_hint, blkg);		rcu_read_unlock();		return blkg;	}	rcu_read_unlock(); 	/*	 * Create blkgs walking down from blkcg_root to @blkcg, so that all	 * non-root blkgs have access to their parents.  Returns the closest	 * blkg to the intended blkg should blkg_create() fail.	 */	while (true) {		struct blkcg *pos = blkcg;		struct blkcg *parent = blkcg_parent(blkcg);		struct blkcg_gq *ret_blkg = q->root_blkg; 		while (parent) {			blkg = blkg_lookup(parent, q);			if (blkg) {				/* remember closest blkg */

Lines 1–500 of 2262.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t