/* * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public Licens * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111- * */#include <linux/mm.h>#include <linux/swap.h>#include <linux/bio.h>#include <linux/blkdev.h>#include <linux/uio.h>#include <linux/iocontext.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/mempool.h>#include <linux/workqueue.h>#include <linux/cgroup.h> #include <trace/events/block.h>#include "blk.h" /* * Test patch to inline a certain number of bi_io_vec's inside the bio * itself, to shrink a bio data allocation from two mempool calls to one */#define BIO_INLINE_VECS		4 /* * if you change this list, also change bvec_alloc or things will * break badly! cannot be bigger than what you can fit into an * unsigned short */#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {	BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),};#undef BV /* * fs_bio_set is the bio_set containing bio and iovec memory pools used by * IO code that does not need private memory pools. */struct bio_set *fs_bio_set;EXPORT_SYMBOL(fs_bio_set); /* * Our slab pool management */struct bio_slab {	struct kmem_cache *slab;	unsigned int slab_ref;	unsigned int slab_size;	char name[8];};static DEFINE_MUTEX(bio_slab_lock);static struct bio_slab *bio_slabs;static unsigned int bio_slab_nr, bio_slab_max; static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size){	unsigned int sz = sizeof(struct bio) + extra_size;	struct kmem_cache *slab = NULL;	struct bio_slab *bslab, *new_bio_slabs;	unsigned int new_bio_slab_max;	unsigned int i, entry = -1; 	mutex_lock(&bio_slab_lock); 	i = 0;	while (i < bio_slab_nr) {		bslab = &bio_slabs[i]; 		if (!bslab->slab && entry == -1)			entry = i;		else if (bslab->slab_size == sz) {			slab = bslab->slab;			bslab->slab_ref++;			break;		}		i++;	} 	if (slab)		goto out_unlock; 	if (bio_slab_nr == bio_slab_max && entry == -1) {		new_bio_slab_max = bio_slab_max << 1;		new_bio_slabs = krealloc(bio_slabs,					 new_bio_slab_max * sizeof(struct bio_slab),					 GFP_KERNEL);		if (!new_bio_slabs)			goto out_unlock;		bio_slab_max = new_bio_slab_max;		bio_slabs = new_bio_slabs;	}	if (entry == -1)		entry = bio_slab_nr++; 	bslab = &bio_slabs[entry]; 	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);	slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,				 SLAB_HWCACHE_ALIGN, NULL);	if (!slab)		goto out_unlock; 	bslab->slab = slab;	bslab->slab_ref = 1;	bslab->slab_size = sz;out_unlock:	mutex_unlock(&bio_slab_lock);	return slab;} static void bio_put_slab(struct bio_set *bs){	struct bio_slab *bslab = NULL;	unsigned int i; 	mutex_lock(&bio_slab_lock); 	for (i = 0; i < bio_slab_nr; i++) {		if (bs->bio_slab == bio_slabs[i].slab) {			bslab = &bio_slabs[i];			break;		}	} 	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))		goto out; 	WARN_ON(!bslab->slab_ref); 	if (--bslab->slab_ref)		goto out; 	kmem_cache_destroy(bslab->slab);	bslab->slab = NULL; out:	mutex_unlock(&bio_slab_lock);} unsigned int bvec_nr_vecs(unsigned short idx){	return bvec_slabs[idx].nr_vecs;} void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx){	if (!idx)		return;	idx--; 	BIO_BUG_ON(idx >= BVEC_POOL_NR); 	if (idx == BVEC_POOL_MAX) {		mempool_free(bv, pool);	} else {		struct biovec_slab *bvs = bvec_slabs + idx; 		kmem_cache_free(bvs->slab, bv);	}} struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,			   mempool_t *pool){	struct bio_vec *bvl; 	/*	 * see comment near bvec_array define!	 */	switch (nr) {	case 1:		*idx = 0;		break;	case 2 ... 4:		*idx = 1;		break;	case 5 ... 16:		*idx = 2;		break;	case 17 ... 64:		*idx = 3;		break;	case 65 ... 128:		*idx = 4;		break;	case 129 ... BIO_MAX_PAGES:		*idx = 5;		break;	default:		return NULL;	} 	/*	 * idx now points to the pool we want to allocate from. only the	 * 1-vec entry pool is mempool backed.	 */	if (*idx == BVEC_POOL_MAX) {fallback:		bvl = mempool_alloc(pool, gfp_mask);	} else {		struct biovec_slab *bvs = bvec_slabs + *idx;		gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO); 		/*		 * Make this allocation restricted and don't dump info on		 * allocation failures, since we'll fallback to the mempool		 * in case of failure.		 */		__gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 		/*		 * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM		 * is set, retry with the 1-entry mempool		 */		bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);		if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {			*idx = BVEC_POOL_MAX;			goto fallback;		}	} 	(*idx)++;	return bvl;} void bio_uninit(struct bio *bio){	bio_disassociate_task(bio);}EXPORT_SYMBOL(bio_uninit); static void bio_free(struct bio *bio){	struct bio_set *bs = bio->bi_pool;	void *p; 	bio_uninit(bio); 	if (bs) {		bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio)); 		/*		 * If we have front padding, adjust the bio pointer before freeing		 */		p = bio;		p -= bs->front_pad; 		mempool_free(p, bs->bio_pool);	} else {		/* Bio was allocated by bio_kmalloc() */		kfree(bio);	}} /* * Users of this function have their own bio allocation. Subsequently, * they must remember to pair any call to bio_init() with bio_uninit() * when IO has completed, or when the bio is released. */void bio_init(struct bio *bio, struct bio_vec *table,	      unsigned short max_vecs){	memset(bio, 0, sizeof(*bio));	atomic_set(&bio->__bi_remaining, 1);	atomic_set(&bio->__bi_cnt, 1); 	bio->bi_io_vec = table;	bio->bi_max_vecs = max_vecs;}EXPORT_SYMBOL(bio_init); /** * bio_reset - reinitialize a bio * @bio:	bio to reset * * Description: *   After calling bio_reset(), @bio will be in the same state as a freshly *   allocated bio returned bio bio_alloc_bioset() - the only fields that are *   preserved are the ones that are initialized by bio_alloc_bioset(). See *   comment in struct bio. */void bio_reset(struct bio *bio){	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS); 	bio_uninit(bio); 	memset(bio, 0, BIO_RESET_BYTES);	bio->bi_flags = flags;	atomic_set(&bio->__bi_remaining, 1);}EXPORT_SYMBOL(bio_reset); static struct bio *__bio_chain_endio(struct bio *bio){	struct bio *parent = bio->bi_private; 	if (!parent->bi_status)		parent->bi_status = bio->bi_status;	bio_put(bio);	return parent;} static void bio_chain_endio(struct bio *bio){	bio_endio(__bio_chain_endio(bio));} /** * bio_chain - chain bio completions * @bio: the target bio * @parent: the @bio's parent bio * * The caller won't have a bi_end_io called when @bio completes - instead, * @parent's bi_end_io won't be called until both @parent and @bio have * completed; the chained bio will also be freed when it completes. * * The caller must not set bi_private or bi_end_io in @bio. */void bio_chain(struct bio *bio, struct bio *parent){	BUG_ON(bio->bi_private || bio->bi_end_io); 	bio->bi_private = parent;	bio->bi_end_io	= bio_chain_endio;	bio_inc_remaining(parent);}EXPORT_SYMBOL(bio_chain); static void bio_alloc_rescue(struct work_struct *work){	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);	struct bio *bio; 	while (1) {		spin_lock(&bs->rescue_lock);		bio = bio_list_pop(&bs->rescue_list);		spin_unlock(&bs->rescue_lock); 		if (!bio)			break; 		generic_make_request(bio);	}} static void punt_bios_to_rescuer(struct bio_set *bs){	struct bio_list punt, nopunt;	struct bio *bio; 	if (WARN_ON_ONCE(!bs->rescue_workqueue))		return;	/*	 * In order to guarantee forward progress we must punt only bios that	 * were allocated from this bio_set; otherwise, if there was a bio on	 * there for a stacking driver higher up in the stack, processing it	 * could require allocating bios from this bio_set, and doing that from	 * our own rescuer would be bad.	 *	 * Since bio lists are singly linked, pop them all instead of trying to	 * remove from the middle of the list:	 */ 	bio_list_init(&punt);	bio_list_init(&nopunt); 	while ((bio = bio_list_pop(&current->bio_list[0])))		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);	current->bio_list[0] = nopunt; 	bio_list_init(&nopunt);	while ((bio = bio_list_pop(&current->bio_list[1])))		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);	current->bio_list[1] = nopunt; 	spin_lock(&bs->rescue_lock);	bio_list_merge(&bs->rescue_list, &punt);	spin_unlock(&bs->rescue_lock); 	queue_work(bs->rescue_workqueue, &bs->rescue_work);} /** * bio_alloc_bioset - allocate a bio for I/O * @gfp_mask:   the GFP_ mask given to the slab allocator * @nr_iovecs:	number of iovecs to pre-allocate * @bs:		the bio_set to allocate from. * * Description: *   If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is *   backed by the @bs's mempool. * *   When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will *   always be able to allocate a bio. This is due to the mempool guarantees. *   To make this work, callers must never allocate more than 1 bio at a time *   from this pool. Callers that need to allocate more than 1 bio must always *   submit the previously allocated bio for IO before attempting to allocate *   a new one. Failure to do so can cause deadlocks under memory pressure. * *   Note that when running under generic_make_request() (i.e. any block *   driver), bios are not submitted until after you return - see the code in *   generic_make_request() that converts recursion into iteration, to prevent *   stack overflows. * *   This would normally mean allocating multiple bios under *   generic_make_request() would be susceptible to deadlocks, but we have *   deadlock avoidance code that resubmits any blocked bios from a rescuer *   thread. * *   However, we do not guarantee forward progress for allocations from other *   mempools. Doing multiple allocations from the same mempool under *   generic_make_request() should be avoided - instead, use bio_set's front_pad *   for per bio allocations. * *   RETURNS: *   Pointer to new bio on success, NULL on failure. */struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,			     struct bio_set *bs){	gfp_t saved_gfp = gfp_mask;	unsigned front_pad;	unsigned inline_vecs;	struct bio_vec *bvl = NULL;	struct bio *bio;	void *p; 	if (!bs) {		if (nr_iovecs > UIO_MAXIOV)			return NULL; 		p = kmalloc(sizeof(struct bio) +			    nr_iovecs * sizeof(struct bio_vec),			    gfp_mask);		front_pad = 0;		inline_vecs = nr_iovecs;	} else {		/* should not use nobvec bioset for nr_iovecs > 0 */		if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))			return NULL;		/*		 * generic_make_request() converts recursion to iteration; this		 * means if we're running beneath it, any bios we allocate and		 * submit will not be submitted (and thus freed) until after we		 * return.		 *		 * This exposes us to a potential deadlock if we allocate		 * multiple bios from the same bio_set() while running		 * underneath generic_make_request(). If we were to allocate		 * multiple bios (say a stacking block driver that was splitting		 * bios), we would deadlock if we exhausted the mempool's		 * reserve.		 *		 * We solve this, and guarantee forward progress, with a rescuer		 * workqueue per bio_set. If we go to allocate and there are		 * bios on current->bio_list, we first try the allocation		 * without __GFP_DIRECT_RECLAIM; if that fails, we punt those		 * bios we would be blocking to the rescuer workqueue before		 * we retry with the original gfp_flags.		 */ 		if (current->bio_list &&		    (!bio_list_empty(&current->bio_list[0]) ||		     !bio_list_empty(&current->bio_list[1])) &&		    bs->rescue_workqueue)			gfp_mask &= ~__GFP_DIRECT_RECLAIM; 		p = mempool_alloc(bs->bio_pool, gfp_mask);		if (!p && gfp_mask != saved_gfp) {			punt_bios_to_rescuer(bs);			gfp_mask = saved_gfp;			p = mempool_alloc(bs->bio_pool, gfp_mask);		} 		front_pad = bs->front_pad;		inline_vecs = BIO_INLINE_VECS;	} 	if (unlikely(!p))		return NULL; 	bio = p + front_pad;

Lines 1–500 of 2130. Load more