/* * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public Licens * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- * */#include <linux/mm.h>#include <linux/swap.h>#include <linux/bio.h>#include <linux/blkdev.h>#include <linux/iocontext.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/mempool.h>#include <linux/workqueue.h>#include <linux/cgroup.h>#include <scsi/sg.h> /* for struct sg_iovec */ #include <trace/events/block.h> /* * Test patch to inline a certain number of bi_io_vec's inside the bio * itself, to shrink a bio data allocation from two mempool calls to one */#define BIO_INLINE_VECS 4 static mempool_t *bio_split_pool __read_mostly; /* * if you change this list, also change bvec_alloc or things will * break badly! cannot be bigger than what you can fit into an * unsigned short */#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = { BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),};#undef BV /* * fs_bio_set is the bio_set containing bio and iovec memory pools used by * IO code that does not need private memory pools. */struct bio_set *fs_bio_set; /* * Our slab pool management */struct bio_slab { struct kmem_cache *slab; unsigned int slab_ref; unsigned int slab_size; char name[8];};static DEFINE_MUTEX(bio_slab_lock);static struct bio_slab *bio_slabs;static unsigned int bio_slab_nr, bio_slab_max; static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size){ unsigned int sz = sizeof(struct bio) + extra_size; struct kmem_cache *slab = NULL; struct bio_slab *bslab; unsigned int i, entry = -1; mutex_lock(&bio_slab_lock); i = 0; while (i < bio_slab_nr) { bslab = &bio_slabs[i]; if (!bslab->slab && entry == -1) entry = i; else if (bslab->slab_size == sz) { slab = bslab->slab; bslab->slab_ref++; break; } i++; } if (slab) goto out_unlock; if (bio_slab_nr == bio_slab_max && entry == -1) { bio_slab_max <<= 1; bio_slabs = krealloc(bio_slabs, bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL); if (!bio_slabs) goto out_unlock; } if (entry == -1) entry = bio_slab_nr++; bslab = &bio_slabs[entry]; snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry); slab = kmem_cache_create(bslab->name, sz, 0, SLAB_HWCACHE_ALIGN, NULL); if (!slab) goto out_unlock; printk(KERN_INFO "bio: create slab <%s> at %d\n", bslab->name, entry); bslab->slab = slab; bslab->slab_ref = 1; bslab->slab_size = sz;out_unlock: mutex_unlock(&bio_slab_lock); return slab;} static void bio_put_slab(struct bio_set *bs){ struct bio_slab *bslab = NULL; unsigned int i; mutex_lock(&bio_slab_lock); for (i = 0; i < bio_slab_nr; i++) { if (bs->bio_slab == bio_slabs[i].slab) { bslab = &bio_slabs[i]; break; } } if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n")) goto out; WARN_ON(!bslab->slab_ref); if (--bslab->slab_ref) goto out; kmem_cache_destroy(bslab->slab); bslab->slab = NULL; out: mutex_unlock(&bio_slab_lock);} unsigned int bvec_nr_vecs(unsigned short idx){ return bvec_slabs[idx].nr_vecs;} void bvec_free_bs(struct bio_set *bs, struct bio_vec *bv, unsigned int idx){ BIO_BUG_ON(idx >= BIOVEC_NR_POOLS); if (idx == BIOVEC_MAX_IDX) mempool_free(bv, bs->bvec_pool); else { struct biovec_slab *bvs = bvec_slabs + idx; kmem_cache_free(bvs->slab, bv); }} struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs){ struct bio_vec *bvl; /* * see comment near bvec_array define! */ switch (nr) { case 1: *idx = 0; break; case 2 ... 4: *idx = 1; break; case 5 ... 16: *idx = 2; break; case 17 ... 64: *idx = 3; break; case 65 ... 128: *idx = 4; break; case 129 ... BIO_MAX_PAGES: *idx = 5; break; default: return NULL; } /* * idx now points to the pool we want to allocate from. only the * 1-vec entry pool is mempool backed. */ if (*idx == BIOVEC_MAX_IDX) {fallback: bvl = mempool_alloc(bs->bvec_pool, gfp_mask); } else { struct biovec_slab *bvs = bvec_slabs + *idx; gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO); /* * Make this allocation restricted and don't dump info on * allocation failures, since we'll fallback to the mempool * in case of failure. */ __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; /* * Try a slab allocation. If this fails and __GFP_WAIT * is set, retry with the 1-entry mempool */ bvl = kmem_cache_alloc(bvs->slab, __gfp_mask); if (unlikely(!bvl && (gfp_mask & __GFP_WAIT))) { *idx = BIOVEC_MAX_IDX; goto fallback; } } return bvl;} void bio_free(struct bio *bio, struct bio_set *bs){ void *p; if (bio_has_allocated_vec(bio)) bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio)); if (bio_integrity(bio)) bio_integrity_free(bio, bs); /* * If we have front padding, adjust the bio pointer before freeing */ p = bio; if (bs->front_pad) p -= bs->front_pad; mempool_free(p, bs->bio_pool);}EXPORT_SYMBOL(bio_free); void bio_init(struct bio *bio){ memset(bio, 0, sizeof(*bio)); bio->bi_flags = 1 << BIO_UPTODATE; atomic_set(&bio->bi_cnt, 1);}EXPORT_SYMBOL(bio_init); /** * bio_alloc_bioset - allocate a bio for I/O * @gfp_mask: the GFP_ mask given to the slab allocator * @nr_iovecs: number of iovecs to pre-allocate * @bs: the bio_set to allocate from. * * Description: * bio_alloc_bioset will try its own mempool to satisfy the allocation. * If %__GFP_WAIT is set then we will block on the internal pool waiting * for a &struct bio to become free. * * Note that the caller must set ->bi_destructor on successful return * of a bio, to do the appropriate freeing of the bio once the reference * count drops to zero. **/struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs){ unsigned long idx = BIO_POOL_NONE; struct bio_vec *bvl = NULL; struct bio *bio; void *p; p = mempool_alloc(bs->bio_pool, gfp_mask); if (unlikely(!p)) return NULL; bio = p + bs->front_pad; bio_init(bio); if (unlikely(!nr_iovecs)) goto out_set; if (nr_iovecs <= BIO_INLINE_VECS) { bvl = bio->bi_inline_vecs; nr_iovecs = BIO_INLINE_VECS; } else { bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs); if (unlikely(!bvl)) goto err_free; nr_iovecs = bvec_nr_vecs(idx); }out_set: bio->bi_flags |= idx << BIO_POOL_OFFSET; bio->bi_max_vecs = nr_iovecs; bio->bi_io_vec = bvl; return bio; err_free: mempool_free(p, bs->bio_pool); return NULL;}EXPORT_SYMBOL(bio_alloc_bioset); static void bio_fs_destructor(struct bio *bio){ bio_free(bio, fs_bio_set);} /** * bio_alloc - allocate a new bio, memory pool backed * @gfp_mask: allocation mask to use * @nr_iovecs: number of iovecs * * bio_alloc will allocate a bio and associated bio_vec array that can hold * at least @nr_iovecs entries. Allocations will be done from the * fs_bio_set. Also see @bio_alloc_bioset and @bio_kmalloc. * * If %__GFP_WAIT is set, then bio_alloc will always be able to allocate * a bio. This is due to the mempool guarantees. To make this work, callers * must never allocate more than 1 bio at a time from this pool. Callers * that need to allocate more than 1 bio must always submit the previously * allocated bio for IO before attempting to allocate a new one. Failure to * do so can cause livelocks under memory pressure. * * RETURNS: * Pointer to new bio on success, NULL on failure. */struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs){ struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set); if (bio) bio->bi_destructor = bio_fs_destructor; return bio;}EXPORT_SYMBOL(bio_alloc); static void bio_kmalloc_destructor(struct bio *bio){ if (bio_integrity(bio)) bio_integrity_free(bio, fs_bio_set); kfree(bio);} /** * bio_kmalloc - allocate a bio for I/O using kmalloc() * @gfp_mask: the GFP_ mask given to the slab allocator * @nr_iovecs: number of iovecs to pre-allocate * * Description: * Allocate a new bio with @nr_iovecs bvecs. If @gfp_mask contains * %__GFP_WAIT, the allocation is guaranteed to succeed. * **/struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs){ struct bio *bio; if (nr_iovecs > UIO_MAXIOV) return NULL; bio = kmalloc(sizeof(struct bio) + nr_iovecs * sizeof(struct bio_vec), gfp_mask); if (unlikely(!bio)) return NULL; bio_init(bio); bio->bi_flags |= BIO_POOL_NONE << BIO_POOL_OFFSET; bio->bi_max_vecs = nr_iovecs; bio->bi_io_vec = bio->bi_inline_vecs; bio->bi_destructor = bio_kmalloc_destructor; return bio;}EXPORT_SYMBOL(bio_kmalloc); void zero_fill_bio(struct bio *bio){ unsigned long flags; struct bio_vec *bv; int i; bio_for_each_segment(bv, bio, i) { char *data = bvec_kmap_irq(bv, &flags); memset(data, 0, bv->bv_len); flush_dcache_page(bv->bv_page); bvec_kunmap_irq(data, &flags); }}EXPORT_SYMBOL(zero_fill_bio); /** * bio_put - release a reference to a bio * @bio: bio to release reference to * * Description: * Put a reference to a &struct bio, either one you have gotten with * bio_alloc, bio_get or bio_clone. The last put of a bio will free it. **/void bio_put(struct bio *bio){ BIO_BUG_ON(!atomic_read(&bio->bi_cnt)); /* * last put frees it */ if (atomic_dec_and_test(&bio->bi_cnt)) { bio_disassociate_task(bio); bio->bi_next = NULL; bio->bi_destructor(bio); }}EXPORT_SYMBOL(bio_put); inline int bio_phys_segments(struct request_queue *q, struct bio *bio){ if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) blk_recount_segments(q, bio); return bio->bi_phys_segments;}EXPORT_SYMBOL(bio_phys_segments); /** * __bio_clone - clone a bio * @bio: destination bio * @bio_src: bio to clone * * Clone a &bio. Caller will own the returned bio, but not * the actual data it points to. Reference count of returned * bio will be one. */void __bio_clone(struct bio *bio, struct bio *bio_src){ memcpy(bio->bi_io_vec, bio_src->bi_io_vec, bio_src->bi_max_vecs * sizeof(struct bio_vec)); /* * most users will be overriding ->bi_bdev with a new target, * so we don't set nor calculate new physical/hw segment counts here */ bio->bi_sector = bio_src->bi_sector; bio->bi_bdev = bio_src->bi_bdev; bio->bi_flags |= 1 << BIO_CLONED; bio->bi_rw = bio_src->bi_rw; bio->bi_vcnt = bio_src->bi_vcnt; bio->bi_size = bio_src->bi_size; bio->bi_idx = bio_src->bi_idx;}EXPORT_SYMBOL(__bio_clone); /** * bio_clone - clone a bio * @bio: bio to clone * @gfp_mask: allocation priority * * Like __bio_clone, only also allocates the returned bio */struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask){ struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set); if (!b) return NULL; b->bi_destructor = bio_fs_destructor; __bio_clone(b, bio); if (bio_integrity(bio)) { int ret; ret = bio_integrity_clone(b, bio, gfp_mask, fs_bio_set); if (ret < 0) { bio_put(b); return NULL; } } return b;}EXPORT_SYMBOL(bio_clone); /** * bio_get_nr_vecs - return approx number of vecsLines 1–500 of 1754. Load more
/* * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public Licens * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- * */ #include <linux/mm.h> #include <linux/swap.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/iocontext.h> #include <linux/slab.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/export.h> #include <linux/mempool.h> #include <linux/workqueue.h> #include <linux/cgroup.h> #include <scsi/sg.h> /* for struct sg_iovec */ #include <trace/events/block.h> /* * Test patch to inline a certain number of bi_io_vec's inside the bio * itself, to shrink a bio data allocation from two mempool calls to one */ #define BIO_INLINE_VECS 4 static mempool_t *bio_split_pool __read_mostly; /* * if you change this list, also change bvec_alloc or things will * break badly! cannot be bigger than what you can fit into an * unsigned short */ #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) } static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = { BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES), }; #undef BV /* * fs_bio_set is the bio_set containing bio and iovec memory pools used by * IO code that does not need private memory pools. */ struct bio_set *fs_bio_set; /* * Our slab pool management */ struct bio_slab { struct kmem_cache *slab; unsigned int slab_ref; unsigned int slab_size; char name[8]; }; static DEFINE_MUTEX(bio_slab_lock); static struct bio_slab *bio_slabs; static unsigned int bio_slab_nr, bio_slab_max; static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size) { unsigned int sz = sizeof(struct bio) + extra_size; struct kmem_cache *slab = NULL; struct bio_slab *bslab; unsigned int i, entry = -1; mutex_lock(&bio_slab_lock); i = 0; while (i < bio_slab_nr) { bslab = &bio_slabs[i]; if (!bslab->slab && entry == -1) entry = i; else if (bslab->slab_size == sz) { slab = bslab->slab; bslab->slab_ref++; break; } i++; } if (slab) goto out_unlock; if (bio_slab_nr == bio_slab_max && entry == -1) { bio_slab_max <<= 1; bio_slabs = krealloc(bio_slabs, bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL); if (!bio_slabs) goto out_unlock; } if (entry == -1) entry = bio_slab_nr++; bslab = &bio_slabs[entry]; snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry); slab = kmem_cache_create(bslab->name, sz, 0, SLAB_HWCACHE_ALIGN, NULL); if (!slab) goto out_unlock; printk(KERN_INFO "bio: create slab <%s> at %d\n", bslab->name, entry); bslab->slab = slab; bslab->slab_ref = 1; bslab->slab_size = sz; out_unlock: mutex_unlock(&bio_slab_lock); return slab; } static void bio_put_slab(struct bio_set *bs) { struct bio_slab *bslab = NULL; unsigned int i; mutex_lock(&bio_slab_lock); for (i = 0; i < bio_slab_nr; i++) { if (bs->bio_slab == bio_slabs[i].slab) { bslab = &bio_slabs[i]; break; } } if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n")) goto out; WARN_ON(!bslab->slab_ref); if (--bslab->slab_ref) goto out; kmem_cache_destroy(bslab->slab); bslab->slab = NULL; out: mutex_unlock(&bio_slab_lock); } unsigned int bvec_nr_vecs(unsigned short idx) { return bvec_slabs[idx].nr_vecs; } void bvec_free_bs(struct bio_set *bs, struct bio_vec *bv, unsigned int idx) { BIO_BUG_ON(idx >= BIOVEC_NR_POOLS); if (idx == BIOVEC_MAX_IDX) mempool_free(bv, bs->bvec_pool); else { struct biovec_slab *bvs = bvec_slabs + idx; kmem_cache_free(bvs->slab, bv); } } struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs) { struct bio_vec *bvl; /* * see comment near bvec_array define! */ switch (nr) { case 1: *idx = 0; break; case 2 ... 4: *idx = 1; break; case 5 ... 16: *idx = 2; break; case 17 ... 64: *idx = 3; break; case 65 ... 128: *idx = 4; break; case 129 ... BIO_MAX_PAGES: *idx = 5; break; default: return NULL; } /* * idx now points to the pool we want to allocate from. only the * 1-vec entry pool is mempool backed. */ if (*idx == BIOVEC_MAX_IDX) { fallback: bvl = mempool_alloc(bs->bvec_pool, gfp_mask); } else { struct biovec_slab *bvs = bvec_slabs + *idx; gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO); /* * Make this allocation restricted and don't dump info on * allocation failures, since we'll fallback to the mempool * in case of failure. */ __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; /* * Try a slab allocation. If this fails and __GFP_WAIT * is set, retry with the 1-entry mempool */ bvl = kmem_cache_alloc(bvs->slab, __gfp_mask); if (unlikely(!bvl && (gfp_mask & __GFP_WAIT))) { *idx = BIOVEC_MAX_IDX; goto fallback; } } return bvl; } void bio_free(struct bio *bio, struct bio_set *bs) { void *p; if (bio_has_allocated_vec(bio)) bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio)); if (bio_integrity(bio)) bio_integrity_free(bio, bs); /* * If we have front padding, adjust the bio pointer before freeing */ p = bio; if (bs->front_pad) p -= bs->front_pad; mempool_free(p, bs->bio_pool); } EXPORT_SYMBOL(bio_free); void bio_init(struct bio *bio) { memset(bio, 0, sizeof(*bio)); bio->bi_flags = 1 << BIO_UPTODATE; atomic_set(&bio->bi_cnt, 1); } EXPORT_SYMBOL(bio_init); /** * bio_alloc_bioset - allocate a bio for I/O * @gfp_mask: the GFP_ mask given to the slab allocator * @nr_iovecs: number of iovecs to pre-allocate * @bs: the bio_set to allocate from. * * Description: * bio_alloc_bioset will try its own mempool to satisfy the allocation. * If %__GFP_WAIT is set then we will block on the internal pool waiting * for a &struct bio to become free. * * Note that the caller must set ->bi_destructor on successful return * of a bio, to do the appropriate freeing of the bio once the reference * count drops to zero. **/ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) { unsigned long idx = BIO_POOL_NONE; struct bio_vec *bvl = NULL; struct bio *bio; void *p; p = mempool_alloc(bs->bio_pool, gfp_mask); if (unlikely(!p)) return NULL; bio = p + bs->front_pad; bio_init(bio); if (unlikely(!nr_iovecs)) goto out_set; if (nr_iovecs <= BIO_INLINE_VECS) { bvl = bio->bi_inline_vecs; nr_iovecs = BIO_INLINE_VECS; } else { bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs); if (unlikely(!bvl)) goto err_free; nr_iovecs = bvec_nr_vecs(idx); } out_set: bio->bi_flags |= idx << BIO_POOL_OFFSET; bio->bi_max_vecs = nr_iovecs; bio->bi_io_vec = bvl; return bio; err_free: mempool_free(p, bs->bio_pool); return NULL; } EXPORT_SYMBOL(bio_alloc_bioset); static void bio_fs_destructor(struct bio *bio) { bio_free(bio, fs_bio_set); } /** * bio_alloc - allocate a new bio, memory pool backed * @gfp_mask: allocation mask to use * @nr_iovecs: number of iovecs * * bio_alloc will allocate a bio and associated bio_vec array that can hold * at least @nr_iovecs entries. Allocations will be done from the * fs_bio_set. Also see @bio_alloc_bioset and @bio_kmalloc. * * If %__GFP_WAIT is set, then bio_alloc will always be able to allocate * a bio. This is due to the mempool guarantees. To make this work, callers * must never allocate more than 1 bio at a time from this pool. Callers * that need to allocate more than 1 bio must always submit the previously * allocated bio for IO before attempting to allocate a new one. Failure to * do so can cause livelocks under memory pressure. * * RETURNS: * Pointer to new bio on success, NULL on failure. */ struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) { struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set); if (bio) bio->bi_destructor = bio_fs_destructor; return bio; } EXPORT_SYMBOL(bio_alloc); static void bio_kmalloc_destructor(struct bio *bio) { if (bio_integrity(bio)) bio_integrity_free(bio, fs_bio_set); kfree(bio); } /** * bio_kmalloc - allocate a bio for I/O using kmalloc() * @gfp_mask: the GFP_ mask given to the slab allocator * @nr_iovecs: number of iovecs to pre-allocate * * Description: * Allocate a new bio with @nr_iovecs bvecs. If @gfp_mask contains * %__GFP_WAIT, the allocation is guaranteed to succeed. * **/ struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs) { struct bio *bio; if (nr_iovecs > UIO_MAXIOV) return NULL; bio = kmalloc(sizeof(struct bio) + nr_iovecs * sizeof(struct bio_vec), gfp_mask); if (unlikely(!bio)) return NULL; bio_init(bio); bio->bi_flags |= BIO_POOL_NONE << BIO_POOL_OFFSET; bio->bi_max_vecs = nr_iovecs; bio->bi_io_vec = bio->bi_inline_vecs; bio->bi_destructor = bio_kmalloc_destructor; return bio; } EXPORT_SYMBOL(bio_kmalloc); void zero_fill_bio(struct bio *bio) { unsigned long flags; struct bio_vec *bv; int i; bio_for_each_segment(bv, bio, i) { char *data = bvec_kmap_irq(bv, &flags); memset(data, 0, bv->bv_len); flush_dcache_page(bv->bv_page); bvec_kunmap_irq(data, &flags); } } EXPORT_SYMBOL(zero_fill_bio); /** * bio_put - release a reference to a bio * @bio: bio to release reference to * * Description: * Put a reference to a &struct bio, either one you have gotten with * bio_alloc, bio_get or bio_clone. The last put of a bio will free it. **/ void bio_put(struct bio *bio) { BIO_BUG_ON(!atomic_read(&bio->bi_cnt)); /* * last put frees it */ if (atomic_dec_and_test(&bio->bi_cnt)) { bio_disassociate_task(bio); bio->bi_next = NULL; bio->bi_destructor(bio); } } EXPORT_SYMBOL(bio_put); inline int bio_phys_segments(struct request_queue *q, struct bio *bio) { if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) blk_recount_segments(q, bio); return bio->bi_phys_segments; } EXPORT_SYMBOL(bio_phys_segments); /** * __bio_clone - clone a bio * @bio: destination bio * @bio_src: bio to clone * * Clone a &bio. Caller will own the returned bio, but not * the actual data it points to. Reference count of returned * bio will be one. */ void __bio_clone(struct bio *bio, struct bio *bio_src) { memcpy(bio->bi_io_vec, bio_src->bi_io_vec, bio_src->bi_max_vecs * sizeof(struct bio_vec)); /* * most users will be overriding ->bi_bdev with a new target, * so we don't set nor calculate new physical/hw segment counts here */ bio->bi_sector = bio_src->bi_sector; bio->bi_bdev = bio_src->bi_bdev; bio->bi_flags |= 1 << BIO_CLONED; bio->bi_rw = bio_src->bi_rw; bio->bi_vcnt = bio_src->bi_vcnt; bio->bi_size = bio_src->bi_size; bio->bi_idx = bio_src->bi_idx; } EXPORT_SYMBOL(__bio_clone); /** * bio_clone - clone a bio * @bio: bio to clone * @gfp_mask: allocation priority * * Like __bio_clone, only also allocates the returned bio */ struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask) { struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set); if (!b) return NULL; b->bi_destructor = bio_fs_destructor; __bio_clone(b, bio); if (bio_integrity(bio)) { int ret; ret = bio_integrity_clone(b, bio, gfp_mask, fs_bio_set); if (ret < 0) { bio_put(b); return NULL; } } return b; } EXPORT_SYMBOL(bio_clone); /** * bio_get_nr_vecs - return approx number of vecs