/* * Copyright (C) 2001 Jens Axboe <axboe@suse.de> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public Licens * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- * */#include <linux/mm.h>#include <linux/swap.h>#include <linux/bio.h>#include <linux/blkdev.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/kernel.h>#include <linux/module.h>#include <linux/mempool.h>#include <linux/workqueue.h>#include <linux/blktrace_api.h>#include <scsi/sg.h> /* for struct sg_iovec */ #define BIO_POOL_SIZE 256 static kmem_cache_t *bio_slab __read_mostly; #define BIOVEC_NR_POOLS 6 /* * a small number of entries is fine, not going to be performance critical. * basically we just need to survive */#define BIO_SPLIT_ENTRIES 8 mempool_t *bio_split_pool __read_mostly; struct biovec_slab { int nr_vecs; char *name; kmem_cache_t *slab;}; /* * if you change this list, also change bvec_alloc or things will * break badly! cannot be bigger than what you can fit into an * unsigned short */ #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = { BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),};#undef BV /* * bio_set is used to allow other portions of the IO system to * allocate their own private memory pools for bio and iovec structures. * These memory pools in turn all allocate from the bio_slab * and the bvec_slabs[]. */struct bio_set { mempool_t *bio_pool; mempool_t *bvec_pools[BIOVEC_NR_POOLS];}; /* * fs_bio_set is the bio_set containing bio and iovec memory pools used by * IO code that does not need private memory pools. */static struct bio_set *fs_bio_set; static inline struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs){ struct bio_vec *bvl; struct biovec_slab *bp; /* * see comment near bvec_array define! */ switch (nr) { case 1 : *idx = 0; break; case 2 ... 4: *idx = 1; break; case 5 ... 16: *idx = 2; break; case 17 ... 64: *idx = 3; break; case 65 ... 128: *idx = 4; break; case 129 ... BIO_MAX_PAGES: *idx = 5; break; default: return NULL; } /* * idx now points to the pool we want to allocate from */ bp = bvec_slabs + *idx; bvl = mempool_alloc(bs->bvec_pools[*idx], gfp_mask); if (bvl) memset(bvl, 0, bp->nr_vecs * sizeof(struct bio_vec)); return bvl;} void bio_free(struct bio *bio, struct bio_set *bio_set){ const int pool_idx = BIO_POOL_IDX(bio); BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS); mempool_free(bio->bi_io_vec, bio_set->bvec_pools[pool_idx]); mempool_free(bio, bio_set->bio_pool);} /* * default destructor for a bio allocated with bio_alloc_bioset() */static void bio_fs_destructor(struct bio *bio){ bio_free(bio, fs_bio_set);} void bio_init(struct bio *bio){ bio->bi_next = NULL; bio->bi_bdev = NULL; bio->bi_flags = 1 << BIO_UPTODATE; bio->bi_rw = 0; bio->bi_vcnt = 0; bio->bi_idx = 0; bio->bi_phys_segments = 0; bio->bi_hw_segments = 0; bio->bi_hw_front_size = 0; bio->bi_hw_back_size = 0; bio->bi_size = 0; bio->bi_max_vecs = 0; bio->bi_end_io = NULL; atomic_set(&bio->bi_cnt, 1); bio->bi_private = NULL;} /** * bio_alloc_bioset - allocate a bio for I/O * @gfp_mask: the GFP_ mask given to the slab allocator * @nr_iovecs: number of iovecs to pre-allocate * @bs: the bio_set to allocate from * * Description: * bio_alloc_bioset will first try it's on mempool to satisfy the allocation. * If %__GFP_WAIT is set then we will block on the internal pool waiting * for a &struct bio to become free. * * allocate bio and iovecs from the memory pools specified by the * bio_set structure. **/struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs){ struct bio *bio = mempool_alloc(bs->bio_pool, gfp_mask); if (likely(bio)) { struct bio_vec *bvl = NULL; bio_init(bio); if (likely(nr_iovecs)) { unsigned long idx; bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs); if (unlikely(!bvl)) { mempool_free(bio, bs->bio_pool); bio = NULL; goto out; } bio->bi_flags |= idx << BIO_POOL_OFFSET; bio->bi_max_vecs = bvec_slabs[idx].nr_vecs; } bio->bi_io_vec = bvl; }out: return bio;} struct bio *bio_alloc(gfp_t gfp_mask, int nr_iovecs){ struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set); if (bio) bio->bi_destructor = bio_fs_destructor; return bio;} void zero_fill_bio(struct bio *bio){ unsigned long flags; struct bio_vec *bv; int i; bio_for_each_segment(bv, bio, i) { char *data = bvec_kmap_irq(bv, &flags); memset(data, 0, bv->bv_len); flush_dcache_page(bv->bv_page); bvec_kunmap_irq(data, &flags); }}EXPORT_SYMBOL(zero_fill_bio); /** * bio_put - release a reference to a bio * @bio: bio to release reference to * * Description: * Put a reference to a &struct bio, either one you have gotten with * bio_alloc or bio_get. The last put of a bio will free it. **/void bio_put(struct bio *bio){ BIO_BUG_ON(!atomic_read(&bio->bi_cnt)); /* * last put frees it */ if (atomic_dec_and_test(&bio->bi_cnt)) { bio->bi_next = NULL; bio->bi_destructor(bio); }} inline int bio_phys_segments(request_queue_t *q, struct bio *bio){ if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) blk_recount_segments(q, bio); return bio->bi_phys_segments;} inline int bio_hw_segments(request_queue_t *q, struct bio *bio){ if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) blk_recount_segments(q, bio); return bio->bi_hw_segments;} /** * __bio_clone - clone a bio * @bio: destination bio * @bio_src: bio to clone * * Clone a &bio. Caller will own the returned bio, but not * the actual data it points to. Reference count of returned * bio will be one. */void __bio_clone(struct bio *bio, struct bio *bio_src){ request_queue_t *q = bdev_get_queue(bio_src->bi_bdev); memcpy(bio->bi_io_vec, bio_src->bi_io_vec, bio_src->bi_max_vecs * sizeof(struct bio_vec)); bio->bi_sector = bio_src->bi_sector; bio->bi_bdev = bio_src->bi_bdev; bio->bi_flags |= 1 << BIO_CLONED; bio->bi_rw = bio_src->bi_rw; bio->bi_vcnt = bio_src->bi_vcnt; bio->bi_size = bio_src->bi_size; bio->bi_idx = bio_src->bi_idx; bio_phys_segments(q, bio); bio_hw_segments(q, bio);} /** * bio_clone - clone a bio * @bio: bio to clone * @gfp_mask: allocation priority * * Like __bio_clone, only also allocates the returned bio */struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask){ struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set); if (b) { b->bi_destructor = bio_fs_destructor; __bio_clone(b, bio); } return b;} /** * bio_get_nr_vecs - return approx number of vecs * @bdev: I/O target * * Return the approximate number of pages we can send to this target. * There's no guarantee that you will be able to fit this number of pages * into a bio, it does not account for dynamic restrictions that vary * on offset. */int bio_get_nr_vecs(struct block_device *bdev){ request_queue_t *q = bdev_get_queue(bdev); int nr_pages; nr_pages = ((q->max_sectors << 9) + PAGE_SIZE - 1) >> PAGE_SHIFT; if (nr_pages > q->max_phys_segments) nr_pages = q->max_phys_segments; if (nr_pages > q->max_hw_segments) nr_pages = q->max_hw_segments; return nr_pages;} static int __bio_add_page(request_queue_t *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, unsigned short max_sectors){ int retried_segments = 0; struct bio_vec *bvec; /* * cloned bio must not modify vec list */ if (unlikely(bio_flagged(bio, BIO_CLONED))) return 0; if (((bio->bi_size + len) >> 9) > max_sectors) return 0; /* * For filesystems with a blocksize smaller than the pagesize * we will often be called with the same page as last time and * a consecutive offset. Optimize this special case. */ if (bio->bi_vcnt > 0) { struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1]; if (page == prev->bv_page && offset == prev->bv_offset + prev->bv_len) { prev->bv_len += len; if (q->merge_bvec_fn && q->merge_bvec_fn(q, bio, prev) < len) { prev->bv_len -= len; return 0; } goto done; } } if (bio->bi_vcnt >= bio->bi_max_vecs) return 0; /* * we might lose a segment or two here, but rather that than * make this too complex. */ while (bio->bi_phys_segments >= q->max_phys_segments || bio->bi_hw_segments >= q->max_hw_segments || BIOVEC_VIRT_OVERSIZE(bio->bi_size)) { if (retried_segments) return 0; retried_segments = 1; blk_recount_segments(q, bio); } /* * setup the new entry, we might clear it again later if we * cannot add the page */ bvec = &bio->bi_io_vec[bio->bi_vcnt]; bvec->bv_page = page; bvec->bv_len = len; bvec->bv_offset = offset; /* * if queue has other restrictions (eg varying max sector size * depending on offset), it can specify a merge_bvec_fn in the * queue to get further control */ if (q->merge_bvec_fn) { /* * merge_bvec_fn() returns number of bytes it can accept * at this offset */ if (q->merge_bvec_fn(q, bio, bvec) < len) { bvec->bv_page = NULL; bvec->bv_len = 0; bvec->bv_offset = 0; return 0; } } /* If we may be able to merge these biovecs, force a recount */ if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec) || BIOVEC_VIRT_MERGEABLE(bvec-1, bvec))) bio->bi_flags &= ~(1 << BIO_SEG_VALID); bio->bi_vcnt++; bio->bi_phys_segments++; bio->bi_hw_segments++; done: bio->bi_size += len; return len;} /** * bio_add_pc_page - attempt to add page to bio * @q: the target queue * @bio: destination bio * @page: page to add * @len: vec entry length * @offset: vec entry offset * * Attempt to add a page to the bio_vec maplist. This can fail for a * number of reasons, such as the bio being full or target block * device limitations. The target block device must allow bio's * smaller than PAGE_SIZE, so it is always possible to add a single * page to an empty bio. This should only be used by REQ_PC bios. */int bio_add_pc_page(request_queue_t *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset){ return __bio_add_page(q, bio, page, len, offset, q->max_hw_sectors);} /** * bio_add_page - attempt to add page to bio * @bio: destination bio * @page: page to add * @len: vec entry length * @offset: vec entry offset * * Attempt to add a page to the bio_vec maplist. This can fail for a * number of reasons, such as the bio being full or target block * device limitations. The target block device must allow bio's * smaller than PAGE_SIZE, so it is always possible to add a single * page to an empty bio. */int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset){ struct request_queue *q = bdev_get_queue(bio->bi_bdev); return __bio_add_page(q, bio, page, len, offset, q->max_sectors);} struct bio_map_data { struct bio_vec *iovecs; void __user *userptr;}; static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio){ memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt); bio->bi_private = bmd;} static void bio_free_map_data(struct bio_map_data *bmd){ kfree(bmd->iovecs); kfree(bmd);} static struct bio_map_data *bio_alloc_map_data(int nr_segs){ struct bio_map_data *bmd = kmalloc(sizeof(*bmd), GFP_KERNEL); if (!bmd) return NULL; bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, GFP_KERNEL); if (bmd->iovecs) return bmd; kfree(bmd); return NULL;} /** * bio_uncopy_user - finish previously mapped bio * @bio: bio being terminated * * Free pages allocated from bio_copy_user() and write back data * to user space in case of a read. */int bio_uncopy_user(struct bio *bio){ struct bio_map_data *bmd = bio->bi_private; const int read = bio_data_dir(bio) == READ; struct bio_vec *bvec; int i, ret = 0; __bio_for_each_segment(bvec, bio, i, 0) { char *addr = page_address(bvec->bv_page);Lines 1–500 of 1280. Load more
/* * Copyright (C) 2001 Jens Axboe <axboe@suse.de> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public Licens * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- * */ #include <linux/mm.h> #include <linux/swap.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/slab.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/mempool.h> #include <linux/workqueue.h> #include <linux/blktrace_api.h> #include <scsi/sg.h> /* for struct sg_iovec */ #define BIO_POOL_SIZE 256 static kmem_cache_t *bio_slab __read_mostly; #define BIOVEC_NR_POOLS 6 /* * a small number of entries is fine, not going to be performance critical. * basically we just need to survive */ #define BIO_SPLIT_ENTRIES 8 mempool_t *bio_split_pool __read_mostly; struct biovec_slab { int nr_vecs; char *name; kmem_cache_t *slab; }; /* * if you change this list, also change bvec_alloc or things will * break badly! cannot be bigger than what you can fit into an * unsigned short */ #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) } static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = { BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES), }; #undef BV /* * bio_set is used to allow other portions of the IO system to * allocate their own private memory pools for bio and iovec structures. * These memory pools in turn all allocate from the bio_slab * and the bvec_slabs[]. */ struct bio_set { mempool_t *bio_pool; mempool_t *bvec_pools[BIOVEC_NR_POOLS]; }; /* * fs_bio_set is the bio_set containing bio and iovec memory pools used by * IO code that does not need private memory pools. */ static struct bio_set *fs_bio_set; static inline struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs) { struct bio_vec *bvl; struct biovec_slab *bp; /* * see comment near bvec_array define! */ switch (nr) { case 1 : *idx = 0; break; case 2 ... 4: *idx = 1; break; case 5 ... 16: *idx = 2; break; case 17 ... 64: *idx = 3; break; case 65 ... 128: *idx = 4; break; case 129 ... BIO_MAX_PAGES: *idx = 5; break; default: return NULL; } /* * idx now points to the pool we want to allocate from */ bp = bvec_slabs + *idx; bvl = mempool_alloc(bs->bvec_pools[*idx], gfp_mask); if (bvl) memset(bvl, 0, bp->nr_vecs * sizeof(struct bio_vec)); return bvl; } void bio_free(struct bio *bio, struct bio_set *bio_set) { const int pool_idx = BIO_POOL_IDX(bio); BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS); mempool_free(bio->bi_io_vec, bio_set->bvec_pools[pool_idx]); mempool_free(bio, bio_set->bio_pool); } /* * default destructor for a bio allocated with bio_alloc_bioset() */ static void bio_fs_destructor(struct bio *bio) { bio_free(bio, fs_bio_set); } void bio_init(struct bio *bio) { bio->bi_next = NULL; bio->bi_bdev = NULL; bio->bi_flags = 1 << BIO_UPTODATE; bio->bi_rw = 0; bio->bi_vcnt = 0; bio->bi_idx = 0; bio->bi_phys_segments = 0; bio->bi_hw_segments = 0; bio->bi_hw_front_size = 0; bio->bi_hw_back_size = 0; bio->bi_size = 0; bio->bi_max_vecs = 0; bio->bi_end_io = NULL; atomic_set(&bio->bi_cnt, 1); bio->bi_private = NULL; } /** * bio_alloc_bioset - allocate a bio for I/O * @gfp_mask: the GFP_ mask given to the slab allocator * @nr_iovecs: number of iovecs to pre-allocate * @bs: the bio_set to allocate from * * Description: * bio_alloc_bioset will first try it's on mempool to satisfy the allocation. * If %__GFP_WAIT is set then we will block on the internal pool waiting * for a &struct bio to become free. * * allocate bio and iovecs from the memory pools specified by the * bio_set structure. **/ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) { struct bio *bio = mempool_alloc(bs->bio_pool, gfp_mask); if (likely(bio)) { struct bio_vec *bvl = NULL; bio_init(bio); if (likely(nr_iovecs)) { unsigned long idx; bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs); if (unlikely(!bvl)) { mempool_free(bio, bs->bio_pool); bio = NULL; goto out; } bio->bi_flags |= idx << BIO_POOL_OFFSET; bio->bi_max_vecs = bvec_slabs[idx].nr_vecs; } bio->bi_io_vec = bvl; } out: return bio; } struct bio *bio_alloc(gfp_t gfp_mask, int nr_iovecs) { struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set); if (bio) bio->bi_destructor = bio_fs_destructor; return bio; } void zero_fill_bio(struct bio *bio) { unsigned long flags; struct bio_vec *bv; int i; bio_for_each_segment(bv, bio, i) { char *data = bvec_kmap_irq(bv, &flags); memset(data, 0, bv->bv_len); flush_dcache_page(bv->bv_page); bvec_kunmap_irq(data, &flags); } } EXPORT_SYMBOL(zero_fill_bio); /** * bio_put - release a reference to a bio * @bio: bio to release reference to * * Description: * Put a reference to a &struct bio, either one you have gotten with * bio_alloc or bio_get. The last put of a bio will free it. **/ void bio_put(struct bio *bio) { BIO_BUG_ON(!atomic_read(&bio->bi_cnt)); /* * last put frees it */ if (atomic_dec_and_test(&bio->bi_cnt)) { bio->bi_next = NULL; bio->bi_destructor(bio); } } inline int bio_phys_segments(request_queue_t *q, struct bio *bio) { if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) blk_recount_segments(q, bio); return bio->bi_phys_segments; } inline int bio_hw_segments(request_queue_t *q, struct bio *bio) { if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) blk_recount_segments(q, bio); return bio->bi_hw_segments; } /** * __bio_clone - clone a bio * @bio: destination bio * @bio_src: bio to clone * * Clone a &bio. Caller will own the returned bio, but not * the actual data it points to. Reference count of returned * bio will be one. */ void __bio_clone(struct bio *bio, struct bio *bio_src) { request_queue_t *q = bdev_get_queue(bio_src->bi_bdev); memcpy(bio->bi_io_vec, bio_src->bi_io_vec, bio_src->bi_max_vecs * sizeof(struct bio_vec)); bio->bi_sector = bio_src->bi_sector; bio->bi_bdev = bio_src->bi_bdev; bio->bi_flags |= 1 << BIO_CLONED; bio->bi_rw = bio_src->bi_rw; bio->bi_vcnt = bio_src->bi_vcnt; bio->bi_size = bio_src->bi_size; bio->bi_idx = bio_src->bi_idx; bio_phys_segments(q, bio); bio_hw_segments(q, bio); } /** * bio_clone - clone a bio * @bio: bio to clone * @gfp_mask: allocation priority * * Like __bio_clone, only also allocates the returned bio */ struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask) { struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set); if (b) { b->bi_destructor = bio_fs_destructor; __bio_clone(b, bio); } return b; } /** * bio_get_nr_vecs - return approx number of vecs * @bdev: I/O target * * Return the approximate number of pages we can send to this target. * There's no guarantee that you will be able to fit this number of pages * into a bio, it does not account for dynamic restrictions that vary * on offset. */ int bio_get_nr_vecs(struct block_device *bdev) { request_queue_t *q = bdev_get_queue(bdev); int nr_pages; nr_pages = ((q->max_sectors << 9) + PAGE_SIZE - 1) >> PAGE_SHIFT; if (nr_pages > q->max_phys_segments) nr_pages = q->max_phys_segments; if (nr_pages > q->max_hw_segments) nr_pages = q->max_hw_segments; return nr_pages; } static int __bio_add_page(request_queue_t *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, unsigned short max_sectors) { int retried_segments = 0; struct bio_vec *bvec; /* * cloned bio must not modify vec list */ if (unlikely(bio_flagged(bio, BIO_CLONED))) return 0; if (((bio->bi_size + len) >> 9) > max_sectors) return 0; /* * For filesystems with a blocksize smaller than the pagesize * we will often be called with the same page as last time and * a consecutive offset. Optimize this special case. */ if (bio->bi_vcnt > 0) { struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1]; if (page == prev->bv_page && offset == prev->bv_offset + prev->bv_len) { prev->bv_len += len; if (q->merge_bvec_fn && q->merge_bvec_fn(q, bio, prev) < len) { prev->bv_len -= len; return 0; } goto done; } } if (bio->bi_vcnt >= bio->bi_max_vecs) return 0; /* * we might lose a segment or two here, but rather that than * make this too complex. */ while (bio->bi_phys_segments >= q->max_phys_segments || bio->bi_hw_segments >= q->max_hw_segments || BIOVEC_VIRT_OVERSIZE(bio->bi_size)) { if (retried_segments) return 0; retried_segments = 1; blk_recount_segments(q, bio); } /* * setup the new entry, we might clear it again later if we * cannot add the page */ bvec = &bio->bi_io_vec[bio->bi_vcnt]; bvec->bv_page = page; bvec->bv_len = len; bvec->bv_offset = offset; /* * if queue has other restrictions (eg varying max sector size * depending on offset), it can specify a merge_bvec_fn in the * queue to get further control */ if (q->merge_bvec_fn) { /* * merge_bvec_fn() returns number of bytes it can accept * at this offset */ if (q->merge_bvec_fn(q, bio, bvec) < len) { bvec->bv_page = NULL; bvec->bv_len = 0; bvec->bv_offset = 0; return 0; } } /* If we may be able to merge these biovecs, force a recount */ if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec) || BIOVEC_VIRT_MERGEABLE(bvec-1, bvec))) bio->bi_flags &= ~(1 << BIO_SEG_VALID); bio->bi_vcnt++; bio->bi_phys_segments++; bio->bi_hw_segments++; done: bio->bi_size += len; return len; } /** * bio_add_pc_page - attempt to add page to bio * @q: the target queue * @bio: destination bio * @page: page to add * @len: vec entry length * @offset: vec entry offset * * Attempt to add a page to the bio_vec maplist. This can fail for a * number of reasons, such as the bio being full or target block * device limitations. The target block device must allow bio's * smaller than PAGE_SIZE, so it is always possible to add a single * page to an empty bio. This should only be used by REQ_PC bios. */ int bio_add_pc_page(request_queue_t *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { return __bio_add_page(q, bio, page, len, offset, q->max_hw_sectors); } /** * bio_add_page - attempt to add page to bio * @bio: destination bio * @page: page to add * @len: vec entry length * @offset: vec entry offset * * Attempt to add a page to the bio_vec maplist. This can fail for a * number of reasons, such as the bio being full or target block * device limitations. The target block device must allow bio's * smaller than PAGE_SIZE, so it is always possible to add a single * page to an empty bio. */ int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { struct request_queue *q = bdev_get_queue(bio->bi_bdev); return __bio_add_page(q, bio, page, len, offset, q->max_sectors); } struct bio_map_data { struct bio_vec *iovecs; void __user *userptr; }; static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio) { memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt); bio->bi_private = bmd; } static void bio_free_map_data(struct bio_map_data *bmd) { kfree(bmd->iovecs); kfree(bmd); } static struct bio_map_data *bio_alloc_map_data(int nr_segs) { struct bio_map_data *bmd = kmalloc(sizeof(*bmd), GFP_KERNEL); if (!bmd) return NULL; bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, GFP_KERNEL); if (bmd->iovecs) return bmd; kfree(bmd); return NULL; } /** * bio_uncopy_user - finish previously mapped bio * @bio: bio being terminated * * Free pages allocated from bio_copy_user() and write back data * to user space in case of a read. */ int bio_uncopy_user(struct bio *bio) { struct bio_map_data *bmd = bio->bi_private; const int read = bio_data_dir(bio) == READ; struct bio_vec *bvec; int i, ret = 0; __bio_for_each_segment(bvec, bio, i, 0) { char *addr = page_address(bvec->bv_page);