/* * 2.5 block I/O model * * Copyright (C) 2001 Jens Axboe <axboe@suse.de> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public Licens * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- */#ifndef __LINUX_BIO_H#define __LINUX_BIO_H #include <linux/highmem.h>#include <linux/mempool.h>#include <linux/ioprio.h>#include <linux/bug.h> #ifdef CONFIG_BLOCK #include <asm/io.h> /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */#include <linux/blk_types.h> #define BIO_DEBUG #ifdef BIO_DEBUG#define BIO_BUG_ON BUG_ON#else#define BIO_BUG_ON#endif #define BIO_MAX_PAGES 256#define BIO_MAX_SIZE (BIO_MAX_PAGES << PAGE_CACHE_SHIFT)#define BIO_MAX_SECTORS (BIO_MAX_SIZE >> 9) /* * upper 16 bits of bi_rw define the io priority of this bio */#define BIO_PRIO_SHIFT (8 * sizeof(unsigned long) - IOPRIO_BITS)#define bio_prio(bio) ((bio)->bi_rw >> BIO_PRIO_SHIFT)#define bio_prio_valid(bio) ioprio_valid(bio_prio(bio)) #define bio_set_prio(bio, prio) do { \ WARN_ON(prio >= (1 << IOPRIO_BITS)); \ (bio)->bi_rw &= ((1UL << BIO_PRIO_SHIFT) - 1); \ (bio)->bi_rw |= ((unsigned long) (prio) << BIO_PRIO_SHIFT); \} while (0) /* * various member access, note that bio_data should of course not be used * on highmem page vectors */#define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx]) #define bvec_iter_page(bvec, iter) \ (__bvec_iter_bvec((bvec), (iter))->bv_page) #define bvec_iter_len(bvec, iter) \ min((iter).bi_size, \ __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done) #define bvec_iter_offset(bvec, iter) \ (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done) #define bvec_iter_bvec(bvec, iter) \((struct bio_vec) { \ .bv_page = bvec_iter_page((bvec), (iter)), \ .bv_len = bvec_iter_len((bvec), (iter)), \ .bv_offset = bvec_iter_offset((bvec), (iter)), \}) #define bio_iter_iovec(bio, iter) \ bvec_iter_bvec((bio)->bi_io_vec, (iter)) #define bio_iter_page(bio, iter) \ bvec_iter_page((bio)->bi_io_vec, (iter))#define bio_iter_len(bio, iter) \ bvec_iter_len((bio)->bi_io_vec, (iter))#define bio_iter_offset(bio, iter) \ bvec_iter_offset((bio)->bi_io_vec, (iter)) #define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter)#define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter)#define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter) #define bio_multiple_segments(bio) \ ((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len)#define bio_sectors(bio) ((bio)->bi_iter.bi_size >> 9)#define bio_end_sector(bio) ((bio)->bi_iter.bi_sector + bio_sectors((bio))) /* * Check whether this bio carries any data or not. A NULL bio is allowed. */static inline bool bio_has_data(struct bio *bio){ if (bio && bio->bi_iter.bi_size && !(bio->bi_rw & REQ_DISCARD)) return true; return false;} static inline bool bio_is_rw(struct bio *bio){ if (!bio_has_data(bio)) return false; if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK) return false; return true;} static inline bool bio_mergeable(struct bio *bio){ if (bio->bi_rw & REQ_NOMERGE_FLAGS) return false; return true;} static inline unsigned int bio_cur_bytes(struct bio *bio){ if (bio_has_data(bio)) return bio_iovec(bio).bv_len; else /* dataless requests such as discard */ return bio->bi_iter.bi_size;} static inline void *bio_data(struct bio *bio){ if (bio_has_data(bio)) return page_address(bio_page(bio)) + bio_offset(bio); return NULL;} /* * will die */#define bio_to_phys(bio) (page_to_phys(bio_page((bio))) + (unsigned long) bio_offset((bio)))#define bvec_to_phys(bv) (page_to_phys((bv)->bv_page) + (unsigned long) (bv)->bv_offset) /* * queues that have highmem support enabled may still need to revert to * PIO transfers occasionally and thus map high pages temporarily. For * permanent PIO fall back, user is probably better off disabling highmem * I/O completely on that queue (see ide-dma for example) */#define __bio_kmap_atomic(bio, iter) \ (kmap_atomic(bio_iter_iovec((bio), (iter)).bv_page) + \ bio_iter_iovec((bio), (iter)).bv_offset) #define __bio_kunmap_atomic(addr) kunmap_atomic(addr) /* * merge helpers etc */ /* Default implementation of BIOVEC_PHYS_MERGEABLE */#define __BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2))) /* * allow arch override, for eg virtualized architectures (put in asm/io.h) */#ifndef BIOVEC_PHYS_MERGEABLE#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ __BIOVEC_PHYS_MERGEABLE(vec1, vec2)#endif #define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \ (((addr1) | (mask)) == (((addr2) - 1) | (mask)))#define BIOVEC_SEG_BOUNDARY(q, b1, b2) \ __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q))) /* * drivers should _never_ use the all version - the bio may have been split * before it got to the driver and the driver won't own all of it */#define bio_for_each_segment_all(bvl, bio, i) \ for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++) static inline void bvec_iter_advance(struct bio_vec *bv, struct bvec_iter *iter, unsigned bytes){ WARN_ONCE(bytes > iter->bi_size, "Attempted to advance past end of bvec iter\n"); while (bytes) { unsigned len = min(bytes, bvec_iter_len(bv, *iter)); bytes -= len; iter->bi_size -= len; iter->bi_bvec_done += len; if (iter->bi_bvec_done == __bvec_iter_bvec(bv, *iter)->bv_len) { iter->bi_bvec_done = 0; iter->bi_idx++; } }} #define for_each_bvec(bvl, bio_vec, iter, start) \ for (iter = (start); \ (iter).bi_size && \ ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \ bvec_iter_advance((bio_vec), &(iter), (bvl).bv_len)) static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter, unsigned bytes){ iter->bi_sector += bytes >> 9; if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK) iter->bi_size -= bytes; else bvec_iter_advance(bio->bi_io_vec, iter, bytes);} #define __bio_for_each_segment(bvl, bio, iter, start) \ for (iter = (start); \ (iter).bi_size && \ ((bvl = bio_iter_iovec((bio), (iter))), 1); \ bio_advance_iter((bio), &(iter), (bvl).bv_len)) #define bio_for_each_segment(bvl, bio, iter) \ __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter) #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len) static inline unsigned bio_segments(struct bio *bio){ unsigned segs = 0; struct bio_vec bv; struct bvec_iter iter; /* * We special case discard/write same, because they interpret bi_size * differently: */ if (bio->bi_rw & REQ_DISCARD) return 1; if (bio->bi_rw & REQ_WRITE_SAME) return 1; bio_for_each_segment(bv, bio, iter) segs++; return segs;} /* * get a reference to a bio, so it won't disappear. the intended use is * something like: * * bio_get(bio); * submit_bio(rw, bio); * if (bio->bi_flags ...) * do_something * bio_put(bio); * * without the bio_get(), it could potentially complete I/O before submit_bio * returns. and then bio would be freed memory when if (bio->bi_flags ...) * runs */static inline void bio_get(struct bio *bio){ bio->bi_flags |= (1 << BIO_REFFED); smp_mb__before_atomic(); atomic_inc(&bio->__bi_cnt);} static inline void bio_cnt_set(struct bio *bio, unsigned int count){ if (count != 1) { bio->bi_flags |= (1 << BIO_REFFED); smp_mb__before_atomic(); } atomic_set(&bio->__bi_cnt, count);} static inline bool bio_flagged(struct bio *bio, unsigned int bit){ return (bio->bi_flags & (1U << bit)) != 0;} static inline void bio_set_flag(struct bio *bio, unsigned int bit){ bio->bi_flags |= (1U << bit);} static inline void bio_clear_flag(struct bio *bio, unsigned int bit){ bio->bi_flags &= ~(1U << bit);} enum bip_flags { BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */ BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */ BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */}; #if defined(CONFIG_BLK_DEV_INTEGRITY) static inline struct bio_integrity_payload *bio_integrity(struct bio *bio){ if (bio->bi_rw & REQ_INTEGRITY) return bio->bi_integrity; return NULL;} /* * bio integrity payload */struct bio_integrity_payload { struct bio *bip_bio; /* parent bio */ struct bvec_iter bip_iter; bio_end_io_t *bip_end_io; /* saved I/O completion fn */ unsigned short bip_slab; /* slab the bip came from */ unsigned short bip_vcnt; /* # of integrity bio_vecs */ unsigned short bip_max_vcnt; /* integrity bio_vec slots */ unsigned short bip_flags; /* control flags */ struct work_struct bip_work; /* I/O completion */ struct bio_vec *bip_vec; struct bio_vec bip_inline_vecs[0];/* embedded bvec array */}; static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag){ struct bio_integrity_payload *bip = bio_integrity(bio); if (bip) return bip->bip_flags & flag; return false;} static inline sector_t bip_get_seed(struct bio_integrity_payload *bip){ return bip->bip_iter.bi_sector;} static inline void bip_set_seed(struct bio_integrity_payload *bip, sector_t seed){ bip->bip_iter.bi_sector = seed;} #endif /* CONFIG_BLK_DEV_INTEGRITY */ extern void bio_trim(struct bio *bio, int offset, int size);extern struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs); /** * bio_next_split - get next @sectors from a bio, splitting if necessary * @bio: bio to split * @sectors: number of sectors to split from the front of @bio * @gfp: gfp mask * @bs: bio set to allocate from * * Returns a bio representing the next @sectors of @bio - if the bio is smaller * than @sectors, returns the original bio unchanged. */static inline struct bio *bio_next_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs){ if (sectors >= bio_sectors(bio)) return bio; return bio_split(bio, sectors, gfp, bs);} extern struct bio_set *bioset_create(unsigned int, unsigned int);extern struct bio_set *bioset_create_nobvec(unsigned int, unsigned int);extern void bioset_free(struct bio_set *);extern mempool_t *biovec_create_pool(int pool_entries); extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);extern void bio_put(struct bio *); extern void __bio_clone_fast(struct bio *, struct bio *);extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs); extern struct bio_set *fs_bio_set; static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs){ return bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);} static inline struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask){ return bio_clone_bioset(bio, gfp_mask, fs_bio_set);} static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs){ return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);} static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask){ return bio_clone_bioset(bio, gfp_mask, NULL); } extern void bio_endio(struct bio *); static inline void bio_io_error(struct bio *bio){ bio->bi_error = -EIO; bio_endio(bio);} struct request_queue;extern int bio_phys_segments(struct request_queue *, struct bio *); extern int submit_bio_wait(int rw, struct bio *bio);extern void bio_advance(struct bio *, unsigned); extern void bio_init(struct bio *);extern void bio_reset(struct bio *);void bio_chain(struct bio *, struct bio *); extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, unsigned int, unsigned int);struct rq_map_data;extern struct bio *bio_map_user_iov(struct request_queue *, const struct iov_iter *, gfp_t);extern void bio_unmap_user(struct bio *);extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int, gfp_t);extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int, gfp_t, int);extern void bio_set_pages_dirty(struct bio *bio);extern void bio_check_pages_dirty(struct bio *bio); void generic_start_io_acct(int rw, unsigned long sectors, struct hd_struct *part);void generic_end_io_acct(int rw, struct hd_struct *part, unsigned long start_time); #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"#endif#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGEextern void bio_flush_dcache_pages(struct bio *bi);#elsestatic inline void bio_flush_dcache_pages(struct bio *bi){}#endif extern void bio_copy_data(struct bio *dst, struct bio *src);extern int bio_alloc_pages(struct bio *bio, gfp_t gfp); extern struct bio *bio_copy_user_iov(struct request_queue *, struct rq_map_data *, const struct iov_iter *, gfp_t);extern int bio_uncopy_user(struct bio *);void zero_fill_bio(struct bio *bio);extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);extern unsigned int bvec_nr_vecs(unsigned short idx); #ifdef CONFIG_BLK_CGROUPint bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css);int bio_associate_current(struct bio *bio);void bio_disassociate_task(struct bio *bio);#else /* CONFIG_BLK_CGROUP */static inline int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css) { return 0; }Lines 1–500 of 801. Load more
/* * 2.5 block I/O model * * Copyright (C) 2001 Jens Axboe <axboe@suse.de> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public Licens * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- */ #ifndef __LINUX_BIO_H #define __LINUX_BIO_H #include <linux/highmem.h> #include <linux/mempool.h> #include <linux/ioprio.h> #include <linux/bug.h> #ifdef CONFIG_BLOCK #include <asm/io.h> /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */ #include <linux/blk_types.h> #define BIO_DEBUG #ifdef BIO_DEBUG #define BIO_BUG_ON BUG_ON #else #define BIO_BUG_ON #endif #define BIO_MAX_PAGES 256 #define BIO_MAX_SIZE (BIO_MAX_PAGES << PAGE_CACHE_SHIFT) #define BIO_MAX_SECTORS (BIO_MAX_SIZE >> 9) /* * upper 16 bits of bi_rw define the io priority of this bio */ #define BIO_PRIO_SHIFT (8 * sizeof(unsigned long) - IOPRIO_BITS) #define bio_prio(bio) ((bio)->bi_rw >> BIO_PRIO_SHIFT) #define bio_prio_valid(bio) ioprio_valid(bio_prio(bio)) #define bio_set_prio(bio, prio) do { \ WARN_ON(prio >= (1 << IOPRIO_BITS)); \ (bio)->bi_rw &= ((1UL << BIO_PRIO_SHIFT) - 1); \ (bio)->bi_rw |= ((unsigned long) (prio) << BIO_PRIO_SHIFT); \ } while (0) /* * various member access, note that bio_data should of course not be used * on highmem page vectors */ #define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx]) #define bvec_iter_page(bvec, iter) \ (__bvec_iter_bvec((bvec), (iter))->bv_page) #define bvec_iter_len(bvec, iter) \ min((iter).bi_size, \ __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done) #define bvec_iter_offset(bvec, iter) \ (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done) #define bvec_iter_bvec(bvec, iter) \ ((struct bio_vec) { \ .bv_page = bvec_iter_page((bvec), (iter)), \ .bv_len = bvec_iter_len((bvec), (iter)), \ .bv_offset = bvec_iter_offset((bvec), (iter)), \ }) #define bio_iter_iovec(bio, iter) \ bvec_iter_bvec((bio)->bi_io_vec, (iter)) #define bio_iter_page(bio, iter) \ bvec_iter_page((bio)->bi_io_vec, (iter)) #define bio_iter_len(bio, iter) \ bvec_iter_len((bio)->bi_io_vec, (iter)) #define bio_iter_offset(bio, iter) \ bvec_iter_offset((bio)->bi_io_vec, (iter)) #define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter) #define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter) #define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter) #define bio_multiple_segments(bio) \ ((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len) #define bio_sectors(bio) ((bio)->bi_iter.bi_size >> 9) #define bio_end_sector(bio) ((bio)->bi_iter.bi_sector + bio_sectors((bio))) /* * Check whether this bio carries any data or not. A NULL bio is allowed. */ static inline bool bio_has_data(struct bio *bio) { if (bio && bio->bi_iter.bi_size && !(bio->bi_rw & REQ_DISCARD)) return true; return false; } static inline bool bio_is_rw(struct bio *bio) { if (!bio_has_data(bio)) return false; if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK) return false; return true; } static inline bool bio_mergeable(struct bio *bio) { if (bio->bi_rw & REQ_NOMERGE_FLAGS) return false; return true; } static inline unsigned int bio_cur_bytes(struct bio *bio) { if (bio_has_data(bio)) return bio_iovec(bio).bv_len; else /* dataless requests such as discard */ return bio->bi_iter.bi_size; } static inline void *bio_data(struct bio *bio) { if (bio_has_data(bio)) return page_address(bio_page(bio)) + bio_offset(bio); return NULL; } /* * will die */ #define bio_to_phys(bio) (page_to_phys(bio_page((bio))) + (unsigned long) bio_offset((bio))) #define bvec_to_phys(bv) (page_to_phys((bv)->bv_page) + (unsigned long) (bv)->bv_offset) /* * queues that have highmem support enabled may still need to revert to * PIO transfers occasionally and thus map high pages temporarily. For * permanent PIO fall back, user is probably better off disabling highmem * I/O completely on that queue (see ide-dma for example) */ #define __bio_kmap_atomic(bio, iter) \ (kmap_atomic(bio_iter_iovec((bio), (iter)).bv_page) + \ bio_iter_iovec((bio), (iter)).bv_offset) #define __bio_kunmap_atomic(addr) kunmap_atomic(addr) /* * merge helpers etc */ /* Default implementation of BIOVEC_PHYS_MERGEABLE */ #define __BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2))) /* * allow arch override, for eg virtualized architectures (put in asm/io.h) */ #ifndef BIOVEC_PHYS_MERGEABLE #define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \ __BIOVEC_PHYS_MERGEABLE(vec1, vec2) #endif #define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \ (((addr1) | (mask)) == (((addr2) - 1) | (mask))) #define BIOVEC_SEG_BOUNDARY(q, b1, b2) \ __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q))) /* * drivers should _never_ use the all version - the bio may have been split * before it got to the driver and the driver won't own all of it */ #define bio_for_each_segment_all(bvl, bio, i) \ for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++) static inline void bvec_iter_advance(struct bio_vec *bv, struct bvec_iter *iter, unsigned bytes) { WARN_ONCE(bytes > iter->bi_size, "Attempted to advance past end of bvec iter\n"); while (bytes) { unsigned len = min(bytes, bvec_iter_len(bv, *iter)); bytes -= len; iter->bi_size -= len; iter->bi_bvec_done += len; if (iter->bi_bvec_done == __bvec_iter_bvec(bv, *iter)->bv_len) { iter->bi_bvec_done = 0; iter->bi_idx++; } } } #define for_each_bvec(bvl, bio_vec, iter, start) \ for (iter = (start); \ (iter).bi_size && \ ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \ bvec_iter_advance((bio_vec), &(iter), (bvl).bv_len)) static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter, unsigned bytes) { iter->bi_sector += bytes >> 9; if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK) iter->bi_size -= bytes; else bvec_iter_advance(bio->bi_io_vec, iter, bytes); } #define __bio_for_each_segment(bvl, bio, iter, start) \ for (iter = (start); \ (iter).bi_size && \ ((bvl = bio_iter_iovec((bio), (iter))), 1); \ bio_advance_iter((bio), &(iter), (bvl).bv_len)) #define bio_for_each_segment(bvl, bio, iter) \ __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter) #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len) static inline unsigned bio_segments(struct bio *bio) { unsigned segs = 0; struct bio_vec bv; struct bvec_iter iter; /* * We special case discard/write same, because they interpret bi_size * differently: */ if (bio->bi_rw & REQ_DISCARD) return 1; if (bio->bi_rw & REQ_WRITE_SAME) return 1; bio_for_each_segment(bv, bio, iter) segs++; return segs; } /* * get a reference to a bio, so it won't disappear. the intended use is * something like: * * bio_get(bio); * submit_bio(rw, bio); * if (bio->bi_flags ...) * do_something * bio_put(bio); * * without the bio_get(), it could potentially complete I/O before submit_bio * returns. and then bio would be freed memory when if (bio->bi_flags ...) * runs */ static inline void bio_get(struct bio *bio) { bio->bi_flags |= (1 << BIO_REFFED); smp_mb__before_atomic(); atomic_inc(&bio->__bi_cnt); } static inline void bio_cnt_set(struct bio *bio, unsigned int count) { if (count != 1) { bio->bi_flags |= (1 << BIO_REFFED); smp_mb__before_atomic(); } atomic_set(&bio->__bi_cnt, count); } static inline bool bio_flagged(struct bio *bio, unsigned int bit) { return (bio->bi_flags & (1U << bit)) != 0; } static inline void bio_set_flag(struct bio *bio, unsigned int bit) { bio->bi_flags |= (1U << bit); } static inline void bio_clear_flag(struct bio *bio, unsigned int bit) { bio->bi_flags &= ~(1U << bit); } enum bip_flags { BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */ BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */ BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */ }; #if defined(CONFIG_BLK_DEV_INTEGRITY) static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) { if (bio->bi_rw & REQ_INTEGRITY) return bio->bi_integrity; return NULL; } /* * bio integrity payload */ struct bio_integrity_payload { struct bio *bip_bio; /* parent bio */ struct bvec_iter bip_iter; bio_end_io_t *bip_end_io; /* saved I/O completion fn */ unsigned short bip_slab; /* slab the bip came from */ unsigned short bip_vcnt; /* # of integrity bio_vecs */ unsigned short bip_max_vcnt; /* integrity bio_vec slots */ unsigned short bip_flags; /* control flags */ struct work_struct bip_work; /* I/O completion */ struct bio_vec *bip_vec; struct bio_vec bip_inline_vecs[0];/* embedded bvec array */ }; static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag) { struct bio_integrity_payload *bip = bio_integrity(bio); if (bip) return bip->bip_flags & flag; return false; } static inline sector_t bip_get_seed(struct bio_integrity_payload *bip) { return bip->bip_iter.bi_sector; } static inline void bip_set_seed(struct bio_integrity_payload *bip, sector_t seed) { bip->bip_iter.bi_sector = seed; } #endif /* CONFIG_BLK_DEV_INTEGRITY */ extern void bio_trim(struct bio *bio, int offset, int size); extern struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs); /** * bio_next_split - get next @sectors from a bio, splitting if necessary * @bio: bio to split * @sectors: number of sectors to split from the front of @bio * @gfp: gfp mask * @bs: bio set to allocate from * * Returns a bio representing the next @sectors of @bio - if the bio is smaller * than @sectors, returns the original bio unchanged. */ static inline struct bio *bio_next_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs) { if (sectors >= bio_sectors(bio)) return bio; return bio_split(bio, sectors, gfp, bs); } extern struct bio_set *bioset_create(unsigned int, unsigned int); extern struct bio_set *bioset_create_nobvec(unsigned int, unsigned int); extern void bioset_free(struct bio_set *); extern mempool_t *biovec_create_pool(int pool_entries); extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *); extern void bio_put(struct bio *); extern void __bio_clone_fast(struct bio *, struct bio *); extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *); extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs); extern struct bio_set *fs_bio_set; static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) { return bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set); } static inline struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask) { return bio_clone_bioset(bio, gfp_mask, fs_bio_set); } static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs) { return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL); } static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask) { return bio_clone_bioset(bio, gfp_mask, NULL); } extern void bio_endio(struct bio *); static inline void bio_io_error(struct bio *bio) { bio->bi_error = -EIO; bio_endio(bio); } struct request_queue; extern int bio_phys_segments(struct request_queue *, struct bio *); extern int submit_bio_wait(int rw, struct bio *bio); extern void bio_advance(struct bio *, unsigned); extern void bio_init(struct bio *); extern void bio_reset(struct bio *); void bio_chain(struct bio *, struct bio *); extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int); extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, unsigned int, unsigned int); struct rq_map_data; extern struct bio *bio_map_user_iov(struct request_queue *, const struct iov_iter *, gfp_t); extern void bio_unmap_user(struct bio *); extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int, gfp_t); extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int, gfp_t, int); extern void bio_set_pages_dirty(struct bio *bio); extern void bio_check_pages_dirty(struct bio *bio); void generic_start_io_acct(int rw, unsigned long sectors, struct hd_struct *part); void generic_end_io_acct(int rw, struct hd_struct *part, unsigned long start_time); #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform" #endif #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE extern void bio_flush_dcache_pages(struct bio *bi); #else static inline void bio_flush_dcache_pages(struct bio *bi) { } #endif extern void bio_copy_data(struct bio *dst, struct bio *src); extern int bio_alloc_pages(struct bio *bio, gfp_t gfp); extern struct bio *bio_copy_user_iov(struct request_queue *, struct rq_map_data *, const struct iov_iter *, gfp_t); extern int bio_uncopy_user(struct bio *); void zero_fill_bio(struct bio *bio); extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *); extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int); extern unsigned int bvec_nr_vecs(unsigned short idx); #ifdef CONFIG_BLK_CGROUP int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css); int bio_associate_current(struct bio *bio); void bio_disassociate_task(struct bio *bio); #else /* CONFIG_BLK_CGROUP */ static inline int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css) { return 0; }