Instead of needing a special macro to iterate over all pages in a bvec just do a second passs over the whole bio. This also matches what we do on the release side. The release side helper is moved up to where we need the get helper to clearly express the symmetry.
Unified · Split
| @@ -861,6 +861,26 @@ int bio_add_page(struct bio *bio, struct page *page, · 860 unmodified lines | |||
| 861 | } | 861 | } |
| 862 | EXPORT_SYMBOL(bio_add_page); | 862 | EXPORT_SYMBOL(bio_add_page); |
| 863 | 863 | ||
| 864 | static void bio_get_pages(struct bio *bio) | ||
| 865 | { | ||
| 866 | struct bvec_iter_all iter_all; | ||
| 867 | struct bio_vec *bvec; | ||
| 868 | int i; | ||
| 869 | |||
| 870 | bio_for_each_segment_all(bvec, bio, i, iter_all) | ||
| 871 | get_page(bvec->bv_page); | ||
| 872 | } | ||
| 873 | |||
| 874 | static void bio_release_pages(struct bio *bio) | ||
| 875 | { | ||
| 876 | struct bvec_iter_all iter_all; | ||
| 877 | struct bio_vec *bvec; | ||
| 878 | int i; | ||
| 879 | |||
| 880 | bio_for_each_segment_all(bvec, bio, i, iter_all) | ||
| 881 | put_page(bvec->bv_page); | ||
| 882 | } | ||
| 883 | |||
| 864 | static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter) | 884 | static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter) |
| 865 | { | 885 | { |
| 866 | const struct bio_vec *bv = iter->bvec; | 886 | const struct bio_vec *bv = iter->bvec; |
| @@ -875,15 +895,6 @@ static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter) · 8 unmodified lines | |||
| 875 | bv->bv_offset + iter->iov_offset); | 895 | bv->bv_offset + iter->iov_offset); |
| 876 | if (unlikely(size != len)) | 896 | if (unlikely(size != len)) |
| 877 | return -EINVAL; | 897 | return -EINVAL; |
| 878 | |||
| 879 | if (!bio_flagged(bio, BIO_NO_PAGE_REF)) { | ||
| 880 | struct page *page; | ||
| 881 | int i; | ||
| 882 | |||
| 883 | mp_bvec_for_each_page(page, bv, i) | ||
| 884 | get_page(page); | ||
| 885 | } | ||
| 886 | |||
| 887 | iov_iter_advance(iter, size); | 898 | iov_iter_advance(iter, size); |
| 888 | return 0; | 899 | return 0; |
| 889 | } | 900 | } |
| @@ -963,13 +974,6 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · 73 unmodified lines | |||
| 963 | if (WARN_ON_ONCE(bio->bi_vcnt)) | 974 | if (WARN_ON_ONCE(bio->bi_vcnt)) |
| 964 | return -EINVAL; | 975 | return -EINVAL; |
| 965 | 976 | ||
| 966 | /* | ||
| 967 | * If this is a BVEC iter, then the pages are kernel pages. Don't | ||
| 968 | * release them on IO completion, if the caller asked us to. | ||
| 969 | */ | ||
| 970 | if (is_bvec && iov_iter_bvec_no_ref(iter)) | ||
| 971 | bio_set_flag(bio, BIO_NO_PAGE_REF); | ||
| 972 | |||
| 973 | do { | 977 | do { |
| 974 | if (is_bvec) | 978 | if (is_bvec) |
| 975 | ret = __bio_iov_bvec_add_pages(bio, iter); | 979 | ret = __bio_iov_bvec_add_pages(bio, iter); |
| @@ -977,6 +981,11 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · 1 unmodified line | |||
| 977 | ret = __bio_iov_iter_get_pages(bio, iter); | 981 | ret = __bio_iov_iter_get_pages(bio, iter); |
| 978 | } while (!ret && iov_iter_count(iter) && !bio_full(bio)); | 982 | } while (!ret && iov_iter_count(iter) && !bio_full(bio)); |
| 979 | 983 | ||
| 984 | if (iov_iter_bvec_no_ref(iter)) | ||
| 985 | bio_set_flag(bio, BIO_NO_PAGE_REF); | ||
| 986 | else | ||
| 987 | bio_get_pages(bio); | ||
| 988 | |||
| 980 | return bio->bi_vcnt ? 0 : ret; | 989 | return bio->bi_vcnt ? 0 : ret; |
| 981 | } | 990 | } |
| 982 | 991 | ||
| @@ -1670,16 +1679,6 @@ void bio_set_pages_dirty(struct bio *bio) · 687 unmodified lines | |||
| 1670 | } | 1679 | } |
| 1671 | } | 1680 | } |
| 1672 | 1681 | ||
| 1673 | static void bio_release_pages(struct bio *bio) | ||
| 1674 | { | ||
| 1675 | struct bio_vec *bvec; | ||
| 1676 | int i; | ||
| 1677 | struct bvec_iter_all iter_all; | ||
| 1678 | |||
| 1679 | bio_for_each_segment_all(bvec, bio, i, iter_all) | ||
| 1680 | put_page(bvec->bv_page); | ||
| 1681 | } | ||
| 1682 | |||
| 1683 | /* | 1682 | /* |
| 1684 | * bio_check_pages_dirty() will check that all the BIO's pages are still dirty. | 1683 | * bio_check_pages_dirty() will check that all the BIO's pages are still dirty. |
| 1685 | * If they are, then fine. If, however, some pages are clean then they must | 1684 | * If they are, then fine. If, however, some pages are clean then they must |
| @@ -189,9 +189,4 @@ static inline void mp_bvec_last_segment(const struct bio_vec *bvec, · 188 unmodified lines | |||
| 189 | } | 189 | } |
| 190 | } | 190 | } |
| 191 | 191 | ||
| 192 | #define mp_bvec_for_each_page(pg, bv, i) \ | ||
| 193 | for (i = (bv)->bv_offset / PAGE_SIZE; \ | ||
| 194 | (i <= (((bv)->bv_offset + (bv)->bv_len - 1) / PAGE_SIZE)) && \ | ||
| 195 | (pg = bvec_nth_page((bv)->bv_page, i)); i += 1) | ||
| 196 | |||
| 197 | #endif /* __LINUX_BVEC_ITER_H */ | 192 | #endif /* __LINUX_BVEC_ITER_H */ |