Move the max_sectors check into bvec_split_segs() such that a single call to that function can do all the necessary checks. This patch optimizes the fast path further, namely if a bvec fits in a page.
Unified · Split
| @@ -157,22 +157,36 @@ static unsigned get_max_segment_size(const struct request_queue *q, · 156 unmodified lines | |||
| 157 | queue_max_segment_size(q)); | 157 | queue_max_segment_size(q)); |
| 158 | } | 158 | } |
| 159 | 159 | ||
| 160 | /* | 160 | /** |
| 161 | * Split the bvec @bv into segments, and update all kinds of | 161 | * bvec_split_segs - verify whether or not a bvec should be split in the middle |
| 162 | * variables. | 162 | * @q: [in] request queue associated with the bio associated with @bv |
| 163 | * @bv: [in] bvec to examine | ||
| 164 | * @nsegs: [in,out] Number of segments in the bio being built. Incremented | ||
| 165 | * by the number of segments from @bv that may be appended to that | ||
| 166 | * bio without exceeding @max_segs | ||
| 167 | * @sectors: [in,out] Number of sectors in the bio being built. Incremented | ||
| 168 | * by the number of sectors from @bv that may be appended to that | ||
| 169 | * bio without exceeding @max_sectors | ||
| 170 | * @max_segs: [in] upper bound for *@nsegs | ||
| 171 | * @max_sectors: [in] upper bound for *@sectors | ||
| 172 | * | ||
| 173 | * When splitting a bio, it can happen that a bvec is encountered that is too | ||
| 174 | * big to fit in a single segment and hence that it has to be split in the | ||
| 175 | * middle. This function verifies whether or not that should happen. The value | ||
| 176 | * %true is returned if and only if appending the entire @bv to a bio with | ||
| 177 | * *@nsegs segments and *@sectors sectors would make that bio unacceptable for | ||
| 178 | * the block driver. | ||
| 163 | */ | 179 | */ |
| 164 | static bool bvec_split_segs(const struct request_queue *q, | 180 | static bool bvec_split_segs(const struct request_queue *q, |
| 165 | const struct bio_vec *bv, unsigned *nsegs, | 181 | const struct bio_vec *bv, unsigned *nsegs, |
| 166 | unsigned *sectors, unsigned max_segs) | 182 | unsigned *sectors, unsigned max_segs, |
| 183 | unsigned max_sectors) | ||
| 167 | { | 184 | { |
| 168 | unsigned len = bv->bv_len; | 185 | unsigned max_len = (min(max_sectors, UINT_MAX >> 9) - *sectors) << 9; |
| 186 | unsigned len = min(bv->bv_len, max_len); | ||
| 169 | unsigned total_len = 0; | 187 | unsigned total_len = 0; |
| 170 | unsigned seg_size = 0; | 188 | unsigned seg_size = 0; |
| 171 | 189 | ||
| 172 | /* | ||
| 173 | * Multi-page bvec may be too big to hold in one segment, so the | ||
| 174 | * current bvec has to be splitted as multiple segments. | ||
| 175 | */ | ||
| 176 | while (len && *nsegs < max_segs) { | 190 | while (len && *nsegs < max_segs) { |
| 177 | seg_size = get_max_segment_size(q, bv->bv_offset + total_len); | 191 | seg_size = get_max_segment_size(q, bv->bv_offset + total_len); |
| 178 | seg_size = min(seg_size, len); | 192 | seg_size = min(seg_size, len); |
| @@ -187,8 +201,8 @@ static bool bvec_split_segs(const struct request_queue *q, · 8 unmodified lines | |||
| 187 | 201 | ||
| 188 | *sectors += total_len >> 9; | 202 | *sectors += total_len >> 9; |
| 189 | 203 | ||
| 190 | /* split in the middle of the bvec if len != 0 */ | 204 | /* tell the caller to split the bvec if it is too big to fit */ |
| 191 | return !!len; | 205 | return len > 0 || bv->bv_len > max_len; |
| 192 | } | 206 | } |
| 193 | 207 | ||
| 194 | /** | 208 | /** |
| @@ -229,34 +243,18 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, · 34 unmodified lines | |||
| 229 | if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset)) | 243 | if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset)) |
| 230 | goto split; | 244 | goto split; |
| 231 | 245 | ||
| 232 | if (sectors + (bv.bv_len >> 9) > max_sectors) { | 246 | if (nsegs < max_segs && |
| 233 | /* | 247 | sectors + (bv.bv_len >> 9) <= max_sectors && |
| 234 | * Consider this a new segment if we're splitting in | 248 | bv.bv_offset + bv.bv_len <= PAGE_SIZE) { |
| 235 | * the middle of this vector. | 249 | nsegs++; |
| 236 | */ | 250 | sectors += bv.bv_len >> 9; |
| 237 | if (nsegs < max_segs && | 251 | } else if (bvec_split_segs(q, &bv, &nsegs, §ors, max_segs, |
| 238 | sectors < max_sectors) { | 252 | max_sectors)) { |
| 239 | /* split in the middle of bvec */ | ||
| 240 | bv.bv_len = (max_sectors - sectors) << 9; | ||
| 241 | bvec_split_segs(q, &bv, &nsegs, | ||
| 242 | §ors, max_segs); | ||
| 243 | } | ||
| 244 | goto split; | 253 | goto split; |
| 245 | } | 254 | } |
| 246 | 255 | ||
| 247 | if (nsegs == max_segs) | ||
| 248 | goto split; | ||
| 249 | |||
| 250 | bvprv = bv; | 256 | bvprv = bv; |
| 251 | bvprvp = &bvprv; | 257 | bvprvp = &bvprv; |
| 252 | |||
| 253 | if (bv.bv_offset + bv.bv_len <= PAGE_SIZE) { | ||
| 254 | nsegs++; | ||
| 255 | sectors += bv.bv_len >> 9; | ||
| 256 | } else if (bvec_split_segs(q, &bv, &nsegs, §ors, | ||
| 257 | max_segs)) { | ||
| 258 | goto split; | ||
| 259 | } | ||
| 260 | } | 258 | } |
| 261 | 259 | ||
| 262 | *segs = nsegs; | 260 | *segs = nsegs; |
| @@ -363,7 +361,7 @@ unsigned int blk_recalc_rq_segments(struct request *rq) · 100 unmodified lines | |||
| 363 | 361 | ||
| 364 | rq_for_each_bvec(bv, rq, iter) | 362 | rq_for_each_bvec(bv, rq, iter) |
| 365 | bvec_split_segs(rq->q, &bv, &nr_phys_segs, &nr_sectors, | 363 | bvec_split_segs(rq->q, &bv, &nr_phys_segs, &nr_sectors, |
| 366 | UINT_MAX); | 364 | UINT_MAX, UINT_MAX); |
| 367 | return nr_phys_segs; | 365 | return nr_phys_segs; |
| 368 | } | 366 | } |
| 369 | 367 | ||