Move the max_sectors check into bvec_split_segs() such that a single call to that function can do all the necessary checks. This patch optimizes the fast path further, namely if a bvec fits in a page.
Unified · Split
| @@ -157,22 +157,36 @@ static unsigned get_max_segment_size(const struct request_queue *q, · 156 unmodified lines | ||
| 157 | 157 | queue_max_segment_size(q)); |
| 158 | 158 | } |
| 159 | 159 | |
| 160 | −/* | |
| 161 | − * Split the bvec @bv into segments, and update all kinds of | |
| 162 | − * variables. | |
| 160 | +/** | |
| 161 | + * bvec_split_segs - verify whether or not a bvec should be split in the middle | |
| 162 | + * @q: [in] request queue associated with the bio associated with @bv | |
| 163 | + * @bv: [in] bvec to examine | |
| 164 | + * @nsegs: [in,out] Number of segments in the bio being built. Incremented | |
| 165 | + * by the number of segments from @bv that may be appended to that | |
| 166 | + * bio without exceeding @max_segs | |
| 167 | + * @sectors: [in,out] Number of sectors in the bio being built. Incremented | |
| 168 | + * by the number of sectors from @bv that may be appended to that | |
| 169 | + * bio without exceeding @max_sectors | |
| 170 | + * @max_segs: [in] upper bound for *@nsegs | |
| 171 | + * @max_sectors: [in] upper bound for *@sectors | |
| 172 | + * | |
| 173 | + * When splitting a bio, it can happen that a bvec is encountered that is too | |
| 174 | + * big to fit in a single segment and hence that it has to be split in the | |
| 175 | + * middle. This function verifies whether or not that should happen. The value | |
| 176 | + * %true is returned if and only if appending the entire @bv to a bio with | |
| 177 | + * *@nsegs segments and *@sectors sectors would make that bio unacceptable for | |
| 178 | + * the block driver. | |
| 163 | 179 | */ |
| 164 | 180 | static bool bvec_split_segs(const struct request_queue *q, |
| 165 | 181 | const struct bio_vec *bv, unsigned *nsegs, |
| 166 | − unsigned *sectors, unsigned max_segs) | |
| 182 | + unsigned *sectors, unsigned max_segs, | |
| 183 | + unsigned max_sectors) | |
| 167 | 184 | { |
| 168 | − unsigned len = bv->bv_len; | |
| 185 | + unsigned max_len = (min(max_sectors, UINT_MAX >> 9) - *sectors) << 9; | |
| 186 | + unsigned len = min(bv->bv_len, max_len); | |
| 169 | 187 | unsigned total_len = 0; |
| 170 | 188 | unsigned seg_size = 0; |
| 171 | 189 | |
| 172 | − /* | |
| 173 | − * Multi-page bvec may be too big to hold in one segment, so the | |
| 174 | − * current bvec has to be splitted as multiple segments. | |
| 175 | − */ | |
| 176 | 190 | while (len && *nsegs < max_segs) { |
| 177 | 191 | seg_size = get_max_segment_size(q, bv->bv_offset + total_len); |
| 178 | 192 | seg_size = min(seg_size, len); |
| @@ -187,8 +201,8 @@ static bool bvec_split_segs(const struct request_queue *q, · 8 unmodified lines | ||
| 187 | 201 | |
| 188 | 202 | *sectors += total_len >> 9; |
| 189 | 203 | |
| 190 | − /* split in the middle of the bvec if len != 0 */ | |
| 191 | − return !!len; | |
| 204 | + /* tell the caller to split the bvec if it is too big to fit */ | |
| 205 | + return len > 0 || bv->bv_len > max_len; | |
| 192 | 206 | } |
| 193 | 207 | |
| 194 | 208 | /** |
| @@ -229,34 +243,18 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, · 34 unmodified lines | ||
| 229 | 243 | if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset)) |
| 230 | 244 | goto split; |
| 231 | 245 | |
| 232 | − if (sectors + (bv.bv_len >> 9) > max_sectors) { | |
| 233 | − /* | |
| 234 | − * Consider this a new segment if we're splitting in | |
| 235 | − * the middle of this vector. | |
| 236 | − */ | |
| 237 | − if (nsegs < max_segs && | |
| 238 | − sectors < max_sectors) { | |
| 239 | − /* split in the middle of bvec */ | |
| 240 | − bv.bv_len = (max_sectors - sectors) << 9; | |
| 241 | − bvec_split_segs(q, &bv, &nsegs, | |
| 242 | − §ors, max_segs); | |
| 243 | − } | |
| 246 | + if (nsegs < max_segs && | |
| 247 | + sectors + (bv.bv_len >> 9) <= max_sectors && | |
| 248 | + bv.bv_offset + bv.bv_len <= PAGE_SIZE) { | |
| 249 | + nsegs++; | |
| 250 | + sectors += bv.bv_len >> 9; | |
| 251 | + } else if (bvec_split_segs(q, &bv, &nsegs, §ors, max_segs, | |
| 252 | + max_sectors)) { | |
| 244 | 253 | goto split; |
| 245 | 254 | } |
| 246 | 255 | |
| 247 | − if (nsegs == max_segs) | |
| 248 | − goto split; | |
| 249 | − | |
| 250 | 256 | bvprv = bv; |
| 251 | 257 | bvprvp = &bvprv; |
| 252 | − | |
| 253 | − if (bv.bv_offset + bv.bv_len <= PAGE_SIZE) { | |
| 254 | − nsegs++; | |
| 255 | − sectors += bv.bv_len >> 9; | |
| 256 | − } else if (bvec_split_segs(q, &bv, &nsegs, §ors, | |
| 257 | − max_segs)) { | |
| 258 | − goto split; | |
| 259 | − } | |
| 260 | 258 | } |
| 261 | 259 | |
| 262 | 260 | *segs = nsegs; |
| @@ -363,7 +361,7 @@ unsigned int blk_recalc_rq_segments(struct request *rq) · 100 unmodified lines | ||
| 363 | 361 | |
| 364 | 362 | rq_for_each_bvec(bv, rq, iter) |
| 365 | 363 | bvec_split_segs(rq->q, &bv, &nr_phys_segs, &nr_sectors, |
| 366 | − UINT_MAX); | |
| 364 | + UINT_MAX, UINT_MAX); | |
| 367 | 365 | return nr_phys_segs; |
| 368 | 366 | } |
| 369 | 367 | |