History

block: stop calling blk_queue_bounce for passthrough requests

Instead of overloading the passthrough fast path with the deprecated block layer bounce buffering let the users that combine an old undermaintained driver with a highmem system pay the price by always falling back to copies in that case.

Christoph Hellwig committed 5 years ago

Commit 393bb121 parent 9bb33f25 files · +36 −99Browse files

Unified · Split

5 files
  1. block/blk-map.c +29 −87
  2. block/bounce.c +3 −8
  3. drivers/nvme/host/lightnvm.c +1 −1
  4. drivers/target/target_core_pscsi.c +2 −2
  5. include/linux/blkdev.h +1 −1
block/blk-map.c+29−87
@@ -123,7 +123,6 @@ static int bio_uncopy_user(struct bio *bio) · 122 unmodified lines
123123 bio_free_pages(bio);
124124 }
125125 kfree(bmd);
126− bio_put(bio);
127126 return ret;
128127 }
129128
@@ -132,7 +131,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, · 2 unmodified lines
132131 {
133132 struct bio_map_data *bmd;
134133 struct page *page;
135− struct bio *bio, *bounce_bio;
134+ struct bio *bio;
136135 int i = 0, ret;
137136 int nr_pages;
138137 unsigned int len = iter->count;
@@ -218,16 +217,9 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, · 79 unmodified lines
218217
219218 bio->bi_private = bmd;
220219
221− bounce_bio = bio;
222− ret = blk_rq_append_bio(rq, &bounce_bio);
220+ ret = blk_rq_append_bio(rq, bio);
223221 if (ret)
224222 goto cleanup;
225−
226− /*
227− * We link the bounce buffer in and could have to traverse it later, so
228− * we have to get a ref to prevent it from being freed
229− */
230− bio_get(bounce_bio);
231223 return 0;
232224 cleanup:
233225 if (!map_data)
@@ -242,7 +234,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, · 8 unmodified lines
242234 gfp_t gfp_mask)
243235 {
244236 unsigned int max_sectors = queue_max_hw_sectors(rq->q);
245− struct bio *bio, *bounce_bio;
237+ struct bio *bio;
246238 int ret;
247239 int j;
248240
@@ -304,49 +296,17 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, · 55 unmodified lines
304296 break;
305297 }
306298
307− /*
308− * Subtle: if we end up needing to bounce a bio, it would normally
309− * disappear when its bi_end_io is run. However, we need the original
310− * bio for the unmap, so grab an extra reference to it
311− */
312− bio_get(bio);
313−
314− bounce_bio = bio;
315− ret = blk_rq_append_bio(rq, &bounce_bio);
299+ ret = blk_rq_append_bio(rq, bio);
316300 if (ret)
317− goto out_put_orig;
318−
319− /*
320− * We link the bounce buffer in and could have to traverse it
321− * later, so we have to get a ref to prevent it from being freed
322− */
323− bio_get(bounce_bio);
301+ goto out_unmap;
324302 return 0;
325303
326− out_put_orig:
327− bio_put(bio);
328304 out_unmap:
329305 bio_release_pages(bio, false);
330306 bio_put(bio);
331307 return ret;
332308 }
333309
334−/**
335− * bio_unmap_user - unmap a bio
336− * @bio: the bio being unmapped
337− *
338− * Unmap a bio previously mapped by bio_map_user_iov(). Must be called from
339− * process context.
340− *
341− * bio_unmap_user() may sleep.
342− */
343−static void bio_unmap_user(struct bio *bio)
344−{
345− bio_release_pages(bio, bio_data_dir(bio) == READ);
346− bio_put(bio);
347− bio_put(bio);
348−}
349−
350310 static void bio_invalidate_vmalloc_pages(struct bio *bio)
351311 {
352312 #ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
@@ -519,33 +479,27 @@ cleanup: · 166 unmodified lines
519479 * Append a bio to a passthrough request. Only works if the bio can be merged
520480 * into the request based on the driver constraints.
521481 */
522−int blk_rq_append_bio(struct request *rq, struct bio **bio)
482+int blk_rq_append_bio(struct request *rq, struct bio *bio)
523483 {
524− struct bio *orig_bio = *bio;
525484 struct bvec_iter iter;
526485 struct bio_vec bv;
527486 unsigned int nr_segs = 0;
528487
529− blk_queue_bounce(rq->q, bio);
488+ if (WARN_ON_ONCE(rq->q->limits.bounce != BLK_BOUNCE_NONE))
489+ return -EINVAL;
530490
531− bio_for_each_bvec(bv, *bio, iter)
491+ bio_for_each_bvec(bv, bio, iter)
532492 nr_segs++;
533493
534494 if (!rq->bio) {
535− blk_rq_bio_prep(rq, *bio, nr_segs);
495+ blk_rq_bio_prep(rq, bio, nr_segs);
536496 } else {
537− if (!ll_back_merge_fn(rq, *bio, nr_segs)) {
538− if (orig_bio != *bio) {
539− bio_put(*bio);
540− *bio = orig_bio;
541− }
497+ if (!ll_back_merge_fn(rq, bio, nr_segs))
542498 return -EINVAL;
543− }
544−
545− rq->biotail->bi_next = *bio;
546− rq->biotail = *bio;
547− rq->__data_len += (*bio)->bi_iter.bi_size;
548− bio_crypt_free_ctx(*bio);
499+ rq->biotail->bi_next = bio;
500+ rq->biotail = bio;
501+ rq->__data_len += (bio)->bi_iter.bi_size;
502+ bio_crypt_free_ctx(bio);
549503 }
550504
551505 return 0;
@@ -566,12 +520,6 @@ EXPORT_SYMBOL(blk_rq_append_bio); · 14 unmodified lines
566520 *
567521 * A matching blk_rq_unmap_user() must be issued at the end of I/O, while
568522 * still in process context.
569− *
570− * Note: The mapped bio may need to be bounced through blk_queue_bounce()
571− * before being submitted to the device, as pages mapped may be out of
572− * reach. It's the callers responsibility to make sure this happens. The
573− * original bio must be passed back in to blk_rq_unmap_user() for proper
574− * unmapping.
575523 */
576524 int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
577525 struct rq_map_data *map_data,
@@ -588,6 +536,8 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, · 10 unmodified lines
588536
589537 if (map_data)
590538 copy = true;
539+ else if (blk_queue_may_bounce(q))
540+ copy = true;
591541 else if (iov_iter_alignment(iter) & align)
592542 copy = true;
593543 else if (queue_virt_boundary(q))
@@ -641,25 +591,21 @@ EXPORT_SYMBOL(blk_rq_map_user); · 47 unmodified lines
641591 */
642592 int blk_rq_unmap_user(struct bio *bio)
643593 {
644− struct bio *mapped_bio;
594+ struct bio *next_bio;
645595 int ret = 0, ret2;
646596
647597 while (bio) {
648− mapped_bio = bio;
649− if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
650− mapped_bio = bio->bi_private;
651−
652598 if (bio->bi_private) {
653− ret2 = bio_uncopy_user(mapped_bio);
599+ ret2 = bio_uncopy_user(bio);
654600 if (ret2 && !ret)
655601 ret = ret2;
656602 } else {
657− bio_unmap_user(mapped_bio);
603+ bio_release_pages(bio, bio_data_dir(bio) == READ);
658604 }
659605
660− mapped_bio = bio;
606+ next_bio = bio;
661607 bio = bio->bi_next;
662− bio_put(mapped_bio);
608+ bio_put(next_bio);
663609 }
664610
665611 return ret;
@@ -684,7 +630,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 18 unmodified lines
684630 {
685631 int reading = rq_data_dir(rq) == READ;
686632 unsigned long addr = (unsigned long) kbuf;
687− struct bio *bio, *orig_bio;
633+ struct bio *bio;
688634 int ret;
689635
690636 if (len > (queue_max_hw_sectors(q) << 9))
@@ -692,7 +638,8 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 1 unmodified line
692638 if (!len || !kbuf)
693639 return -EINVAL;
694640
695− if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf))
641+ if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) ||
642+ blk_queue_may_bounce(q))
696643 bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading);
697644 else
698645 bio = bio_map_kern(q, kbuf, len, gfp_mask);
@@ -703,14 +650,9 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 4 unmodified lines
703650 bio->bi_opf &= ~REQ_OP_MASK;
704651 bio->bi_opf |= req_op(rq);
705652
706− orig_bio = bio;
707− ret = blk_rq_append_bio(rq, &bio);
708− if (unlikely(ret)) {
709− /* request is too big */
710− bio_put(orig_bio);
711− return ret;
712− }
713−
714− return 0;
653+ ret = blk_rq_append_bio(rq, bio);
654+ if (unlikely(ret))
655+ bio_put(bio);
656+ return ret;
715657 }
716658 EXPORT_SYMBOL(blk_rq_map_kern);
block/bounce.c+3−8
@@ -180,12 +180,8 @@ static struct bio *bounce_clone_bio(struct bio *bio_src) · 179 unmodified lines
180180 * asking for trouble and would force extra work on
181181 * __bio_clone_fast() anyways.
182182 */
183− if (bio_is_passthrough(bio_src))
184− bio = bio_kmalloc(GFP_NOIO | __GFP_NOFAIL,
185− bio_segments(bio_src));
186− else
187− bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src),
188− &bounce_bio_set);
183+ bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src),
184+ &bounce_bio_set);
189185 bio->bi_bdev = bio_src->bi_bdev;
190186 if (bio_flagged(bio_src, BIO_REMAPPED))
191187 bio_set_flag(bio, BIO_REMAPPED);
@@ -245,8 +241,7 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · 53 unmodified lines
245241 if (!bounce)
246242 return;
247243
248− if (!bio_is_passthrough(*bio_orig) &&
249− sectors < bio_sectors(*bio_orig)) {
244+ if (sectors < bio_sectors(*bio_orig)) {
250245 bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
251246 bio_chain(bio, *bio_orig);
252247 submit_bio_noacct(*bio_orig);
drivers/nvme/host/lightnvm.c+1−1
@@ -660,7 +660,7 @@ static struct request *nvme_nvm_alloc_request(struct request_queue *q, · 659 unmodified lines
660660 rq->cmd_flags &= ~REQ_FAILFAST_DRIVER;
661661
662662 if (rqd->bio)
663− blk_rq_append_bio(rq, &rqd->bio);
663+ blk_rq_append_bio(rq, rqd->bio);
664664 else
665665 rq->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_NORM);
666666
drivers/target/target_core_pscsi.c+2−2
@@ -911,7 +911,7 @@ new_bio: · 910 unmodified lines
911911 " %d i: %d bio: %p, allocating another"
912912 " bio\n", bio->bi_vcnt, i, bio);
913913
914− rc = blk_rq_append_bio(req, &bio);
914+ rc = blk_rq_append_bio(req, bio);
915915 if (rc) {
916916 pr_err("pSCSI: failed to append bio\n");
917917 goto fail;
@@ -930,7 +930,7 @@ new_bio: · 12 unmodified lines
930930 }
931931
932932 if (bio) {
933− rc = blk_rq_append_bio(req, &bio);
933+ rc = blk_rq_append_bio(req, bio);
934934 if (rc) {
935935 pr_err("pSCSI: failed to append bio\n");
936936 goto fail;
include/linux/blkdev.h+1−1
@@ -909,7 +909,7 @@ extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src, · 908 unmodified lines
909909 extern void blk_rq_unprep_clone(struct request *rq);
910910 extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
911911 struct request *rq);
912−extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
912+int blk_rq_append_bio(struct request *rq, struct bio *bio);
913913 extern void blk_queue_split(struct bio **);
914914 extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
915915 extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t