History

block: stop calling blk_queue_bounce for passthrough requests

Instead of overloading the passthrough fast path with the deprecated block layer bounce buffering let the users that combine an old undermaintained driver with a highmem system pay the price by always falling back to copies in that case.

Christoph Hellwig committed 5 years ago

Commit 393bb121 parent 9bb33f25 files · +36 −99Browse files

Unified · Split

5 files
  1. block/blk-map.c +29 −87
  2. block/bounce.c +3 −8
  3. drivers/nvme/host/lightnvm.c +1 −1
  4. drivers/target/target_core_pscsi.c +2 −2
  5. include/linux/blkdev.h +1 −1
block/blk-map.c+29−87
@@ -123,7 +123,6 @@ static int bio_uncopy_user(struct bio *bio) · 122 unmodified lines
123 bio_free_pages(bio);123 bio_free_pages(bio);
124 }124 }
125 kfree(bmd);125 kfree(bmd);
126 bio_put(bio);
127 return ret;126 return ret;
128}127}
129 128
@@ -132,7 +131,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, · 2 unmodified lines
132{131{
133 struct bio_map_data *bmd;132 struct bio_map_data *bmd;
134 struct page *page;133 struct page *page;
135 struct bio *bio, *bounce_bio;134 struct bio *bio;
136 int i = 0, ret;135 int i = 0, ret;
137 int nr_pages;136 int nr_pages;
138 unsigned int len = iter->count;137 unsigned int len = iter->count;
@@ -218,16 +217,9 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, · 79 unmodified lines
218 217
219 bio->bi_private = bmd;218 bio->bi_private = bmd;
220 219
221 bounce_bio = bio;220 ret = blk_rq_append_bio(rq, bio);
222 ret = blk_rq_append_bio(rq, &bounce_bio);
223 if (ret)221 if (ret)
224 goto cleanup;222 goto cleanup;
225
226 /*
227 * We link the bounce buffer in and could have to traverse it later, so
228 * we have to get a ref to prevent it from being freed
229 */
230 bio_get(bounce_bio);
231 return 0;223 return 0;
232cleanup:224cleanup:
233 if (!map_data)225 if (!map_data)
@@ -242,7 +234,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, · 8 unmodified lines
242 gfp_t gfp_mask)234 gfp_t gfp_mask)
243{235{
244 unsigned int max_sectors = queue_max_hw_sectors(rq->q);236 unsigned int max_sectors = queue_max_hw_sectors(rq->q);
245 struct bio *bio, *bounce_bio;237 struct bio *bio;
246 int ret;238 int ret;
247 int j;239 int j;
248 240
@@ -304,49 +296,17 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, · 55 unmodified lines
304 break;296 break;
305 }297 }
306 298
307 /*299 ret = blk_rq_append_bio(rq, bio);
308 * Subtle: if we end up needing to bounce a bio, it would normally
309 * disappear when its bi_end_io is run. However, we need the original
310 * bio for the unmap, so grab an extra reference to it
311 */
312 bio_get(bio);
313
314 bounce_bio = bio;
315 ret = blk_rq_append_bio(rq, &bounce_bio);
316 if (ret)300 if (ret)
317 goto out_put_orig;301 goto out_unmap;
318
319 /*
320 * We link the bounce buffer in and could have to traverse it
321 * later, so we have to get a ref to prevent it from being freed
322 */
323 bio_get(bounce_bio);
324 return 0;302 return 0;
325 303
326 out_put_orig:
327 bio_put(bio);
328 out_unmap:304 out_unmap:
329 bio_release_pages(bio, false);305 bio_release_pages(bio, false);
330 bio_put(bio);306 bio_put(bio);
331 return ret;307 return ret;
332}308}
333 309
334/**
335 * bio_unmap_user - unmap a bio
336 * @bio: the bio being unmapped
337 *
338 * Unmap a bio previously mapped by bio_map_user_iov(). Must be called from
339 * process context.
340 *
341 * bio_unmap_user() may sleep.
342 */
343static void bio_unmap_user(struct bio *bio)
344{
345 bio_release_pages(bio, bio_data_dir(bio) == READ);
346 bio_put(bio);
347 bio_put(bio);
348}
349
350static void bio_invalidate_vmalloc_pages(struct bio *bio)310static void bio_invalidate_vmalloc_pages(struct bio *bio)
351{311{
352#ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE312#ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
@@ -519,33 +479,27 @@ cleanup: · 166 unmodified lines
519 * Append a bio to a passthrough request. Only works if the bio can be merged479 * Append a bio to a passthrough request. Only works if the bio can be merged
520 * into the request based on the driver constraints.480 * into the request based on the driver constraints.
521 */481 */
522int blk_rq_append_bio(struct request *rq, struct bio **bio)482int blk_rq_append_bio(struct request *rq, struct bio *bio)
523{483{
524 struct bio *orig_bio = *bio;
525 struct bvec_iter iter;484 struct bvec_iter iter;
526 struct bio_vec bv;485 struct bio_vec bv;
527 unsigned int nr_segs = 0;486 unsigned int nr_segs = 0;
528 487
529 blk_queue_bounce(rq->q, bio);488 if (WARN_ON_ONCE(rq->q->limits.bounce != BLK_BOUNCE_NONE))
489 return -EINVAL;
530 490
531 bio_for_each_bvec(bv, *bio, iter)491 bio_for_each_bvec(bv, bio, iter)
532 nr_segs++;492 nr_segs++;
533 493
534 if (!rq->bio) {494 if (!rq->bio) {
535 blk_rq_bio_prep(rq, *bio, nr_segs);495 blk_rq_bio_prep(rq, bio, nr_segs);
536 } else {496 } else {
537 if (!ll_back_merge_fn(rq, *bio, nr_segs)) {497 if (!ll_back_merge_fn(rq, bio, nr_segs))
538 if (orig_bio != *bio) {
539 bio_put(*bio);
540 *bio = orig_bio;
541 }
542 return -EINVAL;498 return -EINVAL;
543 }499 rq->biotail->bi_next = bio;
544 500 rq->biotail = bio;
545 rq->biotail->bi_next = *bio;501 rq->__data_len += (bio)->bi_iter.bi_size;
546 rq->biotail = *bio;502 bio_crypt_free_ctx(bio);
547 rq->__data_len += (*bio)->bi_iter.bi_size;
548 bio_crypt_free_ctx(*bio);
549 }503 }
550 504
551 return 0;505 return 0;
@@ -566,12 +520,6 @@ EXPORT_SYMBOL(blk_rq_append_bio); · 14 unmodified lines
566 *520 *
567 * A matching blk_rq_unmap_user() must be issued at the end of I/O, while521 * A matching blk_rq_unmap_user() must be issued at the end of I/O, while
568 * still in process context.522 * still in process context.
569 *
570 * Note: The mapped bio may need to be bounced through blk_queue_bounce()
571 * before being submitted to the device, as pages mapped may be out of
572 * reach. It's the callers responsibility to make sure this happens. The
573 * original bio must be passed back in to blk_rq_unmap_user() for proper
574 * unmapping.
575 */523 */
576int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,524int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
577 struct rq_map_data *map_data,525 struct rq_map_data *map_data,
@@ -588,6 +536,8 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, · 10 unmodified lines
588 536
589 if (map_data)537 if (map_data)
590 copy = true;538 copy = true;
539 else if (blk_queue_may_bounce(q))
540 copy = true;
591 else if (iov_iter_alignment(iter) & align)541 else if (iov_iter_alignment(iter) & align)
592 copy = true;542 copy = true;
593 else if (queue_virt_boundary(q))543 else if (queue_virt_boundary(q))
@@ -641,25 +591,21 @@ EXPORT_SYMBOL(blk_rq_map_user); · 47 unmodified lines
641 */591 */
642int blk_rq_unmap_user(struct bio *bio)592int blk_rq_unmap_user(struct bio *bio)
643{593{
644 struct bio *mapped_bio;594 struct bio *next_bio;
645 int ret = 0, ret2;595 int ret = 0, ret2;
646 596
647 while (bio) {597 while (bio) {
648 mapped_bio = bio;
649 if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
650 mapped_bio = bio->bi_private;
651
652 if (bio->bi_private) {598 if (bio->bi_private) {
653 ret2 = bio_uncopy_user(mapped_bio);599 ret2 = bio_uncopy_user(bio);
654 if (ret2 && !ret)600 if (ret2 && !ret)
655 ret = ret2;601 ret = ret2;
656 } else {602 } else {
657 bio_unmap_user(mapped_bio);603 bio_release_pages(bio, bio_data_dir(bio) == READ);
658 }604 }
659 605
660 mapped_bio = bio;606 next_bio = bio;
661 bio = bio->bi_next;607 bio = bio->bi_next;
662 bio_put(mapped_bio);608 bio_put(next_bio);
663 }609 }
664 610
665 return ret;611 return ret;
@@ -684,7 +630,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 18 unmodified lines
684{630{
685 int reading = rq_data_dir(rq) == READ;631 int reading = rq_data_dir(rq) == READ;
686 unsigned long addr = (unsigned long) kbuf;632 unsigned long addr = (unsigned long) kbuf;
687 struct bio *bio, *orig_bio;633 struct bio *bio;
688 int ret;634 int ret;
689 635
690 if (len > (queue_max_hw_sectors(q) << 9))636 if (len > (queue_max_hw_sectors(q) << 9))
@@ -692,7 +638,8 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 1 unmodified line
692 if (!len || !kbuf)638 if (!len || !kbuf)
693 return -EINVAL;639 return -EINVAL;
694 640
695 if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf))641 if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) ||
642 blk_queue_may_bounce(q))
696 bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading);643 bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading);
697 else644 else
698 bio = bio_map_kern(q, kbuf, len, gfp_mask);645 bio = bio_map_kern(q, kbuf, len, gfp_mask);
@@ -703,14 +650,9 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 4 unmodified lines
703 bio->bi_opf &= ~REQ_OP_MASK;650 bio->bi_opf &= ~REQ_OP_MASK;
704 bio->bi_opf |= req_op(rq);651 bio->bi_opf |= req_op(rq);
705 652
706 orig_bio = bio;653 ret = blk_rq_append_bio(rq, bio);
707 ret = blk_rq_append_bio(rq, &bio);654 if (unlikely(ret))
708 if (unlikely(ret)) {655 bio_put(bio);
709 /* request is too big */656 return ret;
710 bio_put(orig_bio);
711 return ret;
712 }
713
714 return 0;
715}657}
716EXPORT_SYMBOL(blk_rq_map_kern);658EXPORT_SYMBOL(blk_rq_map_kern);
block/bounce.c+3−8
@@ -180,12 +180,8 @@ static struct bio *bounce_clone_bio(struct bio *bio_src) · 179 unmodified lines
180 * asking for trouble and would force extra work on180 * asking for trouble and would force extra work on
181 * __bio_clone_fast() anyways.181 * __bio_clone_fast() anyways.
182 */182 */
183 if (bio_is_passthrough(bio_src))183 bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src),
184 bio = bio_kmalloc(GFP_NOIO | __GFP_NOFAIL,184 &bounce_bio_set);
185 bio_segments(bio_src));
186 else
187 bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src),
188 &bounce_bio_set);
189 bio->bi_bdev = bio_src->bi_bdev;185 bio->bi_bdev = bio_src->bi_bdev;
190 if (bio_flagged(bio_src, BIO_REMAPPED))186 if (bio_flagged(bio_src, BIO_REMAPPED))
191 bio_set_flag(bio, BIO_REMAPPED);187 bio_set_flag(bio, BIO_REMAPPED);
@@ -245,8 +241,7 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · 53 unmodified lines
245 if (!bounce)241 if (!bounce)
246 return;242 return;
247 243
248 if (!bio_is_passthrough(*bio_orig) &&244 if (sectors < bio_sectors(*bio_orig)) {
249 sectors < bio_sectors(*bio_orig)) {
250 bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);245 bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
251 bio_chain(bio, *bio_orig);246 bio_chain(bio, *bio_orig);
252 submit_bio_noacct(*bio_orig);247 submit_bio_noacct(*bio_orig);
drivers/nvme/host/lightnvm.c+1−1
@@ -660,7 +660,7 @@ static struct request *nvme_nvm_alloc_request(struct request_queue *q, · 659 unmodified lines
660 rq->cmd_flags &= ~REQ_FAILFAST_DRIVER;660 rq->cmd_flags &= ~REQ_FAILFAST_DRIVER;
661 661
662 if (rqd->bio)662 if (rqd->bio)
663 blk_rq_append_bio(rq, &rqd->bio);663 blk_rq_append_bio(rq, rqd->bio);
664 else664 else
665 rq->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_NORM);665 rq->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_NORM);
666 666
drivers/target/target_core_pscsi.c+2−2
@@ -911,7 +911,7 @@ new_bio: · 910 unmodified lines
911 " %d i: %d bio: %p, allocating another"911 " %d i: %d bio: %p, allocating another"
912 " bio\n", bio->bi_vcnt, i, bio);912 " bio\n", bio->bi_vcnt, i, bio);
913 913
914 rc = blk_rq_append_bio(req, &bio);914 rc = blk_rq_append_bio(req, bio);
915 if (rc) {915 if (rc) {
916 pr_err("pSCSI: failed to append bio\n");916 pr_err("pSCSI: failed to append bio\n");
917 goto fail;917 goto fail;
@@ -930,7 +930,7 @@ new_bio: · 12 unmodified lines
930 }930 }
931 931
932 if (bio) {932 if (bio) {
933 rc = blk_rq_append_bio(req, &bio);933 rc = blk_rq_append_bio(req, bio);
934 if (rc) {934 if (rc) {
935 pr_err("pSCSI: failed to append bio\n");935 pr_err("pSCSI: failed to append bio\n");
936 goto fail;936 goto fail;
include/linux/blkdev.h+1−1
@@ -909,7 +909,7 @@ extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src, · 908 unmodified lines
909extern void blk_rq_unprep_clone(struct request *rq);909extern void blk_rq_unprep_clone(struct request *rq);
910extern blk_status_t blk_insert_cloned_request(struct request_queue *q,910extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
911 struct request *rq);911 struct request *rq);
912extern int blk_rq_append_bio(struct request *rq, struct bio **bio);912int blk_rq_append_bio(struct request *rq, struct bio *bio);
913extern void blk_queue_split(struct bio **);913extern void blk_queue_split(struct bio **);
914extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);914extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
915extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,915extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t