| @@ -123,7 +123,6 @@ static int bio_uncopy_user(struct bio *bio) · 122 unmodified lines |
| 123 | bio_free_pages(bio); | 123 | bio_free_pages(bio); |
| 124 | } | 124 | } |
| 125 | kfree(bmd); | 125 | kfree(bmd); |
| 126 | bio_put(bio); | | |
| 127 | return ret; | 126 | return ret; |
| 128 | } | 127 | } |
| 129 | | 128 | |
| @@ -132,7 +131,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, · 2 unmodified lines |
| 132 | { | 131 | { |
| 133 | struct bio_map_data *bmd; | 132 | struct bio_map_data *bmd; |
| 134 | struct page *page; | 133 | struct page *page; |
| 135 | struct bio *bio, *bounce_bio; | 134 | struct bio *bio; |
| 136 | int i = 0, ret; | 135 | int i = 0, ret; |
| 137 | int nr_pages; | 136 | int nr_pages; |
| 138 | unsigned int len = iter->count; | 137 | unsigned int len = iter->count; |
| @@ -218,16 +217,9 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, · 79 unmodified lines |
| 218 | | 217 | |
| 219 | bio->bi_private = bmd; | 218 | bio->bi_private = bmd; |
| 220 | | 219 | |
| 221 | bounce_bio = bio; | 220 | ret = blk_rq_append_bio(rq, bio); |
| 222 | ret = blk_rq_append_bio(rq, &bounce_bio); | | |
| 223 | if (ret) | 221 | if (ret) |
| 224 | goto cleanup; | 222 | goto cleanup; |
| 225 | | | |
| 226 | /* | | |
| 227 | * We link the bounce buffer in and could have to traverse it later, so | | |
| 228 | * we have to get a ref to prevent it from being freed | | |
| 229 | */ | | |
| 230 | bio_get(bounce_bio); | | |
| 231 | return 0; | 223 | return 0; |
| 232 | cleanup: | 224 | cleanup: |
| 233 | if (!map_data) | 225 | if (!map_data) |
| @@ -242,7 +234,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, · 8 unmodified lines |
| 242 | gfp_t gfp_mask) | 234 | gfp_t gfp_mask) |
| 243 | { | 235 | { |
| 244 | unsigned int max_sectors = queue_max_hw_sectors(rq->q); | 236 | unsigned int max_sectors = queue_max_hw_sectors(rq->q); |
| 245 | struct bio *bio, *bounce_bio; | 237 | struct bio *bio; |
| 246 | int ret; | 238 | int ret; |
| 247 | int j; | 239 | int j; |
| 248 | | 240 | |
| @@ -304,49 +296,17 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, · 55 unmodified lines |
| 304 | break; | 296 | break; |
| 305 | } | 297 | } |
| 306 | | 298 | |
| 307 | /* | 299 | ret = blk_rq_append_bio(rq, bio); |
| 308 | * Subtle: if we end up needing to bounce a bio, it would normally | | |
| 309 | * disappear when its bi_end_io is run. However, we need the original | | |
| 310 | * bio for the unmap, so grab an extra reference to it | | |
| 311 | */ | | |
| 312 | bio_get(bio); | | |
| 313 | | | |
| 314 | bounce_bio = bio; | | |
| 315 | ret = blk_rq_append_bio(rq, &bounce_bio); | | |
| 316 | if (ret) | 300 | if (ret) |
| 317 | goto out_put_orig; | 301 | goto out_unmap; |
| 318 | | | |
| 319 | /* | | |
| 320 | * We link the bounce buffer in and could have to traverse it | | |
| 321 | * later, so we have to get a ref to prevent it from being freed | | |
| 322 | */ | | |
| 323 | bio_get(bounce_bio); | | |
| 324 | return 0; | 302 | return 0; |
| 325 | | 303 | |
| 326 | out_put_orig: | | |
| 327 | bio_put(bio); | | |
| 328 | out_unmap: | 304 | out_unmap: |
| 329 | bio_release_pages(bio, false); | 305 | bio_release_pages(bio, false); |
| 330 | bio_put(bio); | 306 | bio_put(bio); |
| 331 | return ret; | 307 | return ret; |
| 332 | } | 308 | } |
| 333 | | 309 | |
| 334 | /** | | |
| 335 | * bio_unmap_user - unmap a bio | | |
| 336 | * @bio: the bio being unmapped | | |
| 337 | * | | |
| 338 | * Unmap a bio previously mapped by bio_map_user_iov(). Must be called from | | |
| 339 | * process context. | | |
| 340 | * | | |
| 341 | * bio_unmap_user() may sleep. | | |
| 342 | */ | | |
| 343 | static void bio_unmap_user(struct bio *bio) | | |
| 344 | { | | |
| 345 | bio_release_pages(bio, bio_data_dir(bio) == READ); | | |
| 346 | bio_put(bio); | | |
| 347 | bio_put(bio); | | |
| 348 | } | | |
| 349 | | | |
| 350 | static void bio_invalidate_vmalloc_pages(struct bio *bio) | 310 | static void bio_invalidate_vmalloc_pages(struct bio *bio) |
| 351 | { | 311 | { |
| 352 | #ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE | 312 | #ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE |
| @@ -519,33 +479,27 @@ cleanup: · 166 unmodified lines |
| 519 | * Append a bio to a passthrough request. Only works if the bio can be merged | 479 | * Append a bio to a passthrough request. Only works if the bio can be merged |
| 520 | * into the request based on the driver constraints. | 480 | * into the request based on the driver constraints. |
| 521 | */ | 481 | */ |
| 522 | int blk_rq_append_bio(struct request *rq, struct bio **bio) | 482 | int blk_rq_append_bio(struct request *rq, struct bio *bio) |
| 523 | { | 483 | { |
| 524 | struct bio *orig_bio = *bio; | | |
| 525 | struct bvec_iter iter; | 484 | struct bvec_iter iter; |
| 526 | struct bio_vec bv; | 485 | struct bio_vec bv; |
| 527 | unsigned int nr_segs = 0; | 486 | unsigned int nr_segs = 0; |
| 528 | | 487 | |
| 529 | blk_queue_bounce(rq->q, bio); | 488 | if (WARN_ON_ONCE(rq->q->limits.bounce != BLK_BOUNCE_NONE)) |
| | 489 | return -EINVAL; |
| 530 | | 490 | |
| 531 | bio_for_each_bvec(bv, *bio, iter) | 491 | bio_for_each_bvec(bv, bio, iter) |
| 532 | nr_segs++; | 492 | nr_segs++; |
| 533 | | 493 | |
| 534 | if (!rq->bio) { | 494 | if (!rq->bio) { |
| 535 | blk_rq_bio_prep(rq, *bio, nr_segs); | 495 | blk_rq_bio_prep(rq, bio, nr_segs); |
| 536 | } else { | 496 | } else { |
| 537 | if (!ll_back_merge_fn(rq, *bio, nr_segs)) { | 497 | if (!ll_back_merge_fn(rq, bio, nr_segs)) |
| 538 | if (orig_bio != *bio) { | | |
| 539 | bio_put(*bio); | | |
| 540 | *bio = orig_bio; | | |
| 541 | } | | |
| 542 | return -EINVAL; | 498 | return -EINVAL; |
| 543 | } | 499 | rq->biotail->bi_next = bio; |
| 544 | | 500 | rq->biotail = bio; |
| 545 | rq->biotail->bi_next = *bio; | 501 | rq->__data_len += (bio)->bi_iter.bi_size; |
| 546 | rq->biotail = *bio; | 502 | bio_crypt_free_ctx(bio); |
| 547 | rq->__data_len += (*bio)->bi_iter.bi_size; | | |
| 548 | bio_crypt_free_ctx(*bio); | | |
| 549 | } | 503 | } |
| 550 | | 504 | |
| 551 | return 0; | 505 | return 0; |
| @@ -566,12 +520,6 @@ EXPORT_SYMBOL(blk_rq_append_bio); · 14 unmodified lines |
| 566 | * | 520 | * |
| 567 | * A matching blk_rq_unmap_user() must be issued at the end of I/O, while | 521 | * A matching blk_rq_unmap_user() must be issued at the end of I/O, while |
| 568 | * still in process context. | 522 | * still in process context. |
| 569 | * | | |
| 570 | * Note: The mapped bio may need to be bounced through blk_queue_bounce() | | |
| 571 | * before being submitted to the device, as pages mapped may be out of | | |
| 572 | * reach. It's the callers responsibility to make sure this happens. The | | |
| 573 | * original bio must be passed back in to blk_rq_unmap_user() for proper | | |
| 574 | * unmapping. | | |
| 575 | */ | 523 | */ |
| 576 | int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, | 524 | int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, |
| 577 | struct rq_map_data *map_data, | 525 | struct rq_map_data *map_data, |
| @@ -588,6 +536,8 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, · 10 unmodified lines |
| 588 | | 536 | |
| 589 | if (map_data) | 537 | if (map_data) |
| 590 | copy = true; | 538 | copy = true; |
| | 539 | else if (blk_queue_may_bounce(q)) |
| | 540 | copy = true; |
| 591 | else if (iov_iter_alignment(iter) & align) | 541 | else if (iov_iter_alignment(iter) & align) |
| 592 | copy = true; | 542 | copy = true; |
| 593 | else if (queue_virt_boundary(q)) | 543 | else if (queue_virt_boundary(q)) |
| @@ -641,25 +591,21 @@ EXPORT_SYMBOL(blk_rq_map_user); · 47 unmodified lines |
| 641 | */ | 591 | */ |
| 642 | int blk_rq_unmap_user(struct bio *bio) | 592 | int blk_rq_unmap_user(struct bio *bio) |
| 643 | { | 593 | { |
| 644 | struct bio *mapped_bio; | 594 | struct bio *next_bio; |
| 645 | int ret = 0, ret2; | 595 | int ret = 0, ret2; |
| 646 | | 596 | |
| 647 | while (bio) { | 597 | while (bio) { |
| 648 | mapped_bio = bio; | | |
| 649 | if (unlikely(bio_flagged(bio, BIO_BOUNCED))) | | |
| 650 | mapped_bio = bio->bi_private; | | |
| 651 | | | |
| 652 | if (bio->bi_private) { | 598 | if (bio->bi_private) { |
| 653 | ret2 = bio_uncopy_user(mapped_bio); | 599 | ret2 = bio_uncopy_user(bio); |
| 654 | if (ret2 && !ret) | 600 | if (ret2 && !ret) |
| 655 | ret = ret2; | 601 | ret = ret2; |
| 656 | } else { | 602 | } else { |
| 657 | bio_unmap_user(mapped_bio); | 603 | bio_release_pages(bio, bio_data_dir(bio) == READ); |
| 658 | } | 604 | } |
| 659 | | 605 | |
| 660 | mapped_bio = bio; | 606 | next_bio = bio; |
| 661 | bio = bio->bi_next; | 607 | bio = bio->bi_next; |
| 662 | bio_put(mapped_bio); | 608 | bio_put(next_bio); |
| 663 | } | 609 | } |
| 664 | | 610 | |
| 665 | return ret; | 611 | return ret; |
| @@ -684,7 +630,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 18 unmodified lines |
| 684 | { | 630 | { |
| 685 | int reading = rq_data_dir(rq) == READ; | 631 | int reading = rq_data_dir(rq) == READ; |
| 686 | unsigned long addr = (unsigned long) kbuf; | 632 | unsigned long addr = (unsigned long) kbuf; |
| 687 | struct bio *bio, *orig_bio; | 633 | struct bio *bio; |
| 688 | int ret; | 634 | int ret; |
| 689 | | 635 | |
| 690 | if (len > (queue_max_hw_sectors(q) << 9)) | 636 | if (len > (queue_max_hw_sectors(q) << 9)) |
| @@ -692,7 +638,8 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 1 unmodified line |
| 692 | if (!len || !kbuf) | 638 | if (!len || !kbuf) |
| 693 | return -EINVAL; | 639 | return -EINVAL; |
| 694 | | 640 | |
| 695 | if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf)) | 641 | if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || |
| | 642 | blk_queue_may_bounce(q)) |
| 696 | bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); | 643 | bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); |
| 697 | else | 644 | else |
| 698 | bio = bio_map_kern(q, kbuf, len, gfp_mask); | 645 | bio = bio_map_kern(q, kbuf, len, gfp_mask); |
| @@ -703,14 +650,9 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 4 unmodified lines |
| 703 | bio->bi_opf &= ~REQ_OP_MASK; | 650 | bio->bi_opf &= ~REQ_OP_MASK; |
| 704 | bio->bi_opf |= req_op(rq); | 651 | bio->bi_opf |= req_op(rq); |
| 705 | | 652 | |
| 706 | orig_bio = bio; | 653 | ret = blk_rq_append_bio(rq, bio); |
| 707 | ret = blk_rq_append_bio(rq, &bio); | 654 | if (unlikely(ret)) |
| 708 | if (unlikely(ret)) { | 655 | bio_put(bio); |
| 709 | /* request is too big */ | 656 | return ret; |
| 710 | bio_put(orig_bio); | | |
| 711 | return ret; | | |
| 712 | } | | |
| 713 | | | |
| 714 | return 0; | | |
| 715 | } | 657 | } |
| 716 | EXPORT_SYMBOL(blk_rq_map_kern); | 658 | EXPORT_SYMBOL(blk_rq_map_kern); |