| @@ -123,7 +123,6 @@ static int bio_uncopy_user(struct bio *bio) · 122 unmodified lines |
| 123 | 123 | bio_free_pages(bio); |
| 124 | 124 | } |
| 125 | 125 | kfree(bmd); |
| 126 | | − bio_put(bio); |
| 127 | 126 | return ret; |
| 128 | 127 | } |
| 129 | 128 | |
| @@ -132,7 +131,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, · 2 unmodified lines |
| 132 | 131 | { |
| 133 | 132 | struct bio_map_data *bmd; |
| 134 | 133 | struct page *page; |
| 135 | | − struct bio *bio, *bounce_bio; |
| 134 | + struct bio *bio; |
| 136 | 135 | int i = 0, ret; |
| 137 | 136 | int nr_pages; |
| 138 | 137 | unsigned int len = iter->count; |
| @@ -218,16 +217,9 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, · 79 unmodified lines |
| 218 | 217 | |
| 219 | 218 | bio->bi_private = bmd; |
| 220 | 219 | |
| 221 | | − bounce_bio = bio; |
| 222 | | − ret = blk_rq_append_bio(rq, &bounce_bio); |
| 220 | + ret = blk_rq_append_bio(rq, bio); |
| 223 | 221 | if (ret) |
| 224 | 222 | goto cleanup; |
| 225 | | − |
| 226 | | − /* |
| 227 | | − * We link the bounce buffer in and could have to traverse it later, so |
| 228 | | − * we have to get a ref to prevent it from being freed |
| 229 | | − */ |
| 230 | | − bio_get(bounce_bio); |
| 231 | 223 | return 0; |
| 232 | 224 | cleanup: |
| 233 | 225 | if (!map_data) |
| @@ -242,7 +234,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, · 8 unmodified lines |
| 242 | 234 | gfp_t gfp_mask) |
| 243 | 235 | { |
| 244 | 236 | unsigned int max_sectors = queue_max_hw_sectors(rq->q); |
| 245 | | − struct bio *bio, *bounce_bio; |
| 237 | + struct bio *bio; |
| 246 | 238 | int ret; |
| 247 | 239 | int j; |
| 248 | 240 | |
| @@ -304,49 +296,17 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, · 55 unmodified lines |
| 304 | 296 | break; |
| 305 | 297 | } |
| 306 | 298 | |
| 307 | | − /* |
| 308 | | − * Subtle: if we end up needing to bounce a bio, it would normally |
| 309 | | − * disappear when its bi_end_io is run. However, we need the original |
| 310 | | − * bio for the unmap, so grab an extra reference to it |
| 311 | | − */ |
| 312 | | − bio_get(bio); |
| 313 | | − |
| 314 | | − bounce_bio = bio; |
| 315 | | − ret = blk_rq_append_bio(rq, &bounce_bio); |
| 299 | + ret = blk_rq_append_bio(rq, bio); |
| 316 | 300 | if (ret) |
| 317 | | − goto out_put_orig; |
| 318 | | − |
| 319 | | − /* |
| 320 | | − * We link the bounce buffer in and could have to traverse it |
| 321 | | − * later, so we have to get a ref to prevent it from being freed |
| 322 | | − */ |
| 323 | | − bio_get(bounce_bio); |
| 301 | + goto out_unmap; |
| 324 | 302 | return 0; |
| 325 | 303 | |
| 326 | | − out_put_orig: |
| 327 | | − bio_put(bio); |
| 328 | 304 | out_unmap: |
| 329 | 305 | bio_release_pages(bio, false); |
| 330 | 306 | bio_put(bio); |
| 331 | 307 | return ret; |
| 332 | 308 | } |
| 333 | 309 | |
| 334 | | −/** |
| 335 | | − * bio_unmap_user - unmap a bio |
| 336 | | − * @bio: the bio being unmapped |
| 337 | | − * |
| 338 | | − * Unmap a bio previously mapped by bio_map_user_iov(). Must be called from |
| 339 | | − * process context. |
| 340 | | − * |
| 341 | | − * bio_unmap_user() may sleep. |
| 342 | | − */ |
| 343 | | −static void bio_unmap_user(struct bio *bio) |
| 344 | | −{ |
| 345 | | − bio_release_pages(bio, bio_data_dir(bio) == READ); |
| 346 | | − bio_put(bio); |
| 347 | | − bio_put(bio); |
| 348 | | −} |
| 349 | | − |
| 350 | 310 | static void bio_invalidate_vmalloc_pages(struct bio *bio) |
| 351 | 311 | { |
| 352 | 312 | #ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE |
| @@ -519,33 +479,27 @@ cleanup: · 166 unmodified lines |
| 519 | 479 | * Append a bio to a passthrough request. Only works if the bio can be merged |
| 520 | 480 | * into the request based on the driver constraints. |
| 521 | 481 | */ |
| 522 | | −int blk_rq_append_bio(struct request *rq, struct bio **bio) |
| 482 | +int blk_rq_append_bio(struct request *rq, struct bio *bio) |
| 523 | 483 | { |
| 524 | | − struct bio *orig_bio = *bio; |
| 525 | 484 | struct bvec_iter iter; |
| 526 | 485 | struct bio_vec bv; |
| 527 | 486 | unsigned int nr_segs = 0; |
| 528 | 487 | |
| 529 | | − blk_queue_bounce(rq->q, bio); |
| 488 | + if (WARN_ON_ONCE(rq->q->limits.bounce != BLK_BOUNCE_NONE)) |
| 489 | + return -EINVAL; |
| 530 | 490 | |
| 531 | | − bio_for_each_bvec(bv, *bio, iter) |
| 491 | + bio_for_each_bvec(bv, bio, iter) |
| 532 | 492 | nr_segs++; |
| 533 | 493 | |
| 534 | 494 | if (!rq->bio) { |
| 535 | | − blk_rq_bio_prep(rq, *bio, nr_segs); |
| 495 | + blk_rq_bio_prep(rq, bio, nr_segs); |
| 536 | 496 | } else { |
| 537 | | − if (!ll_back_merge_fn(rq, *bio, nr_segs)) { |
| 538 | | − if (orig_bio != *bio) { |
| 539 | | − bio_put(*bio); |
| 540 | | − *bio = orig_bio; |
| 541 | | − } |
| 497 | + if (!ll_back_merge_fn(rq, bio, nr_segs)) |
| 542 | 498 | return -EINVAL; |
| 543 | | − } |
| 544 | | − |
| 545 | | − rq->biotail->bi_next = *bio; |
| 546 | | − rq->biotail = *bio; |
| 547 | | − rq->__data_len += (*bio)->bi_iter.bi_size; |
| 548 | | − bio_crypt_free_ctx(*bio); |
| 499 | + rq->biotail->bi_next = bio; |
| 500 | + rq->biotail = bio; |
| 501 | + rq->__data_len += (bio)->bi_iter.bi_size; |
| 502 | + bio_crypt_free_ctx(bio); |
| 549 | 503 | } |
| 550 | 504 | |
| 551 | 505 | return 0; |
| @@ -566,12 +520,6 @@ EXPORT_SYMBOL(blk_rq_append_bio); · 14 unmodified lines |
| 566 | 520 | * |
| 567 | 521 | * A matching blk_rq_unmap_user() must be issued at the end of I/O, while |
| 568 | 522 | * still in process context. |
| 569 | | − * |
| 570 | | − * Note: The mapped bio may need to be bounced through blk_queue_bounce() |
| 571 | | − * before being submitted to the device, as pages mapped may be out of |
| 572 | | − * reach. It's the callers responsibility to make sure this happens. The |
| 573 | | − * original bio must be passed back in to blk_rq_unmap_user() for proper |
| 574 | | − * unmapping. |
| 575 | 523 | */ |
| 576 | 524 | int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, |
| 577 | 525 | struct rq_map_data *map_data, |
| @@ -588,6 +536,8 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, · 10 unmodified lines |
| 588 | 536 | |
| 589 | 537 | if (map_data) |
| 590 | 538 | copy = true; |
| 539 | + else if (blk_queue_may_bounce(q)) |
| 540 | + copy = true; |
| 591 | 541 | else if (iov_iter_alignment(iter) & align) |
| 592 | 542 | copy = true; |
| 593 | 543 | else if (queue_virt_boundary(q)) |
| @@ -641,25 +591,21 @@ EXPORT_SYMBOL(blk_rq_map_user); · 47 unmodified lines |
| 641 | 591 | */ |
| 642 | 592 | int blk_rq_unmap_user(struct bio *bio) |
| 643 | 593 | { |
| 644 | | − struct bio *mapped_bio; |
| 594 | + struct bio *next_bio; |
| 645 | 595 | int ret = 0, ret2; |
| 646 | 596 | |
| 647 | 597 | while (bio) { |
| 648 | | − mapped_bio = bio; |
| 649 | | − if (unlikely(bio_flagged(bio, BIO_BOUNCED))) |
| 650 | | − mapped_bio = bio->bi_private; |
| 651 | | − |
| 652 | 598 | if (bio->bi_private) { |
| 653 | | − ret2 = bio_uncopy_user(mapped_bio); |
| 599 | + ret2 = bio_uncopy_user(bio); |
| 654 | 600 | if (ret2 && !ret) |
| 655 | 601 | ret = ret2; |
| 656 | 602 | } else { |
| 657 | | − bio_unmap_user(mapped_bio); |
| 603 | + bio_release_pages(bio, bio_data_dir(bio) == READ); |
| 658 | 604 | } |
| 659 | 605 | |
| 660 | | − mapped_bio = bio; |
| 606 | + next_bio = bio; |
| 661 | 607 | bio = bio->bi_next; |
| 662 | | − bio_put(mapped_bio); |
| 608 | + bio_put(next_bio); |
| 663 | 609 | } |
| 664 | 610 | |
| 665 | 611 | return ret; |
| @@ -684,7 +630,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 18 unmodified lines |
| 684 | 630 | { |
| 685 | 631 | int reading = rq_data_dir(rq) == READ; |
| 686 | 632 | unsigned long addr = (unsigned long) kbuf; |
| 687 | | − struct bio *bio, *orig_bio; |
| 633 | + struct bio *bio; |
| 688 | 634 | int ret; |
| 689 | 635 | |
| 690 | 636 | if (len > (queue_max_hw_sectors(q) << 9)) |
| @@ -692,7 +638,8 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 1 unmodified line |
| 692 | 638 | if (!len || !kbuf) |
| 693 | 639 | return -EINVAL; |
| 694 | 640 | |
| 695 | | − if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf)) |
| 641 | + if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || |
| 642 | + blk_queue_may_bounce(q)) |
| 696 | 643 | bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); |
| 697 | 644 | else |
| 698 | 645 | bio = bio_map_kern(q, kbuf, len, gfp_mask); |
| @@ -703,14 +650,9 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · 4 unmodified lines |
| 703 | 650 | bio->bi_opf &= ~REQ_OP_MASK; |
| 704 | 651 | bio->bi_opf |= req_op(rq); |
| 705 | 652 | |
| 706 | | − orig_bio = bio; |
| 707 | | − ret = blk_rq_append_bio(rq, &bio); |
| 708 | | − if (unlikely(ret)) { |
| 709 | | − /* request is too big */ |
| 710 | | − bio_put(orig_bio); |
| 711 | | − return ret; |
| 712 | | − } |
| 713 | | − |
| 714 | | − return 0; |
| 653 | + ret = blk_rq_append_bio(rq, bio); |
| 654 | + if (unlikely(ret)) |
| 655 | + bio_put(bio); |
| 656 | + return ret; |
| 715 | 657 | } |
| 716 | 658 | EXPORT_SYMBOL(blk_rq_map_kern); |