History

Merge tag 'for-5.12/block-2021-02-17' of git://git.kernel.dk/linux-block

Pull core block updates from Jens Axboe: "Another nice round of removing more code than what is added, mostly due to Christoph's relentless pursuit of tech debt removal/cleanups. This pull request contains: - Two series of BFQ improvements (Paolo, Jan, Jia) - Block iov_iter improvements (Pavel) - bsg error path fix (Pan) - blk-mq scheduler improvements (Jan) - -EBUSY discard fix (Jan) - bvec allocation improvements (Ming, Christoph) - bio allocation and init improvements (Christoph) - Store bdev pointer in bio instead of gendisk + partno (Christoph) - Block trace point cleanups (Christoph) - hard read-only vs read-only split (Christoph) - Block based swap cleanups (Christoph) - Zoned write granularity support (Damien) - Various fixes/tweaks (Chunguang, Guoqing, Lei, Lukas, Huhai)" * tag 'for-5.12/block-2021-02-17' of git://git.kernel.dk/linux-block: (104 commits) mm: simplify swapdev_block sd_zbc: clear zone resources for non-zoned case block: introduce blk_queue_clear_zone_settings() zonefs: use zone write granularity as block size block: introduce zone_write_granularity limit block: use blk_queue_set_zoned in add_partition() nullb: use blk_queue_set_zoned() to setup zoned devices nvme: cleanup zone information initialization block: document zone_append_max_bytes attribute block: use bi_max_vecs to find the bvec pool md/raid10: remove dead code in reshape_request block: mark the bio as cloned in bio_iov_bvec_set block: set BIO_NO_PAGE_REF in bio_iov_bvec_set block: remove a layer of indentation in bio_iov_iter_get_pages block: turn the nr_iovecs argument to bio_alloc* into an unsigned short block: remove the 1 and 4 vec bvec_slabs entries block: streamline bvec_alloc block: factor out a bvec_alloc_gfp helper block: move struct biovec_slab to bio.c block: reuse BIO_INLINE_VECS for integrity bvecs ...

Linus Torvalds committed 5 years ago

Commit 582cd912 parents bd018bb f885056148 files · +1343 −1608Browse files

Unified · Split

148 files
  1. Documentation/block/biovecs.rst +2 −0
  2. Documentation/block/queue-sysfs.rst +13 −0
  3. Documentation/filesystems/f2fs.rst +0 −1
  4. Documentation/filesystems/porting.rst +16 −0
  5. arch/m68k/emu/nfblock.c +1 −1
  6. arch/xtensa/platforms/iss/simdisk.c +1 −1
  7. block/bfq-iosched.c +276 −169
  8. block/bfq-iosched.h +19 −10
  9. block/bfq-wf2q.c +0 −3
  10. block/bio-integrity.c +15 −20
  11. block/bio.c +253 −318
  12. block/blk-cgroup.c +12 −10
  13. block/blk-core.c +46 −53
  14. block/blk-crypto-fallback.c +4 −2
  15. block/blk-crypto.c +1 −1
  16. block/blk-exec.c +5 −9
  17. block/blk-flush.c +6 −11
  18. block/blk-merge.c +8 −9
  19. block/blk-mq.c +61 −8
  20. block/blk-settings.c +39 −2
  21. block/blk-sysfs.c +8 −0
  22. block/blk-throttle.c +1 −1
  23. block/blk-wbt.c +2 −2
  24. block/blk-zoned.c +17 −0
  25. block/blk.h +7 −5
  26. block/bounce.c +3 −1
  27. block/bsg.c +4 −2
  28. block/genhd.c +49 −257
  29. block/kyber-iosched.c +1 −0
  30. block/mq-deadline.c +0 −6
  31. block/partitions/core.c +8 −28
  32. block/scsi_ioctl.c +3 −3
  33. drivers/block/brd.c +2 −6
  34. drivers/block/drbd/drbd_actlog.c +1 −1
  35. drivers/block/drbd/drbd_bitmap.c +1 −1
  36. drivers/block/drbd/drbd_int.h +2 −4
  37. drivers/block/drbd/drbd_main.c +0 −13
  38. drivers/block/drbd/drbd_req.c +5 −2
  39. drivers/block/drbd/drbd_req.h +0 −12
  40. drivers/block/drbd/drbd_worker.c +4 −1
  41. drivers/block/mtip32xx/mtip32xx.c +1 −1
  42. drivers/block/null_blk/main.c +1 −1
  43. drivers/block/null_blk/zoned.c +4 −4
  44. drivers/block/paride/pd.c +1 −1
  45. drivers/block/pktcdvd.c +3 −3
  46. drivers/block/ps3vram.c +1 −1
  47. drivers/block/rbd.c +0 −19
  48. drivers/block/rsxx/dev.c +1 −1
  49. drivers/block/sx8.c +2 −2
  50. drivers/block/umem.c +1 −1
  51. drivers/block/virtio_blk.c +1 −1
  52. drivers/block/zram/zram_drv.c +1 −1
  53. drivers/cdrom/cdrom.c +1 −1
  54. drivers/ide/ide-atapi.c +1 −1
  55. drivers/ide/ide-cd.c +1 −1
  56. drivers/ide/ide-cd_ioctl.c +1 −1
  57. drivers/ide/ide-devsets.c +1 −1
  58. drivers/ide/ide-disk.c +1 −1
  59. drivers/ide/ide-ioctls.c +2 −2
  60. drivers/ide/ide-park.c +1 −1
  61. drivers/ide/ide-pm.c +2 −2
  62. drivers/ide/ide-tape.c +1 −1
  63. drivers/ide/ide-taskfile.c +1 −1
  64. drivers/lightnvm/pblk-init.c +1 −1
  65. drivers/md/bcache/debug.c +1 −1
  66. drivers/md/bcache/request.c +24 −15
  67. drivers/md/bcache/super.c +1 −1
  68. drivers/md/dm-bio-record.h +3 −6
  69. drivers/md/dm-cache-metadata.c +1 −1
  70. drivers/md/dm-clone-target.c +1 −13
  71. drivers/md/dm-raid1.c +5 −5
  72. drivers/md/dm-thin-metadata.c +1 −1
  73. drivers/md/dm-zoned-metadata.c +3 −3
  74. drivers/md/dm.c +7 −7
  75. drivers/md/md-linear.c +1 −1
  76. drivers/md/md.c +29 −44
  77. drivers/md/md.h +3 −5
  78. drivers/md/raid1.c +4 −4
  79. drivers/md/raid10.c +7 −11
  80. drivers/md/raid5-ppl.c +1 −1
  81. drivers/md/raid5.c +46 −64
  82. drivers/mmc/core/block.c +5 −5
  83. drivers/nvdimm/blk.c +2 −2
  84. drivers/nvdimm/btt.c +2 −2
  85. drivers/nvdimm/pmem.c +2 −2
  86. drivers/nvme/host/core.c +16 −15
  87. drivers/nvme/host/lightnvm.c +3 −4
  88. drivers/nvme/host/multipath.c +3 −3
  89. drivers/nvme/host/pci.c +2 −2
  90. drivers/nvme/host/rdma.c +1 −1
  91. drivers/nvme/host/zns.c +3 −8
  92. drivers/nvme/target/io-cmd-bdev.c +1 −1
  93. drivers/nvme/target/passthru.c +1 −1
  94. drivers/s390/block/dasd.c +5 −21
  95. drivers/s390/block/dcssblk.c +1 −5
  96. drivers/s390/block/xpram.c +1 −1
  97. drivers/scsi/scsi_error.c +1 −1
  98. drivers/scsi/scsi_lib.c +1 −1
  99. drivers/scsi/sd_zbc.c +39 −4
  100. drivers/scsi/sg.c +1 −2
  101. drivers/scsi/st.c +1 −1
  102. drivers/target/target_core_file.c +6 −14
  103. drivers/target/target_core_pscsi.c +1 −2
  104. fs/block_dev.c +9 −11
  105. fs/btrfs/check-integrity.c +5 −5
  106. fs/btrfs/raid56.c +2 −5
  107. fs/btrfs/scrub.c +1 −1
  108. fs/btrfs/volumes.c +1 −1
  109. fs/btrfs/zoned.c +2 −2
  110. fs/direct-io.c +3 −1
  111. fs/exfat/file.c +1 −1
  112. fs/ext4/fast_commit.c +2 −2
  113. fs/ext4/fsync.c +1 −1
  114. fs/ext4/ialloc.c +1 −1
  115. fs/ext4/super.c +1 −1
  116. fs/f2fs/data.c +5 −35
  117. fs/f2fs/f2fs.h +0 −2
  118. fs/f2fs/segment.c +1 −11
  119. fs/f2fs/super.c +0 −1
  120. fs/fat/file.c +1 −1
  121. fs/hfsplus/inode.c +1 −1
  122. fs/hfsplus/super.c +1 −1
  123. fs/iomap/direct-io.c +4 −5
  124. fs/jbd2/checkpoint.c +1 −1
  125. fs/jbd2/commit.c +2 −2
  126. fs/jbd2/recovery.c +1 −1
  127. fs/libfs.c +1 −1
  128. fs/nfs/blocklayout/blocklayout.c +0 −5
  129. fs/nfsd/blocklayout.c +1 −1
  130. fs/nilfs2/segbuf.c +0 −4
  131. fs/nilfs2/the_nilfs.h +1 −1
  132. fs/ocfs2/file.c +1 −1
  133. fs/reiserfs/file.c +1 −1
  134. fs/splice.c +6 −3
  135. fs/super.c +2 −1
  136. fs/xfs/xfs_super.c +1 −1
  137. fs/zonefs/super.c +6 −7
  138. include/linux/bio.h +28 −27
  139. include/linux/blk-mq.h +2 −6
  140. include/linux/blk_types.h +3 −30
  141. include/linux/blkdev.h +25 −28
  142. include/linux/elevator.h +2 −0
  143. include/linux/genhd.h +9 −18
  144. include/linux/swap.h +0 −1
  145. kernel/trace/blktrace.c +9 −7
  146. lib/iov_iter.c +19 −2
  147. mm/page_io.c +14 −33
  148. mm/swapfile.c +3 −33
Documentation/block/biovecs.rst+2−0
@@ -40,6 +40,8 @@ normal code doesn't have to deal with bi_bvec_done. · 39 unmodified lines
4040 There is a lower level advance function - bvec_iter_advance() - which takes
4141 a pointer to a biovec, not a bio; this is used by the bio integrity code.
4242
43+As of 5.12 bvec segments with zero bv_len are not supported.
44+
4345 What's all this get us?
4446 =======================
4547
Documentation/block/queue-sysfs.rst+13−0
@@ -261,6 +261,12 @@ For block drivers that support REQ_OP_WRITE_ZEROES, the maximum number of · 260 unmodified lines
261261 bytes that can be zeroed at once. The value 0 means that REQ_OP_WRITE_ZEROES
262262 is not supported.
263263
264+zone_append_max_bytes (RO)
265+--------------------------
266+This is the maximum number of bytes that can be written to a sequential
267+zone of a zoned block device using a zone append write operation
268+(REQ_OP_ZONE_APPEND). This value is always 0 for regular block devices.
269+
264270 zoned (RO)
265271 ----------
266272 This indicates if the device is a zoned block device and the zone model of the
@@ -273,4 +279,11 @@ devices are described in the ZBC (Zoned Block Commands) and ZAC · 6 unmodified lines
273279 do not support zone commands, they will be treated as regular block devices
274280 and zoned will report "none".
275281
282+zone_write_granularity (RO)
283+---------------------------
284+This indicates the alignment constraint, in bytes, for write operations in
285+sequential zones of zoned block devices (devices with a zoned attributed
286+that reports "host-managed" or "host-aware"). This value is always 0 for
287+regular block devices.
288+
276289 Jens Axboe <jens.axboe@oracle.com>, February 2009
Documentation/filesystems/f2fs.rst+0−1
@@ -179,7 +179,6 @@ fault_type=%d Support configuring fault injection type, should be · 178 unmodified lines
179179 FAULT_KVMALLOC 0x000000002
180180 FAULT_PAGE_ALLOC 0x000000004
181181 FAULT_PAGE_GET 0x000000008
182− FAULT_ALLOC_BIO 0x000000010
183182 FAULT_ALLOC_NID 0x000000020
184183 FAULT_ORPHAN 0x000000040
185184 FAULT_BLOCK 0x000000080
Documentation/filesystems/porting.rst+16−0
@@ -865,3 +865,19 @@ no matter what. Everything is handled by the caller. · 864 unmodified lines
865865
866866 clone_private_mount() returns a longterm mount now, so the proper destructor of
867867 its result is kern_unmount() or kern_unmount_array().
868+
869+---
870+
871+**mandatory**
872+
873+zero-length bvec segments are disallowed, they must be filtered out before
874+passed on to an iterator.
875+
876+---
877+
878+**mandatory**
879+
880+For bvec based itererators bio_iov_iter_get_pages() now doesn't copy bvecs but
881+uses the one provided. Anyone issuing kiocb-I/O should ensure that the bvec and
882+page references stay until I/O has completed, i.e. until ->ki_complete() has
883+been called or returned with non -EIOCBQUEUED code.
arch/m68k/emu/nfblock.c+1−1
@@ -61,7 +61,7 @@ struct nfhd_device { · 60 unmodified lines
6161
6262 static blk_qc_t nfhd_submit_bio(struct bio *bio)
6363 {
64− struct nfhd_device *dev = bio->bi_disk->private_data;
64+ struct nfhd_device *dev = bio->bi_bdev->bd_disk->private_data;
6565 struct bio_vec bvec;
6666 struct bvec_iter iter;
6767 int dir, len, shift;
arch/xtensa/platforms/iss/simdisk.c+1−1
@@ -103,7 +103,7 @@ static void simdisk_transfer(struct simdisk *dev, unsigned long sector, · 102 unmodified lines
103103
104104 static blk_qc_t simdisk_submit_bio(struct bio *bio)
105105 {
106− struct simdisk *dev = bio->bi_disk->private_data;
106+ struct simdisk *dev = bio->bi_bdev->bd_disk->private_data;
107107 struct bio_vec bvec;
108108 struct bvec_iter iter;
109109 sector_t sector = bio->bi_iter.bi_sector;
block/bfq-iosched.c+276−169
@@ -158,7 +158,6 @@ BFQ_BFQQ_FNS(in_large_burst); · 157 unmodified lines
158158 BFQ_BFQQ_FNS(coop);
159159 BFQ_BFQQ_FNS(split_coop);
160160 BFQ_BFQQ_FNS(softrt_update);
161−BFQ_BFQQ_FNS(has_waker);
162161 #undef BFQ_BFQQ_FNS \
163162
164163 /* Expiration time of sync (0) and async (1) requests, in ns. */
@@ -1024,9 +1023,16 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, · 859 unmodified lines
10241023 else
10251024 bfq_clear_bfqq_IO_bound(bfqq);
10261025
1026+ bfqq->last_serv_time_ns = bic->saved_last_serv_time_ns;
1027+ bfqq->inject_limit = bic->saved_inject_limit;
1028+ bfqq->decrease_time_jif = bic->saved_decrease_time_jif;
1029+
10271030 bfqq->entity.new_weight = bic->saved_weight;
10281031 bfqq->ttime = bic->saved_ttime;
1032+ bfqq->io_start_time = bic->saved_io_start_time;
1033+ bfqq->tot_idle_time = bic->saved_tot_idle_time;
10291034 bfqq->wr_coeff = bic->saved_wr_coeff;
1035+ bfqq->service_from_wr = bic->saved_service_from_wr;
10301036 bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt;
10311037 bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish;
10321038 bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time;
@@ -1647,6 +1653,8 @@ static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq, · 614 unmodified lines
16471653 return bfqq_weight > in_serv_weight;
16481654 }
16491655
1656+static bool bfq_better_to_idle(struct bfq_queue *bfqq);
1657+
16501658 static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd,
16511659 struct bfq_queue *bfqq,
16521660 int old_wr_coeff,
@@ -1671,15 +1679,19 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, · 18 unmodified lines
16711679 * - it is sync,
16721680 * - it does not belong to a large burst,
16731681 * - it has been idle for enough time or is soft real-time,
1674− * - is linked to a bfq_io_cq (it is not shared in any sense).
1682+ * - is linked to a bfq_io_cq (it is not shared in any sense),
1683+ * - has a default weight (otherwise we assume the user wanted
1684+ * to control its weight explicitly)
16751685 */
16761686 in_burst = bfq_bfqq_in_large_burst(bfqq);
16771687 soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 &&
16781688 !BFQQ_TOTALLY_SEEKY(bfqq) &&
16791689 !in_burst &&
16801690 time_is_before_jiffies(bfqq->soft_rt_next_start) &&
1681− bfqq->dispatched == 0;
1682− *interactive = !in_burst && idle_for_long_time;
1691+ bfqq->dispatched == 0 &&
1692+ bfqq->entity.new_weight == 40;
1693+ *interactive = !in_burst && idle_for_long_time &&
1694+ bfqq->entity.new_weight == 40;
16831695 wr_or_deserves_wr = bfqd->low_latency &&
16841696 (bfqq->wr_coeff > 1 ||
16851697 (bfq_bfqq_sync(bfqq) &&
@@ -1717,17 +1729,6 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, · 31 unmodified lines
17171729
17181730 bfq_clear_bfqq_just_created(bfqq);
17191731
1720−
1721− if (!bfq_bfqq_IO_bound(bfqq)) {
1722− if (arrived_in_time) {
1723− bfqq->requests_within_timer++;
1724− if (bfqq->requests_within_timer >=
1725− bfqd->bfq_requests_within_timer)
1726− bfq_mark_bfqq_IO_bound(bfqq);
1727− } else
1728− bfqq->requests_within_timer = 0;
1729− }
1730−
17311732 if (bfqd->low_latency) {
17321733 if (unlikely(time_is_after_jiffies(bfqq->split_time)))
17331734 /* wraparound */
@@ -1755,10 +1756,10 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, · 21 unmodified lines
17551756 bfq_add_bfqq_busy(bfqd, bfqq);
17561757
17571758 /*
1758− * Expire in-service queue only if preemption may be needed
1759− * for guarantees. In particular, we care only about two
1760− * cases. The first is that bfqq has to recover a service
1761− * hole, as explained in the comments on
1759+ * Expire in-service queue if preemption may be needed for
1760+ * guarantees or throughput. As for guarantees, we care
1761+ * explicitly about two cases. The first is that bfqq has to
1762+ * recover a service hole, as explained in the comments on
17621763 * bfq_bfqq_update_budg_for_activation(), i.e., that
17631764 * bfqq_wants_to_preempt is true. However, if bfqq does not
17641765 * carry time-critical I/O, then bfqq's bandwidth is less
@@ -1785,11 +1786,23 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, · 20 unmodified lines
17851786 * timestamps of the in-service queue would need to be
17861787 * updated, and this operation is quite costly (see the
17871788 * comments on bfq_bfqq_update_budg_for_activation()).
1789+ *
1790+ * As for throughput, we ask bfq_better_to_idle() whether we
1791+ * still need to plug I/O dispatching. If bfq_better_to_idle()
1792+ * says no, then plugging is not needed any longer, either to
1793+ * boost throughput or to perserve service guarantees. Then
1794+ * the best option is to stop plugging I/O, as not doing so
1795+ * would certainly lower throughput. We may end up in this
1796+ * case if: (1) upon a dispatch attempt, we detected that it
1797+ * was better to plug I/O dispatch, and to wait for a new
1798+ * request to arrive for the currently in-service queue, but
1799+ * (2) this switch of bfqq to busy changes the scenario.
17881800 */
17891801 if (bfqd->in_service_queue &&
17901802 ((bfqq_wants_to_preempt &&
17911803 bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) ||
1792− bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue)) &&
1804+ bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue) ||
1805+ !bfq_better_to_idle(bfqd->in_service_queue)) &&
17931806 next_queue_may_preempt(bfqd))
17941807 bfq_bfqq_expire(bfqd, bfqd->in_service_queue,
17951808 false, BFQQE_PREEMPTED);
@@ -1861,6 +1874,138 @@ static void bfq_reset_inject_limit(struct bfq_data *bfqd, · 65 unmodified lines
18611874 bfqq->decrease_time_jif = jiffies;
18621875 }
18631876
1877+static void bfq_update_io_intensity(struct bfq_queue *bfqq, u64 now_ns)
1878+{
1879+ u64 tot_io_time = now_ns - bfqq->io_start_time;
1880+
1881+ if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq->dispatched == 0)
1882+ bfqq->tot_idle_time +=
1883+ now_ns - bfqq->ttime.last_end_request;
1884+
1885+ if (unlikely(bfq_bfqq_just_created(bfqq)))
1886+ return;
1887+
1888+ /*
1889+ * Must be busy for at least about 80% of the time to be
1890+ * considered I/O bound.
1891+ */
1892+ if (bfqq->tot_idle_time * 5 > tot_io_time)
1893+ bfq_clear_bfqq_IO_bound(bfqq);
1894+ else
1895+ bfq_mark_bfqq_IO_bound(bfqq);
1896+
1897+ /*
1898+ * Keep an observation window of at most 200 ms in the past
1899+ * from now.
1900+ */
1901+ if (tot_io_time > 200 * NSEC_PER_MSEC) {
1902+ bfqq->io_start_time = now_ns - (tot_io_time>>1);
1903+ bfqq->tot_idle_time >>= 1;
1904+ }
1905+}
1906+
1907+/*
1908+ * Detect whether bfqq's I/O seems synchronized with that of some
1909+ * other queue, i.e., whether bfqq, after remaining empty, happens to
1910+ * receive new I/O only right after some I/O request of the other
1911+ * queue has been completed. We call waker queue the other queue, and
1912+ * we assume, for simplicity, that bfqq may have at most one waker
1913+ * queue.
1914+ *
1915+ * A remarkable throughput boost can be reached by unconditionally
1916+ * injecting the I/O of the waker queue, every time a new
1917+ * bfq_dispatch_request happens to be invoked while I/O is being
1918+ * plugged for bfqq. In addition to boosting throughput, this
1919+ * unblocks bfqq's I/O, thereby improving bandwidth and latency for
1920+ * bfqq. Note that these same results may be achieved with the general
1921+ * injection mechanism, but less effectively. For details on this
1922+ * aspect, see the comments on the choice of the queue for injection
1923+ * in bfq_select_queue().
1924+ *
1925+ * Turning back to the detection of a waker queue, a queue Q is deemed
1926+ * as a waker queue for bfqq if, for three consecutive times, bfqq
1927+ * happens to become non empty right after a request of Q has been
1928+ * completed. In particular, on the first time, Q is tentatively set
1929+ * as a candidate waker queue, while on the third consecutive time
1930+ * that Q is detected, the field waker_bfqq is set to Q, to confirm
1931+ * that Q is a waker queue for bfqq. These detection steps are
1932+ * performed only if bfqq has a long think time, so as to make it more
1933+ * likely that bfqq's I/O is actually being blocked by a
1934+ * synchronization. This last filter, plus the above three-times
1935+ * requirement, make false positives less likely.
1936+ *
1937+ * NOTE
1938+ *
1939+ * The sooner a waker queue is detected, the sooner throughput can be
1940+ * boosted by injecting I/O from the waker queue. Fortunately,
1941+ * detection is likely to be actually fast, for the following
1942+ * reasons. While blocked by synchronization, bfqq has a long think
1943+ * time. This implies that bfqq's inject limit is at least equal to 1
1944+ * (see the comments in bfq_update_inject_limit()). So, thanks to
1945+ * injection, the waker queue is likely to be served during the very
1946+ * first I/O-plugging time interval for bfqq. This triggers the first
1947+ * step of the detection mechanism. Thanks again to injection, the
1948+ * candidate waker queue is then likely to be confirmed no later than
1949+ * during the next I/O-plugging interval for bfqq.
1950+ *
1951+ * ISSUE
1952+ *
1953+ * On queue merging all waker information is lost.
1954+ */
1955+static void bfq_check_waker(struct bfq_data *bfqd, struct bfq_queue *bfqq,
1956+ u64 now_ns)
1957+{
1958+ if (!bfqd->last_completed_rq_bfqq ||
1959+ bfqd->last_completed_rq_bfqq == bfqq ||
1960+ bfq_bfqq_has_short_ttime(bfqq) ||
1961+ now_ns - bfqd->last_completion >= 4 * NSEC_PER_MSEC ||
1962+ bfqd->last_completed_rq_bfqq == bfqq->waker_bfqq)
1963+ return;
1964+
1965+ if (bfqd->last_completed_rq_bfqq !=
1966+ bfqq->tentative_waker_bfqq) {
1967+ /*
1968+ * First synchronization detected with a
1969+ * candidate waker queue, or with a different
1970+ * candidate waker queue from the current one.
1971+ */
1972+ bfqq->tentative_waker_bfqq =
1973+ bfqd->last_completed_rq_bfqq;
1974+ bfqq->num_waker_detections = 1;
1975+ } else /* Same tentative waker queue detected again */
1976+ bfqq->num_waker_detections++;
1977+
1978+ if (bfqq->num_waker_detections == 3) {
1979+ bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq;
1980+ bfqq->tentative_waker_bfqq = NULL;
1981+
1982+ /*
1983+ * If the waker queue disappears, then
1984+ * bfqq->waker_bfqq must be reset. To
1985+ * this goal, we maintain in each
1986+ * waker queue a list, woken_list, of
1987+ * all the queues that reference the
1988+ * waker queue through their
1989+ * waker_bfqq pointer. When the waker
1990+ * queue exits, the waker_bfqq pointer
1991+ * of all the queues in the woken_list
1992+ * is reset.
1993+ *
1994+ * In addition, if bfqq is already in
1995+ * the woken_list of a waker queue,
1996+ * then, before being inserted into
1997+ * the woken_list of a new waker
1998+ * queue, bfqq must be removed from
1999+ * the woken_list of the old waker
2000+ * queue.
2001+ */
2002+ if (!hlist_unhashed(&bfqq->woken_list_node))
2003+ hlist_del_init(&bfqq->woken_list_node);
2004+ hlist_add_head(&bfqq->woken_list_node,
2005+ &bfqd->last_completed_rq_bfqq->woken_list);
2006+ }
2007+}
2008+
18642009 static void bfq_add_request(struct request *rq)
18652010 {
18662011 struct bfq_queue *bfqq = RQ_BFQQ(rq);
@@ -1868,117 +2013,14 @@ static void bfq_add_request(struct request *rq) · 1 unmodified line
18682013 struct request *next_rq, *prev;
18692014 unsigned int old_wr_coeff = bfqq->wr_coeff;
18702015 bool interactive = false;
2016+ u64 now_ns = ktime_get_ns();
18712017
18722018 bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq));
18732019 bfqq->queued[rq_is_sync(rq)]++;
18742020 bfqd->queued++;
18752021
18762022 if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_sync(bfqq)) {
1877− /*
1878− * Detect whether bfqq's I/O seems synchronized with
1879− * that of some other queue, i.e., whether bfqq, after
1880− * remaining empty, happens to receive new I/O only
1881− * right after some I/O request of the other queue has
1882− * been completed. We call waker queue the other
1883− * queue, and we assume, for simplicity, that bfqq may
1884− * have at most one waker queue.
1885− *
1886− * A remarkable throughput boost can be reached by
1887− * unconditionally injecting the I/O of the waker
1888− * queue, every time a new bfq_dispatch_request
1889− * happens to be invoked while I/O is being plugged
1890− * for bfqq. In addition to boosting throughput, this
1891− * unblocks bfqq's I/O, thereby improving bandwidth
1892− * and latency for bfqq. Note that these same results
1893− * may be achieved with the general injection
1894− * mechanism, but less effectively. For details on
1895− * this aspect, see the comments on the choice of the
1896− * queue for injection in bfq_select_queue().
1897− *
1898− * Turning back to the detection of a waker queue, a
1899− * queue Q is deemed as a waker queue for bfqq if, for
1900− * two consecutive times, bfqq happens to become non
1901− * empty right after a request of Q has been
1902− * completed. In particular, on the first time, Q is
1903− * tentatively set as a candidate waker queue, while
1904− * on the second time, the flag
1905− * bfq_bfqq_has_waker(bfqq) is set to confirm that Q
1906− * is a waker queue for bfqq. These detection steps
1907− * are performed only if bfqq has a long think time,
1908− * so as to make it more likely that bfqq's I/O is
1909− * actually being blocked by a synchronization. This
1910− * last filter, plus the above two-times requirement,
1911− * make false positives less likely.
1912− *
1913− * NOTE
1914− *
1915− * The sooner a waker queue is detected, the sooner
1916− * throughput can be boosted by injecting I/O from the
1917− * waker queue. Fortunately, detection is likely to be
1918− * actually fast, for the following reasons. While
1919− * blocked by synchronization, bfqq has a long think
1920− * time. This implies that bfqq's inject limit is at
1921− * least equal to 1 (see the comments in
1922− * bfq_update_inject_limit()). So, thanks to
1923− * injection, the waker queue is likely to be served
1924− * during the very first I/O-plugging time interval
1925− * for bfqq. This triggers the first step of the
1926− * detection mechanism. Thanks again to injection, the
1927− * candidate waker queue is then likely to be
1928− * confirmed no later than during the next
1929− * I/O-plugging interval for bfqq.
1930− */
1931− if (bfqd->last_completed_rq_bfqq &&
1932− !bfq_bfqq_has_short_ttime(bfqq) &&
1933− ktime_get_ns() - bfqd->last_completion <
1934− 200 * NSEC_PER_USEC) {
1935− if (bfqd->last_completed_rq_bfqq != bfqq &&
1936− bfqd->last_completed_rq_bfqq !=
1937− bfqq->waker_bfqq) {
1938− /*
1939− * First synchronization detected with
1940− * a candidate waker queue, or with a
1941− * different candidate waker queue
1942− * from the current one.
1943− */
1944− bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq;
1945−
1946− /*
1947− * If the waker queue disappears, then
1948− * bfqq->waker_bfqq must be reset. To
1949− * this goal, we maintain in each
1950− * waker queue a list, woken_list, of
1951− * all the queues that reference the
1952− * waker queue through their
1953− * waker_bfqq pointer. When the waker
1954− * queue exits, the waker_bfqq pointer
1955− * of all the queues in the woken_list
1956− * is reset.
1957− *
1958− * In addition, if bfqq is already in
1959− * the woken_list of a waker queue,
1960− * then, before being inserted into
1961− * the woken_list of a new waker
1962− * queue, bfqq must be removed from
1963− * the woken_list of the old waker
1964− * queue.
1965− */
1966− if (!hlist_unhashed(&bfqq->woken_list_node))
1967− hlist_del_init(&bfqq->woken_list_node);
1968− hlist_add_head(&bfqq->woken_list_node,
1969− &bfqd->last_completed_rq_bfqq->woken_list);
1970−
1971− bfq_clear_bfqq_has_waker(bfqq);
1972− } else if (bfqd->last_completed_rq_bfqq ==
1973− bfqq->waker_bfqq &&
1974− !bfq_bfqq_has_waker(bfqq)) {
1975− /*
1976− * synchronization with waker_bfqq
1977− * seen for the second time
1978− */
1979− bfq_mark_bfqq_has_waker(bfqq);
1980− }
1981− }
2023+ bfq_check_waker(bfqd, bfqq, now_ns);
19822024
19832025 /*
19842026 * Periodically reset inject limit, to make sure that
@@ -2047,6 +2089,9 @@ static void bfq_add_request(struct request *rq) · 62 unmodified lines
20472089 }
20482090 }
20492091
2092+ if (bfq_bfqq_sync(bfqq))
2093+ bfq_update_io_intensity(bfqq, now_ns);
2094+
20502095 elv_rb_add(&bfqq->sort_list, rq);
20512096
20522097 /*
@@ -2352,6 +2397,24 @@ static void bfq_requests_merged(struct request_queue *q, struct request *rq, · 299 unmodified lines
23522397 /* Must be called with bfqq != NULL */
23532398 static void bfq_bfqq_end_wr(struct bfq_queue *bfqq)
23542399 {
2400+ /*
2401+ * If bfqq has been enjoying interactive weight-raising, then
2402+ * reset soft_rt_next_start. We do it for the following
2403+ * reason. bfqq may have been conveying the I/O needed to load
2404+ * a soft real-time application. Such an application actually
2405+ * exhibits a soft real-time I/O pattern after it finishes
2406+ * loading, and finally starts doing its job. But, if bfqq has
2407+ * been receiving a lot of bandwidth so far (likely to happen
2408+ * on a fast device), then soft_rt_next_start now contains a
2409+ * high value that. So, without this reset, bfqq would be
2410+ * prevented from being possibly considered as soft_rt for a
2411+ * very long time.
2412+ */
2413+
2414+ if (bfqq->wr_cur_max_time !=
2415+ bfqq->bfqd->bfq_wr_rt_max_time)
2416+ bfqq->soft_rt_next_start = jiffies;
2417+
23552418 if (bfq_bfqq_busy(bfqq))
23562419 bfqq->bfqd->wr_busy_queues--;
23572420 bfqq->wr_coeff = 1;
@@ -2686,10 +2749,16 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq) · 328 unmodified lines
26862749 if (!bic)
26872750 return;
26882751
2752+ bic->saved_last_serv_time_ns = bfqq->last_serv_time_ns;
2753+ bic->saved_inject_limit = bfqq->inject_limit;
2754+ bic->saved_decrease_time_jif = bfqq->decrease_time_jif;
2755+
26892756 bic->saved_weight = bfqq->entity.orig_weight;
26902757 bic->saved_ttime = bfqq->ttime;
26912758 bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq);
26922759 bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);
2760+ bic->saved_io_start_time = bfqq->io_start_time;
2761+ bic->saved_tot_idle_time = bfqq->tot_idle_time;
26932762 bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
26942763 bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
26952764 if (unlikely(bfq_bfqq_just_created(bfqq) &&
@@ -2712,6 +2781,7 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq) · 16 unmodified lines
27122781 bic->saved_wr_coeff = bfqq->wr_coeff;
27132782 bic->saved_wr_start_at_switch_to_srt =
27142783 bfqq->wr_start_at_switch_to_srt;
2784+ bic->saved_service_from_wr = bfqq->service_from_wr;
27152785 bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish;
27162786 bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time;
27172787 }
@@ -2937,6 +3007,7 @@ static void __bfq_set_in_service_queue(struct bfq_data *bfqd, · 219 unmodified lines
29373007 }
29383008
29393009 bfqd->in_service_queue = bfqq;
3010+ bfqd->in_serv_last_pos = 0;
29403011 }
29413012
29423013 /*
@@ -3442,20 +3513,38 @@ static void bfq_dispatch_remove(struct request_queue *q, struct request *rq) · 499 unmodified lines
34423513 * order until all the requests already queued in the device have been
34433514 * served. The last sub-condition commented above somewhat mitigates
34443515 * this problem for weight-raised queues.
3516+ *
3517+ * However, as an additional mitigation for this problem, we preserve
3518+ * plugging for a special symmetric case that may suddenly turn into
3519+ * asymmetric: the case where only bfqq is busy. In this case, not
3520+ * expiring bfqq does not cause any harm to any other queues in terms
3521+ * of service guarantees. In contrast, it avoids the following unlucky
3522+ * sequence of events: (1) bfqq is expired, (2) a new queue with a
3523+ * lower weight than bfqq becomes busy (or more queues), (3) the new
3524+ * queue is served until a new request arrives for bfqq, (4) when bfqq
3525+ * is finally served, there are so many requests of the new queue in
3526+ * the drive that the pending requests for bfqq take a lot of time to
3527+ * be served. In particular, event (2) may case even already
3528+ * dispatched requests of bfqq to be delayed, inside the drive. So, to
3529+ * avoid this series of events, the scenario is preventively declared
3530+ * as asymmetric also if bfqq is the only busy queues
34453531 */
34463532 static bool idling_needed_for_service_guarantees(struct bfq_data *bfqd,
34473533 struct bfq_queue *bfqq)
34483534 {
3535+ int tot_busy_queues = bfq_tot_busy_queues(bfqd);
3536+
34493537 /* No point in idling for bfqq if it won't get requests any longer */
34503538 if (unlikely(!bfqq_process_refs(bfqq)))
34513539 return false;
34523540
34533541 return (bfqq->wr_coeff > 1 &&
34543542 (bfqd->wr_busy_queues <
3455− bfq_tot_busy_queues(bfqd) ||
3543+ tot_busy_queues ||
34563544 bfqd->rq_in_driver >=
34573545 bfqq->dispatched + 4)) ||
3458− bfq_asymmetric_scenario(bfqd, bfqq);
3546+ bfq_asymmetric_scenario(bfqd, bfqq) ||
3547+ tot_busy_queues == 1;
34593548 }
34603549
34613550 static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
@@ -3939,10 +4028,6 @@ void bfq_bfqq_expire(struct bfq_data *bfqd, · 477 unmodified lines
39394028 bfq_bfqq_budget_left(bfqq) >= entity->budget / 3)))
39404029 bfq_bfqq_charge_time(bfqd, bfqq, delta);
39414030
3942− if (reason == BFQQE_TOO_IDLE &&
3943− entity->service <= 2 * entity->budget / 10)
3944− bfq_clear_bfqq_IO_bound(bfqq);
3945−
39464031 if (bfqd->low_latency && bfqq->wr_coeff == 1)
39474032 bfqq->last_wr_start_finish = jiffies;
39484033
@@ -3952,30 +4037,15 @@ void bfq_bfqq_expire(struct bfq_data *bfqd, · 3 unmodified lines
39524037 * If we get here, and there are no outstanding
39534038 * requests, then the request pattern is isochronous
39544039 * (see the comments on the function
3955− * bfq_bfqq_softrt_next_start()). Thus we can compute
3956− * soft_rt_next_start. And we do it, unless bfqq is in
3957− * interactive weight raising. We do not do it in the
3958− * latter subcase, for the following reason. bfqq may
3959− * be conveying the I/O needed to load a soft
3960− * real-time application. Such an application will
3961− * actually exhibit a soft real-time I/O pattern after
3962− * it finally starts doing its job. But, if
3963− * soft_rt_next_start is computed here for an
3964− * interactive bfqq, and bfqq had received a lot of
3965− * service before remaining with no outstanding
3966− * request (likely to happen on a fast device), then
3967− * soft_rt_next_start would be assigned such a high
3968− * value that, for a very long time, bfqq would be
3969− * prevented from being possibly considered as soft
3970− * real time.
4040+ * bfq_bfqq_softrt_next_start()). Therefore we can
4041+ * compute soft_rt_next_start.
39714042 *
39724043 * If, instead, the queue still has outstanding
39734044 * requests, then we have to wait for the completion
39744045 * of all the outstanding requests to discover whether
39754046 * the request pattern is actually isochronous.
39764047 */
3977− if (bfqq->dispatched == 0 &&
3978− bfqq->wr_coeff != bfqd->bfq_wr_coeff)
4048+ if (bfqq->dispatched == 0)
39794049 bfqq->soft_rt_next_start =
39804050 bfq_bfqq_softrt_next_start(bfqd, bfqq);
39814051 else if (bfqq->dispatched > 0) {
@@ -4497,9 +4567,9 @@ check_queue: · 515 unmodified lines
44974567 bfq_serv_to_charge(async_bfqq->next_rq, async_bfqq) <=
44984568 bfq_bfqq_budget_left(async_bfqq))
44994569 bfqq = bfqq->bic->bfqq[0];
4500− else if (bfq_bfqq_has_waker(bfqq) &&
4570+ else if (bfqq->waker_bfqq &&
45014571 bfq_bfqq_busy(bfqq->waker_bfqq) &&
4502− bfqq->next_rq &&
4572+ bfqq->waker_bfqq->next_rq &&
45034573 bfq_serv_to_charge(bfqq->waker_bfqq->next_rq,
45044574 bfqq->waker_bfqq) <=
45054575 bfq_bfqq_budget_left(bfqq->waker_bfqq)
@@ -4559,9 +4629,21 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) · 53 unmodified lines
45594629 bfqq->wr_cur_max_time)) {
45604630 if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time ||
45614631 time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt +
4562− bfq_wr_duration(bfqd)))
4632+ bfq_wr_duration(bfqd))) {
4633+ /*
4634+ * Either in interactive weight
4635+ * raising, or in soft_rt weight
4636+ * raising with the
4637+ * interactive-weight-raising period
4638+ * elapsed (so no switch back to
4639+ * interactive weight raising).
4640+ */
45634641 bfq_bfqq_end_wr(bfqq);
4564− else {
4642+ } else { /*
4643+ * soft_rt finishing while still in
4644+ * interactive period, switch back to
4645+ * interactive weight raising
4646+ */
45654647 switch_back_to_interactive_wr(bfqq, bfqd);
45664648 bfqq->entity.prio_changed = 1;
45674649 }
@@ -4640,9 +4722,6 @@ static bool bfq_has_work(struct blk_mq_hw_ctx *hctx) · 72 unmodified lines
46404722 {
46414723 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
46424724
4643− if (!atomic_read(&hctx->elevator_queued))
4644− return false;
4645−
46464725 /*
46474726 * Avoiding lock: a race on bfqd->busy_queues should cause at
46484727 * most a call to dispatch for nothing
@@ -4892,7 +4971,6 @@ void bfq_put_queue(struct bfq_queue *bfqq) · 243 unmodified lines
48924971 hlist_for_each_entry_safe(item, n, &bfqq->woken_list,
48934972 woken_list_node) {
48944973 item->waker_bfqq = NULL;
4895− bfq_clear_bfqq_has_waker(item);
48964974 hlist_del_init(&item->woken_list_node);
48974975 }
48984976
@@ -5012,6 +5090,8 @@ bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic) · 113 unmodified lines
50125090 }
50135091
50145092 bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio);
5093+ bfq_log_bfqq(bfqd, bfqq, "new_ioprio %d new_weight %d",
5094+ bfqq->new_ioprio, bfqq->entity.new_weight);
50155095 bfqq->entity.prio_changed = 1;
50165096 }
50175097
@@ -5049,6 +5129,8 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio) · 31 unmodified lines
50495129 static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
50505130 struct bfq_io_cq *bic, pid_t pid, int is_sync)
50515131 {
5132+ u64 now_ns = ktime_get_ns();
5133+
50525134 RB_CLEAR_NODE(&bfqq->entity.rb_node);
50535135 INIT_LIST_HEAD(&bfqq->fifo);
50545136 INIT_HLIST_NODE(&bfqq->burst_list_node);
@@ -5076,7 +5158,9 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, · 21 unmodified lines
50765158 bfq_clear_bfqq_sync(bfqq);
50775159
50785160 /* set end request to minus infinity from now */
5079− bfqq->ttime.last_end_request = ktime_get_ns() + 1;
5161+ bfqq->ttime.last_end_request = now_ns + 1;
5162+
5163+ bfqq->io_start_time = now_ns;
50805164
50815165 bfq_mark_bfqq_IO_bound(bfqq);
50825166
@@ -5194,11 +5278,19 @@ static void bfq_update_io_thinktime(struct bfq_data *bfqd, · 111 unmodified lines
51945278 struct bfq_queue *bfqq)
51955279 {
51965280 struct bfq_ttime *ttime = &bfqq->ttime;
5197− u64 elapsed = ktime_get_ns() - bfqq->ttime.last_end_request;
5281+ u64 elapsed;
51985282
5283+ /*
5284+ * We are really interested in how long it takes for the queue to
5285+ * become busy when there is no outstanding IO for this queue. So
5286+ * ignore cases when the bfq queue has already IO queued.
5287+ */
5288+ if (bfqq->dispatched || bfq_bfqq_busy(bfqq))
5289+ return;
5290+ elapsed = ktime_get_ns() - bfqq->ttime.last_end_request;
51995291 elapsed = min_t(u64, elapsed, 2ULL * bfqd->bfq_slice_idle);
52005292
5201− ttime->ttime_samples = (7*bfqq->ttime.ttime_samples + 256) / 8;
5293+ ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8;
52025294 ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed, 8);
52035295 ttime->ttime_mean = div64_ul(ttime->ttime_total + 128,
52045296 ttime->ttime_samples);
@@ -5213,8 +5305,26 @@ bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, · 8 unmodified lines
52135305
52145306 if (bfqq->wr_coeff > 1 &&
52155307 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&
5216− BFQQ_TOTALLY_SEEKY(bfqq))
5217− bfq_bfqq_end_wr(bfqq);
5308+ BFQQ_TOTALLY_SEEKY(bfqq)) {
5309+ if (time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt +
5310+ bfq_wr_duration(bfqd))) {
5311+ /*
5312+ * In soft_rt weight raising with the
5313+ * interactive-weight-raising period
5314+ * elapsed (so no switch back to
5315+ * interactive weight raising).
5316+ */
5317+ bfq_bfqq_end_wr(bfqq);
5318+ } else { /*
5319+ * stopping soft_rt weight raising
5320+ * while still in interactive period,
5321+ * switch back to interactive weight
5322+ * raising
5323+ */
5324+ switch_back_to_interactive_wr(bfqq, bfqd);
5325+ bfqq->entity.prio_changed = 1;
5326+ }
5327+ }
52185328 }
52195329
52205330 static void bfq_update_has_short_ttime(struct bfq_data *bfqd,
@@ -5238,12 +5348,13 @@ static void bfq_update_has_short_ttime(struct bfq_data *bfqd, · 17 unmodified lines
52385348 return;
52395349
52405350 /* Think time is infinite if no process is linked to
5241− * bfqq. Otherwise check average think time to
5242− * decide whether to mark as has_short_ttime
5351+ * bfqq. Otherwise check average think time to decide whether
5352+ * to mark as has_short_ttime. To this goal, compare average
5353+ * think time with half the I/O-plugging timeout.
52435354 */
52445355 if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||
52455356 (bfq_sample_valid(bfqq->ttime.ttime_samples) &&
5246− bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle))
5357+ bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle>>1))
52475358 has_short_ttime = false;
52485359
52495360 state_changed = has_short_ttime != bfq_bfqq_has_short_ttime(bfqq);
@@ -5557,7 +5668,6 @@ static void bfq_insert_requests(struct blk_mq_hw_ctx *hctx, · 307 unmodified lines
55575668 rq = list_first_entry(list, struct request, queuelist);
55585669 list_del_init(&rq->queuelist);
55595670 bfq_insert_request(hctx, rq, at_head);
5560− atomic_inc(&hctx->elevator_queued);
55615671 }
55625672 }
55635673
@@ -5925,7 +6035,6 @@ static void bfq_finish_requeue_request(struct request *rq) · 361 unmodified lines
59256035
59266036 bfq_completed_request(bfqq, bfqd);
59276037 bfq_finish_requeue_request_body(bfqq);
5928− atomic_dec(&rq->mq_hctx->elevator_queued);
59296038
59306039 spin_unlock_irqrestore(&bfqd->lock, flags);
59316040 } else {
@@ -6489,8 +6598,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) · 557 unmodified lines
64896598 bfqd->bfq_slice_idle = bfq_slice_idle;
64906599 bfqd->bfq_timeout = bfq_timeout;
64916600
6492− bfqd->bfq_requests_within_timer = 120;
6493−
64946601 bfqd->bfq_large_burst_thresh = 8;
64956602 bfqd->bfq_burst_interval = msecs_to_jiffies(180);
64966603
block/bfq-iosched.h+19−10
@@ -291,6 +291,11 @@ struct bfq_queue { · 290 unmodified lines
291291 /* associated @bfq_ttime struct */
292292 struct bfq_ttime ttime;
293293
294+ /* when bfqq started to do I/O within the last observation window */
295+ u64 io_start_time;
296+ /* how long bfqq has remained empty during the last observ. window */
297+ u64 tot_idle_time;
298+
294299 /* bit vector: a 1 for each seeky requests in history */
295300 u32 seek_history;
296301
@@ -371,6 +376,11 @@ struct bfq_queue { · 74 unmodified lines
371376 * bfq_select_queue().
372377 */
373378 struct bfq_queue *waker_bfqq;
379+ /* pointer to the curr. tentative waker queue, see bfq_check_waker() */
380+ struct bfq_queue *tentative_waker_bfqq;
381+ /* number of times the same tentative waker has been detected */
382+ unsigned int num_waker_detections;
383+
374384 /* node for woken_list, see below */
375385 struct hlist_node woken_list_node;
376386 /*
@@ -407,6 +417,9 @@ struct bfq_io_cq { · 30 unmodified lines
407417 */
408418 bool saved_IO_bound;
409419
420+ u64 saved_io_start_time;
421+ u64 saved_tot_idle_time;
422+
410423 /*
411424 * Same purpose as the previous fields for the value of the
412425 * field keeping the queue's belonging to a large burst
@@ -432,9 +445,15 @@ struct bfq_io_cq { · 19 unmodified lines
432445 */
433446 unsigned long saved_wr_coeff;
434447 unsigned long saved_last_wr_start_finish;
448+ unsigned long saved_service_from_wr;
435449 unsigned long saved_wr_start_at_switch_to_srt;
436450 unsigned int saved_wr_cur_max_time;
437451 struct bfq_ttime saved_ttime;
452+
453+ /* Save also injection state */
454+ u64 saved_last_serv_time_ns;
455+ unsigned int saved_inject_limit;
456+ unsigned long saved_decrease_time_jif;
438457 };
439458
440459 /**
@@ -641,14 +660,6 @@ struct bfq_data { · 200 unmodified lines
641660 */
642661 unsigned int bfq_timeout;
643662
644− /*
645− * Number of consecutive requests that must be issued within
646− * the idle time slice to set again idling to a queue which
647− * was marked as non-I/O-bound (see the definition of the
648− * IO_bound flag for further details).
649− */
650− unsigned int bfq_requests_within_timer;
651−
652663 /*
653664 * Force device idling whenever needed to provide accurate
654665 * service guarantees, without caring about throughput
@@ -770,7 +781,6 @@ enum bfqq_state_flags { · 115 unmodified lines
770781 */
771782 BFQQF_coop, /* bfqq is shared */
772783 BFQQF_split_coop, /* shared bfqq will be split */
773− BFQQF_has_waker /* bfqq has a waker queue */
774784 };
775785
776786 #define BFQ_BFQQ_FNS(name) \
@@ -790,7 +800,6 @@ BFQ_BFQQ_FNS(in_large_burst); · 13 unmodified lines
790800 BFQ_BFQQ_FNS(coop);
791801 BFQ_BFQQ_FNS(split_coop);
792802 BFQ_BFQQ_FNS(softrt_update);
793−BFQ_BFQQ_FNS(has_waker);
794803 #undef BFQ_BFQQ_FNS
795804
796805 /* Expiration reasons. */
block/bfq-wf2q.c+0−3
@@ -137,9 +137,6 @@ static bool bfq_update_next_in_service(struct bfq_sched_data *sd, · 136 unmodified lines
137137
138138 sd->next_in_service = next_in_service;
139139
140− if (!next_in_service)
141− return parent_sched_may_change;
142−
143140 return parent_sched_may_change;
144141 }
145142
block/bio-integrity.c+15−20
@@ -14,8 +14,6 @@ · 13 unmodified lines
1414 #include <linux/slab.h>
1515 #include "blk.h"
1616
17−#define BIP_INLINE_VECS 4
18−
1917 static struct kmem_cache *bip_slab;
2018 static struct workqueue_struct *kintegrityd_wq;
2119
@@ -30,7 +28,7 @@ static void __bio_integrity_free(struct bio_set *bs, · 8 unmodified lines
3028 if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
3129 if (bip->bip_vec)
3230 bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
33− bip->bip_slab);
31+ bip->bip_max_vcnt);
3432 mempool_free(bip, &bs->bio_integrity_pool);
3533 } else {
3634 kfree(bip);
@@ -63,7 +61,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, · 26 unmodified lines
6361 inline_vecs = nr_vecs;
6462 } else {
6563 bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);
66− inline_vecs = BIP_INLINE_VECS;
64+ inline_vecs = BIO_INLINE_VECS;
6765 }
6866
6967 if (unlikely(!bip))
@@ -72,14 +70,11 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, · 2 unmodified lines
7270 memset(bip, 0, sizeof(*bip));
7371
7472 if (nr_vecs > inline_vecs) {
75− unsigned long idx = 0;
76−
77− bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
78− &bs->bvec_integrity_pool);
73+ bip->bip_max_vcnt = nr_vecs;
74+ bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,
75+ &bip->bip_max_vcnt, gfp_mask);
7976 if (!bip->bip_vec)
8077 goto err;
81− bip->bip_max_vcnt = bvec_nr_vecs(idx);
82− bip->bip_slab = idx;
8378 } else {
8479 bip->bip_vec = bip->bip_inline_vecs;
8580 bip->bip_max_vcnt = inline_vecs;
@@ -140,7 +135,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, · 54 unmodified lines
140135 iv = bip->bip_vec + bip->bip_vcnt;
141136
142137 if (bip->bip_vcnt &&
143− bvec_gap_to_prev(bio->bi_disk->queue,
138+ bvec_gap_to_prev(bio->bi_bdev->bd_disk->queue,
144139 &bip->bip_vec[bip->bip_vcnt - 1], offset))
145140 return 0;
146141
@@ -162,7 +157,7 @@ EXPORT_SYMBOL(bio_integrity_add_page); · 15 unmodified lines
162157 static blk_status_t bio_integrity_process(struct bio *bio,
163158 struct bvec_iter *proc_iter, integrity_processing_fn *proc_fn)
164159 {
165− struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
160+ struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
166161 struct blk_integrity_iter iter;
167162 struct bvec_iter bviter;
168163 struct bio_vec bv;
@@ -171,7 +166,7 @@ static blk_status_t bio_integrity_process(struct bio *bio, · 2 unmodified lines
171166 void *prot_buf = page_address(bip->bip_vec->bv_page) +
172167 bip->bip_vec->bv_offset;
173168
174− iter.disk_name = bio->bi_disk->disk_name;
169+ iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
175170 iter.interval = 1 << bi->interval_exp;
176171 iter.seed = proc_iter->bi_sector;
177172 iter.prot_buf = prot_buf;
@@ -208,8 +203,8 @@ static blk_status_t bio_integrity_process(struct bio *bio, · 30 unmodified lines
208203 bool bio_integrity_prep(struct bio *bio)
209204 {
210205 struct bio_integrity_payload *bip;
211− struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
212− struct request_queue *q = bio->bi_disk->queue;
206+ struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
207+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
213208 void *buf;
214209 unsigned long start, end;
215210 unsigned int len, nr_pages;
@@ -329,7 +324,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) · 113 unmodified lines
329324 struct bio_integrity_payload *bip =
330325 container_of(work, struct bio_integrity_payload, bip_work);
331326 struct bio *bio = bip->bip_bio;
332− struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
327+ struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
333328
334329 /*
335330 * At the moment verify is called bio's iterator was advanced
@@ -355,7 +350,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) · 19 unmodified lines
355350 */
356351 bool __bio_integrity_endio(struct bio *bio)
357352 {
358− struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
353+ struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
359354 struct bio_integrity_payload *bip = bio_integrity(bio);
360355
361356 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
@@ -381,7 +376,7 @@ bool __bio_integrity_endio(struct bio *bio) · 19 unmodified lines
381376 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
382377 {
383378 struct bio_integrity_payload *bip = bio_integrity(bio);
384− struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
379+ struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
385380 unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
386381
387382 bip->bip_iter.bi_sector += bytes_done >> 9;
@@ -397,7 +392,7 @@ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) · 9 unmodified lines
397392 void bio_integrity_trim(struct bio *bio)
398393 {
399394 struct bio_integrity_payload *bip = bio_integrity(bio);
400− struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
395+ struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
401396
402397 bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
403398 }
@@ -470,6 +465,6 @@ void __init bio_integrity_init(void) · 66 unmodified lines
470465
471466 bip_slab = kmem_cache_create("bio_integrity_payload",
472467 sizeof(struct bio_integrity_payload) +
473− sizeof(struct bio_vec) * BIP_INLINE_VECS,
468+ sizeof(struct bio_vec) * BIO_INLINE_VECS,
474469 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
475470 }
block/bio.c+253−318
@@ -19,27 +19,40 @@ · 18 unmodified lines
1919 #include <linux/highmem.h>
2020 #include <linux/sched/sysctl.h>
2121 #include <linux/blk-crypto.h>
22+#include <linux/xarray.h>
2223
2324 #include <trace/events/block.h>
2425 #include "blk.h"
2526 #include "blk-rq-qos.h"
2627
27−/*
28− * Test patch to inline a certain number of bi_io_vec's inside the bio
29− * itself, to shrink a bio data allocation from two mempool calls to one
30− */
31−#define BIO_INLINE_VECS 4
32−
33−/*
34− * if you change this list, also change bvec_alloc or things will
35− * break badly! cannot be bigger than what you can fit into an
36− * unsigned short
37− */
38−#define BV(x, n) { .nr_vecs = x, .name = "biovec-"#n }
39−static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
40− BV(1, 1), BV(4, 4), BV(16, 16), BV(64, 64), BV(128, 128), BV(BIO_MAX_PAGES, max),
28+static struct biovec_slab {
29+ int nr_vecs;
30+ char *name;
31+ struct kmem_cache *slab;
32+} bvec_slabs[] __read_mostly = {
33+ { .nr_vecs = 16, .name = "biovec-16" },
34+ { .nr_vecs = 64, .name = "biovec-64" },
35+ { .nr_vecs = 128, .name = "biovec-128" },
36+ { .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },
4137 };
42−#undef BV
38+
39+static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
40+{
41+ switch (nr_vecs) {
42+ /* smaller bios use inline vecs */
43+ case 5 ... 16:
44+ return &bvec_slabs[0];
45+ case 17 ... 64:
46+ return &bvec_slabs[1];
47+ case 65 ... 128:
48+ return &bvec_slabs[2];
49+ case 129 ... BIO_MAX_PAGES:
50+ return &bvec_slabs[3];
51+ default:
52+ BUG();
53+ return NULL;
54+ }
55+}
4356
4457 /*
4558 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
@@ -58,178 +71,133 @@ struct bio_slab { · 12 unmodified lines
5871 char name[8];
5972 };
6073 static DEFINE_MUTEX(bio_slab_lock);
61−static struct bio_slab *bio_slabs;
62−static unsigned int bio_slab_nr, bio_slab_max;
74+static DEFINE_XARRAY(bio_slabs);
6375
64−static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
76+static struct bio_slab *create_bio_slab(unsigned int size)
6577 {
66− unsigned int sz = sizeof(struct bio) + extra_size;
67− struct kmem_cache *slab = NULL;
68− struct bio_slab *bslab, *new_bio_slabs;
69− unsigned int new_bio_slab_max;
70− unsigned int i, entry = -1;
78+ struct bio_slab *bslab = kzalloc(sizeof(*bslab), GFP_KERNEL);
7179
72− mutex_lock(&bio_slab_lock);
80+ if (!bslab)
81+ return NULL;
7382
74− i = 0;
75− while (i < bio_slab_nr) {
76− bslab = &bio_slabs[i];
83+ snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size);
84+ bslab->slab = kmem_cache_create(bslab->name, size,
85+ ARCH_KMALLOC_MINALIGN, SLAB_HWCACHE_ALIGN, NULL);
86+ if (!bslab->slab)
87+ goto fail_alloc_slab;
7788
78− if (!bslab->slab && entry == -1)
79− entry = i;
80− else if (bslab->slab_size == sz) {
81− slab = bslab->slab;
82− bslab->slab_ref++;
83− break;
84− }
85− i++;
86− }
89+ bslab->slab_ref = 1;
90+ bslab->slab_size = size;
8791
88− if (slab)
89− goto out_unlock;
90−
91− if (bio_slab_nr == bio_slab_max && entry == -1) {
92− new_bio_slab_max = bio_slab_max << 1;
93− new_bio_slabs = krealloc(bio_slabs,
94− new_bio_slab_max * sizeof(struct bio_slab),
95− GFP_KERNEL);
96− if (!new_bio_slabs)
97− goto out_unlock;
98− bio_slab_max = new_bio_slab_max;
99− bio_slabs = new_bio_slabs;
100− }
101− if (entry == -1)
102− entry = bio_slab_nr++;
92+ if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL)))
93+ return bslab;
94+
95+ kmem_cache_destroy(bslab->slab);
10396
104− bslab = &bio_slabs[entry];
97+fail_alloc_slab:
98+ kfree(bslab);
99+ return NULL;
100+}
101+
102+static inline unsigned int bs_bio_slab_size(struct bio_set *bs)
103+{
104+ return bs->front_pad + sizeof(struct bio) + bs->back_pad;
105+}
105106
106− snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
107− slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
108− SLAB_HWCACHE_ALIGN, NULL);
109− if (!slab)
110− goto out_unlock;
107+static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs)
108+{
109+ unsigned int size = bs_bio_slab_size(bs);
110+ struct bio_slab *bslab;
111111
112− bslab->slab = slab;
113− bslab->slab_ref = 1;
114− bslab->slab_size = sz;
115−out_unlock:
112+ mutex_lock(&bio_slab_lock);
113+ bslab = xa_load(&bio_slabs, size);
114+ if (bslab)
115+ bslab->slab_ref++;
116+ else
117+ bslab = create_bio_slab(size);
116118 mutex_unlock(&bio_slab_lock);
117− return slab;
119+
120+ if (bslab)
121+ return bslab->slab;
122+ return NULL;
118123 }
119124
120125 static void bio_put_slab(struct bio_set *bs)
121126 {
122127 struct bio_slab *bslab = NULL;
123− unsigned int i;
128+ unsigned int slab_size = bs_bio_slab_size(bs);
124129
125130 mutex_lock(&bio_slab_lock);
126131
127− for (i = 0; i < bio_slab_nr; i++) {
128− if (bs->bio_slab == bio_slabs[i].slab) {
129− bslab = &bio_slabs[i];
130− break;
131− }
132− }
133−
132+ bslab = xa_load(&bio_slabs, slab_size);
134133 if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
135134 goto out;
136135
136+ WARN_ON_ONCE(bslab->slab != bs->bio_slab);
137+
137138 WARN_ON(!bslab->slab_ref);
138139
139140 if (--bslab->slab_ref)
140141 goto out;
141142
143+ xa_erase(&bio_slabs, slab_size);
144+
142145 kmem_cache_destroy(bslab->slab);
143− bslab->slab = NULL;
146+ kfree(bslab);
144147
145148 out:
146149 mutex_unlock(&bio_slab_lock);
147150 }
148151
149−unsigned int bvec_nr_vecs(unsigned short idx)
152+void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs)
150153 {
151− return bvec_slabs[--idx].nr_vecs;
152−}
153−
154−void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
155−{
156− if (!idx)
157− return;
158− idx--;
154+ BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES);
159155
160− BIO_BUG_ON(idx >= BVEC_POOL_NR);
161−
162− if (idx == BVEC_POOL_MAX) {
156+ if (nr_vecs == BIO_MAX_PAGES)
163157 mempool_free(bv, pool);
164− } else {
165− struct biovec_slab *bvs = bvec_slabs + idx;
158+ else if (nr_vecs > BIO_INLINE_VECS)
159+ kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
160+}
166161
167− kmem_cache_free(bvs->slab, bv);
168− }
162+/*
163+ * Make the first allocation restricted and don't dump info on allocation
164+ * failures, since we'll fall back to the mempool in case of failure.
165+ */
166+static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
167+{
168+ return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) |
169+ __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
169170 }
170171
171−struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
172− mempool_t *pool)
172+struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
173+ gfp_t gfp_mask)
173174 {
174− struct bio_vec *bvl;
175+ struct biovec_slab *bvs = biovec_slab(*nr_vecs);
175176
176− /*
177− * see comment near bvec_array define!
178− */
179− switch (nr) {
180− case 1:
181− *idx = 0;
182− break;
183− case 2 ... 4:
184− *idx = 1;
185− break;
186− case 5 ... 16:
187− *idx = 2;
188− break;
189− case 17 ... 64:
190− *idx = 3;
191− break;
192− case 65 ... 128:
193− *idx = 4;
194− break;
195− case 129 ... BIO_MAX_PAGES:
196− *idx = 5;
197− break;
198− default:
177+ if (WARN_ON_ONCE(!bvs))
199178 return NULL;
200− }
201179
202180 /*
203− * idx now points to the pool we want to allocate from. only the
204− * 1-vec entry pool is mempool backed.
181+ * Upgrade the nr_vecs request to take full advantage of the allocation.
182+ * We also rely on this in the bvec_free path.
205183 */
206− if (*idx == BVEC_POOL_MAX) {
207−fallback:
208− bvl = mempool_alloc(pool, gfp_mask);
209− } else {
210− struct biovec_slab *bvs = bvec_slabs + *idx;
211− gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
184+ *nr_vecs = bvs->nr_vecs;
212185
213− /*
214− * Make this allocation restricted and don't dump info on
215− * allocation failures, since we'll fallback to the mempool
216− * in case of failure.
217− */
218− __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
186+ /*
187+ * Try a slab allocation first for all smaller allocations. If that
188+ * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
189+ * The mempool is sized to handle up to BIO_MAX_PAGES entries.
190+ */
191+ if (*nr_vecs < BIO_MAX_PAGES) {
192+ struct bio_vec *bvl;
219193
220− /*
221− * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
222− * is set, retry with the 1-entry mempool
223− */
224− bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
225− if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
226− *idx = BVEC_POOL_MAX;
227− goto fallback;
228− }
194+ bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
195+ if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM))
196+ return bvl;
197+ *nr_vecs = BIO_MAX_PAGES;
229198 }
230199
231− (*idx)++;
232− return bvl;
200+ return mempool_alloc(pool, gfp_mask);
233201 }
234202
235203 void bio_uninit(struct bio *bio)
@@ -255,7 +223,7 @@ static void bio_free(struct bio *bio) · 19 unmodified lines
255223 bio_uninit(bio);
256224
257225 if (bs) {
258− bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
226+ bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
259227
260228 /*
261229 * If we have front padding, adjust the bio pointer before freeing
@@ -299,12 +267,8 @@ EXPORT_SYMBOL(bio_init); · 37 unmodified lines
299267 */
300268 void bio_reset(struct bio *bio)
301269 {
302− unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
303−
304270 bio_uninit(bio);
305−
306271 memset(bio, 0, BIO_RESET_BYTES);
307− bio->bi_flags = flags;
308272 atomic_set(&bio->__bi_remaining, 1);
309273 }
310274 EXPORT_SYMBOL(bio_reset);
@@ -405,122 +369,97 @@ static void punt_bios_to_rescuer(struct bio_set *bs) · 94 unmodified lines
405369 * @nr_iovecs: number of iovecs to pre-allocate
406370 * @bs: the bio_set to allocate from.
407371 *
408− * Description:
409− * If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
410− * backed by the @bs's mempool.
372+ * Allocate a bio from the mempools in @bs.
411373 *
412− * When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
413− * always be able to allocate a bio. This is due to the mempool guarantees.
414− * To make this work, callers must never allocate more than 1 bio at a time
415− * from this pool. Callers that need to allocate more than 1 bio must always
416− * submit the previously allocated bio for IO before attempting to allocate
417− * a new one. Failure to do so can cause deadlocks under memory pressure.
374+ * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to
375+ * allocate a bio. This is due to the mempool guarantees. To make this work,
376+ * callers must never allocate more than 1 bio at a time from the general pool.
377+ * Callers that need to allocate more than 1 bio must always submit the
378+ * previously allocated bio for IO before attempting to allocate a new one.
379+ * Failure to do so can cause deadlocks under memory pressure.
418380 *
419− * Note that when running under submit_bio_noacct() (i.e. any block
420− * driver), bios are not submitted until after you return - see the code in
421− * submit_bio_noacct() that converts recursion into iteration, to prevent
422− * stack overflows.
381+ * Note that when running under submit_bio_noacct() (i.e. any block driver),
382+ * bios are not submitted until after you return - see the code in
383+ * submit_bio_noacct() that converts recursion into iteration, to prevent
384+ * stack overflows.
423385 *
424− * This would normally mean allocating multiple bios under
425− * submit_bio_noacct() would be susceptible to deadlocks, but we have
426− * deadlock avoidance code that resubmits any blocked bios from a rescuer
427− * thread.
386+ * This would normally mean allocating multiple bios under submit_bio_noacct()
387+ * would be susceptible to deadlocks, but we have
388+ * deadlock avoidance code that resubmits any blocked bios from a rescuer
389+ * thread.
428390 *
429− * However, we do not guarantee forward progress for allocations from other
430− * mempools. Doing multiple allocations from the same mempool under
431− * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
432− * for per bio allocations.
391+ * However, we do not guarantee forward progress for allocations from other
392+ * mempools. Doing multiple allocations from the same mempool under
393+ * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
394+ * for per bio allocations.
433395 *
434− * RETURNS:
435− * Pointer to new bio on success, NULL on failure.
396+ * Returns: Pointer to new bio on success, NULL on failure.
436397 */
437−struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
398+struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs,
438399 struct bio_set *bs)
439400 {
440401 gfp_t saved_gfp = gfp_mask;
441− unsigned front_pad;
442− unsigned inline_vecs;
443− struct bio_vec *bvl = NULL;
444402 struct bio *bio;
445403 void *p;
446404
447− if (!bs) {
448− if (nr_iovecs > UIO_MAXIOV)
449− return NULL;
450−
451− p = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask);
452− front_pad = 0;
453− inline_vecs = nr_iovecs;
454− } else {
455− /* should not use nobvec bioset for nr_iovecs > 0 */
456− if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&
457− nr_iovecs > 0))
458− return NULL;
459− /*
460− * submit_bio_noacct() converts recursion to iteration; this
461− * means if we're running beneath it, any bios we allocate and
462− * submit will not be submitted (and thus freed) until after we
463− * return.
464− *
465− * This exposes us to a potential deadlock if we allocate
466− * multiple bios from the same bio_set() while running
467− * underneath submit_bio_noacct(). If we were to allocate
468− * multiple bios (say a stacking block driver that was splitting
469− * bios), we would deadlock if we exhausted the mempool's
470− * reserve.
471− *
472− * We solve this, and guarantee forward progress, with a rescuer
473− * workqueue per bio_set. If we go to allocate and there are
474− * bios on current->bio_list, we first try the allocation
475− * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
476− * bios we would be blocking to the rescuer workqueue before
477− * we retry with the original gfp_flags.
478− */
479−
480− if (current->bio_list &&
481− (!bio_list_empty(&current->bio_list[0]) ||
482− !bio_list_empty(&current->bio_list[1])) &&
483− bs->rescue_workqueue)
484− gfp_mask &= ~__GFP_DIRECT_RECLAIM;
405+ /* should not use nobvec bioset for nr_iovecs > 0 */
406+ if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_iovecs > 0))
407+ return NULL;
485408
409+ /*
410+ * submit_bio_noacct() converts recursion to iteration; this means if
411+ * we're running beneath it, any bios we allocate and submit will not be
412+ * submitted (and thus freed) until after we return.
413+ *
414+ * This exposes us to a potential deadlock if we allocate multiple bios
415+ * from the same bio_set() while running underneath submit_bio_noacct().
416+ * If we were to allocate multiple bios (say a stacking block driver
417+ * that was splitting bios), we would deadlock if we exhausted the
418+ * mempool's reserve.
419+ *
420+ * We solve this, and guarantee forward progress, with a rescuer
421+ * workqueue per bio_set. If we go to allocate and there are bios on
422+ * current->bio_list, we first try the allocation without
423+ * __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be
424+ * blocking to the rescuer workqueue before we retry with the original
425+ * gfp_flags.
426+ */
427+ if (current->bio_list &&
428+ (!bio_list_empty(&current->bio_list[0]) ||
429+ !bio_list_empty(&current->bio_list[1])) &&
430+ bs->rescue_workqueue)
431+ gfp_mask &= ~__GFP_DIRECT_RECLAIM;
432+
433+ p = mempool_alloc(&bs->bio_pool, gfp_mask);
434+ if (!p && gfp_mask != saved_gfp) {
435+ punt_bios_to_rescuer(bs);
436+ gfp_mask = saved_gfp;
486437 p = mempool_alloc(&bs->bio_pool, gfp_mask);
487− if (!p && gfp_mask != saved_gfp) {
488− punt_bios_to_rescuer(bs);
489− gfp_mask = saved_gfp;
490− p = mempool_alloc(&bs->bio_pool, gfp_mask);
491− }
492−
493− front_pad = bs->front_pad;
494− inline_vecs = BIO_INLINE_VECS;
495438 }
496−
497439 if (unlikely(!p))
498440 return NULL;
499441
500− bio = p + front_pad;
501− bio_init(bio, NULL, 0);
442+ bio = p + bs->front_pad;
443+ if (nr_iovecs > BIO_INLINE_VECS) {
444+ struct bio_vec *bvl = NULL;
502445
503− if (nr_iovecs > inline_vecs) {
504− unsigned long idx = 0;
505−
506− bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
446+ bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
507447 if (!bvl && gfp_mask != saved_gfp) {
508448 punt_bios_to_rescuer(bs);
509449 gfp_mask = saved_gfp;
510− bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
450+ bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
511451 }
512−
513452 if (unlikely(!bvl))
514453 goto err_free;
515454
516− bio->bi_flags |= idx << BVEC_POOL_OFFSET;
455+ bio_init(bio, bvl, nr_iovecs);
517456 } else if (nr_iovecs) {
518− bvl = bio->bi_inline_vecs;
457+ bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS);
458+ } else {
459+ bio_init(bio, NULL, 0);
519460 }
520461
521462 bio->bi_pool = bs;
522− bio->bi_max_vecs = nr_iovecs;
523− bio->bi_io_vec = bvl;
524463 return bio;
525464
526465 err_free:
@@ -529,6 +468,31 @@ err_free: · 2 unmodified lines
529468 }
530469 EXPORT_SYMBOL(bio_alloc_bioset);
531470
471+/**
472+ * bio_kmalloc - kmalloc a bio for I/O
473+ * @gfp_mask: the GFP_* mask given to the slab allocator
474+ * @nr_iovecs: number of iovecs to pre-allocate
475+ *
476+ * Use kmalloc to allocate and initialize a bio.
477+ *
478+ * Returns: Pointer to new bio on success, NULL on failure.
479+ */
480+struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs)
481+{
482+ struct bio *bio;
483+
484+ if (nr_iovecs > UIO_MAXIOV)
485+ return NULL;
486+
487+ bio = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask);
488+ if (unlikely(!bio))
489+ return NULL;
490+ bio_init(bio, nr_iovecs ? bio->bi_inline_vecs : NULL, nr_iovecs);
491+ bio->bi_pool = NULL;
492+ return bio;
493+}
494+EXPORT_SYMBOL(bio_kmalloc);
495+
532496 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
533497 {
534498 unsigned long flags;
@@ -607,16 +571,7 @@ void bio_truncate(struct bio *bio, unsigned new_size) · 72 unmodified lines
607571 */
608572 void guard_bio_eod(struct bio *bio)
609573 {
610− sector_t maxsector;
611− struct block_device *part;
612−
613− rcu_read_lock();
614− part = __disk_get_part(bio->bi_disk, bio->bi_partno);
615− if (part)
616− maxsector = bdev_nr_sectors(part);
617− else
618− maxsector = get_capacity(bio->bi_disk);
619− rcu_read_unlock();
574+ sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
620575
621576 if (!maxsector)
622577 return;
@@ -673,17 +628,18 @@ EXPORT_SYMBOL(bio_put); · 50 unmodified lines
673628 */
674629 void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
675630 {
676− BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
631+ WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs);
677632
678633 /*
679− * most users will be overriding ->bi_disk with a new target,
634+ * most users will be overriding ->bi_bdev with a new target,
680635 * so we don't set nor calculate new physical/hw segment counts here
681636 */
682− bio->bi_disk = bio_src->bi_disk;
683− bio->bi_partno = bio_src->bi_partno;
637+ bio->bi_bdev = bio_src->bi_bdev;
684638 bio_set_flag(bio, BIO_CLONED);
685639 if (bio_flagged(bio_src, BIO_THROTTLED))
686640 bio_set_flag(bio, BIO_THROTTLED);
641+ if (bio_flagged(bio_src, BIO_REMAPPED))
642+ bio_set_flag(bio, BIO_REMAPPED);
687643 bio->bi_opf = bio_src->bi_opf;
688644 bio->bi_ioprio = bio_src->bi_ioprio;
689645 bio->bi_write_hint = bio_src->bi_write_hint;
@@ -730,7 +686,7 @@ EXPORT_SYMBOL(bio_clone_fast); · 40 unmodified lines
730686
731687 const char *bio_devname(struct bio *bio, char *buf)
732688 {
733− return disk_name(bio->bi_disk, bio->bi_partno, buf);
689+ return bdevname(bio->bi_bdev, buf);
734690 }
735691 EXPORT_SYMBOL(bio_devname);
736692
@@ -870,7 +826,7 @@ EXPORT_SYMBOL(bio_add_pc_page); · 133 unmodified lines
870826 int bio_add_zone_append_page(struct bio *bio, struct page *page,
871827 unsigned int len, unsigned int offset)
872828 {
873− struct request_queue *q = bio->bi_disk->queue;
829+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
874830 bool same_page = false;
875831
876832 if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND))
@@ -993,21 +949,18 @@ void bio_release_pages(struct bio *bio, bool mark_dirty) · 116 unmodified lines
993949 }
994950 EXPORT_SYMBOL_GPL(bio_release_pages);
995951
996−static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
952+static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
997953 {
998− const struct bio_vec *bv = iter->bvec;
999− unsigned int len;
1000− size_t size;
1001−
1002− if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len))
1003− return -EINVAL;
1004−
1005− len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count);
1006− size = bio_add_page(bio, bv->bv_page, len,
1007− bv->bv_offset + iter->iov_offset);
1008− if (unlikely(size != len))
1009− return -EINVAL;
1010− iov_iter_advance(iter, size);
954+ WARN_ON_ONCE(bio->bi_max_vecs);
955+
956+ bio->bi_vcnt = iter->nr_segs;
957+ bio->bi_io_vec = (struct bio_vec *)iter->bvec;
958+ bio->bi_iter.bi_bvec_done = iter->iov_offset;
959+ bio->bi_iter.bi_size = iter->count;
960+ bio_set_flag(bio, BIO_NO_PAGE_REF);
961+ bio_set_flag(bio, BIO_CLONED);
962+
963+ iov_iter_advance(iter, iter->count);
1011964 return 0;
1012965 }
1013966
@@ -1070,7 +1023,7 @@ static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) · 56 unmodified lines
10701023 {
10711024 unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
10721025 unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
1073− struct request_queue *q = bio->bi_disk->queue;
1026+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
10741027 unsigned int max_append_sectors = queue_max_zone_append_sectors(q);
10751028 struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
10761029 struct page **pages = (struct page **)bv;
@@ -1121,41 +1074,40 @@ static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) · 44 unmodified lines
11211074 * This takes either an iterator pointing to user memory, or one pointing to
11221075 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
11231076 * map them into the kernel. On IO completion, the caller should put those
1124− * pages. If we're adding kernel pages, and the caller told us it's safe to
1125− * do so, we just have to add the pages to the bio directly. We don't grab an
1126− * extra reference to those pages (the user should already have that), and we
1127− * don't put the page on IO completion. The caller needs to check if the bio is
1128− * flagged BIO_NO_PAGE_REF on IO completion. If it isn't, then pages should be
1129− * released.
1077+ * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided
1078+ * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs
1079+ * to ensure the bvecs and pages stay referenced until the submitted I/O is
1080+ * completed by a call to ->ki_complete() or returns with an error other than
1081+ * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF
1082+ * on IO completion. If it isn't, then pages should be released.
11301083 *
11311084 * The function tries, but does not guarantee, to pin as many pages as
11321085 * fit into the bio, or are requested in @iter, whatever is smaller. If
11331086 * MM encounters an error pinning the requested pages, it stops. Error
11341087 * is returned only if 0 pages could be pinned.
1088+ *
1089+ * It's intended for direct IO, so doesn't do PSI tracking, the caller is
1090+ * responsible for setting BIO_WORKINGSET if necessary.
11351091 */
11361092 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
11371093 {
1138− const bool is_bvec = iov_iter_is_bvec(iter);
1139− int ret;
1094+ int ret = 0;
11401095
1141− if (WARN_ON_ONCE(bio->bi_vcnt))
1142− return -EINVAL;
1096+ if (iov_iter_is_bvec(iter)) {
1097+ if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND))
1098+ return -EINVAL;
1099+ return bio_iov_bvec_set(bio, iter);
1100+ }
11431101
11441102 do {
1145− if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
1146− if (WARN_ON_ONCE(is_bvec))
1147− return -EINVAL;
1103+ if (bio_op(bio) == REQ_OP_ZONE_APPEND)
11481104 ret = __bio_iov_append_get_pages(bio, iter);
1149− } else {
1150− if (is_bvec)
1151− ret = __bio_iov_bvec_add_pages(bio, iter);
1152− else
1153− ret = __bio_iov_iter_get_pages(bio, iter);
1154− }
1105+ else
1106+ ret = __bio_iov_iter_get_pages(bio, iter);
11551107 } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
11561108
1157− if (is_bvec)
1158− bio_set_flag(bio, BIO_NO_PAGE_REF);
1109+ /* don't account direct I/O as memory stall */
1110+ bio_clear_flag(bio, BIO_WORKINGSET);
11591111 return bio->bi_vcnt ? 0 : ret;
11601112 }
11611113 EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
@@ -1178,7 +1130,8 @@ static void submit_bio_wait_endio(struct bio *bio) · 16 unmodified lines
11781130 */
11791131 int submit_bio_wait(struct bio *bio)
11801132 {
1181− DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_disk->lockdep_map);
1133+ DECLARE_COMPLETION_ONSTACK_MAP(done,
1134+ bio->bi_bdev->bd_disk->lockdep_map);
11821135 unsigned long hang_check;
11831136
11841137 bio->bi_private = &done;
@@ -1455,8 +1408,8 @@ again: · 270 unmodified lines
14551408 if (!bio_integrity_endio(bio))
14561409 return;
14571410
1458− if (bio->bi_disk)
1459− rq_qos_done_bio(bio->bi_disk->queue, bio);
1411+ if (bio->bi_bdev)
1412+ rq_qos_done_bio(bio->bi_bdev->bd_disk->queue, bio);
14601413
14611414 /*
14621415 * Need to have a real endio function for chained bios, otherwise
@@ -1471,8 +1424,8 @@ again: · 8 unmodified lines
14711424 goto again;
14721425 }
14731426
1474− if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
1475− trace_block_bio_complete(bio->bi_disk->queue, bio);
1427+ if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
1428+ trace_block_bio_complete(bio->bi_bdev->bd_disk->queue, bio);
14761429 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
14771430 }
14781431
@@ -1559,7 +1512,7 @@ EXPORT_SYMBOL_GPL(bio_trim); · 80 unmodified lines
15591512 */
15601513 int biovec_init_pool(mempool_t *pool, int pool_entries)
15611514 {
1562− struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
1515+ struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;
15631516
15641517 return mempool_init_slab_pool(pool, pool_entries, bp->slab);
15651518 }
@@ -1612,15 +1565,17 @@ int bioset_init(struct bio_set *bs, · 46 unmodified lines
16121565 unsigned int front_pad,
16131566 int flags)
16141567 {
1615− unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1616−
16171568 bs->front_pad = front_pad;
1569+ if (flags & BIOSET_NEED_BVECS)
1570+ bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1571+ else
1572+ bs->back_pad = 0;
16181573
16191574 spin_lock_init(&bs->rescue_lock);
16201575 bio_list_init(&bs->rescue_list);
16211576 INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
16221577
1623− bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
1578+ bs->bio_slab = bio_find_or_create_slab(bs);
16241579 if (!bs->bio_slab)
16251580 return -ENOMEM;
16261581
@@ -1663,39 +1618,19 @@ int bioset_init_from_src(struct bio_set *bs, struct bio_set *src) · 36 unmodified lines
16631618 }
16641619 EXPORT_SYMBOL(bioset_init_from_src);
16651620
1666−static void __init biovec_init_slabs(void)
1621+static int __init init_bio(void)
16671622 {
16681623 int i;
16691624
1670− for (i = 0; i < BVEC_POOL_NR; i++) {
1671− int size;
1672− struct biovec_slab *bvs = bvec_slabs + i;
1625+ bio_integrity_init();
16731626
1674− if (bvs->nr_vecs <= BIO_INLINE_VECS) {
1675− bvs->slab = NULL;
1676− continue;
1677− }
1627+ for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
1628+ struct biovec_slab *bvs = bvec_slabs + i;
16781629
1679− size = bvs->nr_vecs * sizeof(struct bio_vec);
1680− bvs->slab = kmem_cache_create(bvs->name, size, 0,
1681− SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
1630+ bvs->slab = kmem_cache_create(bvs->name,
1631+ bvs->nr_vecs * sizeof(struct bio_vec), 0,
1632+ SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
16821633 }
1683−}
1684−
1685−static int __init init_bio(void)
1686−{
1687− bio_slab_max = 2;
1688− bio_slab_nr = 0;
1689− bio_slabs = kcalloc(bio_slab_max, sizeof(struct bio_slab),
1690− GFP_KERNEL);
1691−
1692− BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);
1693−
1694− if (!bio_slabs)
1695− panic("bio: can't allocate bios\n");
1696−
1697− bio_integrity_init();
1698− biovec_init_slabs();
16991634
17001635 if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))
17011636 panic("bio: can't allocate bios\n");
block/blk-cgroup.c+12−10
@@ -32,8 +32,6 @@ · 31 unmodified lines
3232 #include <linux/psi.h>
3333 #include "blk.h"
3434
35−#define MAX_KEY_LEN 100
36−
3735 /*
3836 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
3937 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
@@ -1765,12 +1763,15 @@ void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay) · 1725 unmodified lines
17651763 if (unlikely(current->flags & PF_KTHREAD))
17661764 return;
17671765
1768− if (!blk_get_queue(q))
1769− return;
1766+ if (current->throttle_queue != q) {
1767+ if (!blk_get_queue(q))
1768+ return;
1769+
1770+ if (current->throttle_queue)
1771+ blk_put_queue(current->throttle_queue);
1772+ current->throttle_queue = q;
1773+ }
17701774
1771− if (current->throttle_queue)
1772− blk_put_queue(current->throttle_queue);
1773− current->throttle_queue = q;
17741775 if (use_memdelay)
17751776 current->use_memdelay = use_memdelay;
17761777 set_notify_resume(current);
@@ -1808,7 +1809,8 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, · 31 unmodified lines
18081809 struct blkcg_gq *blkg, *ret_blkg = NULL;
18091810
18101811 rcu_read_lock();
1811− blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_disk->queue);
1812+ blkg = blkg_lookup_create(css_to_blkcg(css),
1813+ bio->bi_bdev->bd_disk->queue);
18121814 while (blkg) {
18131815 if (blkg_tryget(blkg)) {
18141816 ret_blkg = blkg;
@@ -1844,8 +1846,8 @@ void bio_associate_blkg_from_css(struct bio *bio, · 29 unmodified lines
18441846 if (css && css->parent) {
18451847 bio->bi_blkg = blkg_tryget_closest(bio, css);
18461848 } else {
1847− blkg_get(bio->bi_disk->queue->root_blkg);
1848− bio->bi_blkg = bio->bi_disk->queue->root_blkg;
1849+ blkg_get(bio->bi_bdev->bd_disk->queue->root_blkg);
1850+ bio->bi_blkg = bio->bi_bdev->bd_disk->queue->root_blkg;
18491851 }
18501852 }
18511853 EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
block/blk-core.c+46−53
@@ -476,7 +476,7 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) · 475 unmodified lines
476476
477477 static inline int bio_queue_enter(struct bio *bio)
478478 {
479− struct request_queue *q = bio->bi_disk->queue;
479+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
480480 bool nowait = bio->bi_opf & REQ_NOWAIT;
481481 int ret;
482482
@@ -531,7 +531,7 @@ struct request_queue *blk_alloc_queue(int node_id) · 48 unmodified lines
531531 if (q->id < 0)
532532 goto fail_q;
533533
534− ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
534+ ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
535535 if (ret)
536536 goto fail_id;
537537
@@ -692,11 +692,9 @@ static inline bool should_fail_request(struct block_device *part, · 154 unmodified lines
692692
693693 #endif /* CONFIG_FAIL_MAKE_REQUEST */
694694
695−static inline bool bio_check_ro(struct bio *bio, struct block_device *part)
695+static inline bool bio_check_ro(struct bio *bio)
696696 {
697− const int op = bio_op(bio);
698−
699− if (part->bd_read_only && op_is_write(op)) {
697+ if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) {
700698 char b[BDEVNAME_SIZE];
701699
702700 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
@@ -704,7 +702,7 @@ static inline bool bio_check_ro(struct bio *bio, struct block_device *part) · 1 unmodified line
704702
705703 WARN_ONCE(1,
706704 "Trying to write to read-only block-device %s (partno %d)\n",
707− bio_devname(bio, b), part->bd_partno);
705+ bio_devname(bio, b), bio->bi_bdev->bd_partno);
708706 /* Older lvm-tools actually trigger this */
709707 return false;
710708 }
@@ -714,7 +712,7 @@ static inline bool bio_check_ro(struct bio *bio, struct block_device *part) · 3 unmodified lines
714712
715713 static noinline int should_fail_bio(struct bio *bio)
716714 {
717− if (should_fail_request(bio->bi_disk->part0, bio->bi_iter.bi_size))
715+ if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size))
718716 return -EIO;
719717 return 0;
720718 }
@@ -725,8 +723,9 @@ ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO); · 4 unmodified lines
725723 * This may well happen - the kernel calls bread() without checking the size of
726724 * the device, e.g., when mounting a file system.
727725 */
728−static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
726+static inline int bio_check_eod(struct bio *bio)
729727 {
728+ sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
730729 unsigned int nr_sectors = bio_sectors(bio);
731730
732731 if (nr_sectors && maxsector &&
@@ -741,33 +740,20 @@ static inline int bio_check_eod(struct bio *bio, sector_t maxsector) · 8 unmodified lines
741740 /*
742741 * Remap block n of partition p to block n+start(p) of the disk.
743742 */
744−static inline int blk_partition_remap(struct bio *bio)
743+static int blk_partition_remap(struct bio *bio)
745744 {
746− struct block_device *p;
747− int ret = -EIO;
745+ struct block_device *p = bio->bi_bdev;
748746
749− rcu_read_lock();
750− p = __disk_get_part(bio->bi_disk, bio->bi_partno);
751− if (unlikely(!p))
752− goto out;
753747 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
754− goto out;
755− if (unlikely(bio_check_ro(bio, p)))
756− goto out;
757−
748+ return -EIO;
758749 if (bio_sectors(bio)) {
759− if (bio_check_eod(bio, bdev_nr_sectors(p)))
760− goto out;
761750 bio->bi_iter.bi_sector += p->bd_start_sect;
762751 trace_block_bio_remap(bio, p->bd_dev,
763752 bio->bi_iter.bi_sector -
764753 p->bd_start_sect);
765754 }
766− bio->bi_partno = 0;
767− ret = 0;
768−out:
769− rcu_read_unlock();
770− return ret;
755+ bio_set_flag(bio, BIO_REMAPPED);
756+ return 0;
771757 }
772758
773759 /*
@@ -807,7 +793,8 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q, · 33 unmodified lines
807793
808794 static noinline_for_stack bool submit_bio_checks(struct bio *bio)
809795 {
810− struct request_queue *q = bio->bi_disk->queue;
796+ struct block_device *bdev = bio->bi_bdev;
797+ struct request_queue *q = bdev->bd_disk->queue;
811798 blk_status_t status = BLK_STS_IOERR;
812799 struct blk_plug *plug;
813800
@@ -826,14 +813,12 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio) · 12 unmodified lines
826813
827814 if (should_fail_bio(bio))
828815 goto end_io;
829−
830− if (bio->bi_partno) {
831− if (unlikely(blk_partition_remap(bio)))
832− goto end_io;
833− } else {
834− if (unlikely(bio_check_ro(bio, bio->bi_disk->part0)))
816+ if (unlikely(bio_check_ro(bio)))
817+ goto end_io;
818+ if (!bio_flagged(bio, BIO_REMAPPED)) {
819+ if (unlikely(bio_check_eod(bio)))
835820 goto end_io;
836− if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
821+ if (bdev->bd_partno && unlikely(blk_partition_remap(bio)))
837822 goto end_io;
838823 }
839824
@@ -926,7 +911,7 @@ end_io: · 86 unmodified lines
926911
927912 static blk_qc_t __submit_bio(struct bio *bio)
928913 {
929− struct gendisk *disk = bio->bi_disk;
914+ struct gendisk *disk = bio->bi_bdev->bd_disk;
930915 blk_qc_t ret = BLK_QC_T_NONE;
931916
932917 if (blk_crypto_bio_prep(&bio)) {
@@ -968,7 +953,7 @@ static blk_qc_t __submit_bio_noacct(struct bio *bio) · 35 unmodified lines
968953 current->bio_list = bio_list_on_stack;
969954
970955 do {
971− struct request_queue *q = bio->bi_disk->queue;
956+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
972957 struct bio_list lower, same;
973958
974959 if (unlikely(bio_queue_enter(bio) != 0))
@@ -989,7 +974,7 @@ static blk_qc_t __submit_bio_noacct(struct bio *bio) · 14 unmodified lines
989974 bio_list_init(&lower);
990975 bio_list_init(&same);
991976 while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
992− if (q == bio->bi_disk->queue)
977+ if (q == bio->bi_bdev->bd_disk->queue)
993978 bio_list_add(&same, bio);
994979 else
995980 bio_list_add(&lower, bio);
@@ -1014,7 +999,7 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) · 18 unmodified lines
1014999 current->bio_list = bio_list;
10151000
10161001 do {
1017− struct gendisk *disk = bio->bi_disk;
1002+ struct gendisk *disk = bio->bi_bdev->bd_disk;
10181003
10191004 if (unlikely(bio_queue_enter(bio) != 0))
10201005 continue;
@@ -1057,7 +1042,7 @@ blk_qc_t submit_bio_noacct(struct bio *bio) · 36 unmodified lines
10571042 return BLK_QC_T_NONE;
10581043 }
10591044
1060− if (!bio->bi_disk->fops->submit_bio)
1045+ if (!bio->bi_bdev->bd_disk->fops->submit_bio)
10611046 return __submit_bio_noacct_mq(bio);
10621047 return __submit_bio_noacct(bio);
10631048 }
@@ -1069,7 +1054,7 @@ EXPORT_SYMBOL(submit_bio_noacct); · 5 unmodified lines
10691054 *
10701055 * submit_bio() is used to submit I/O requests to block devices. It is passed a
10711056 * fully set up &struct bio that describes the I/O that needs to be done. The
1072− * bio will be send to the device described by the bi_disk and bi_partno fields.
1057+ * bio will be send to the device described by the bi_bdev field.
10731058 *
10741059 * The success/failure status of the request, along with notification of
10751060 * completion, is delivered asynchronously through the ->bi_end_io() callback
@@ -1089,7 +1074,8 @@ blk_qc_t submit_bio(struct bio *bio) · 13 unmodified lines
10891074 unsigned int count;
10901075
10911076 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
1092− count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
1077+ count = queue_logical_block_size(
1078+ bio->bi_bdev->bd_disk->queue) >> 9;
10931079 else
10941080 count = bio_sectors(bio);
10951081
@@ -1313,7 +1299,11 @@ void blk_account_io_start(struct request *rq) · 217 unmodified lines
13131299 if (!blk_do_io_stat(rq))
13141300 return;
13151301
1316− rq->part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
1302+ /* passthrough requests can hold bios that do not have ->bi_bdev set */
1303+ if (rq->bio && rq->bio->bi_bdev)
1304+ rq->part = rq->bio->bi_bdev;
1305+ else
1306+ rq->part = rq->rq_disk->part0;
13171307
13181308 part_stat_lock();
13191309 update_io_ticks(rq->part, jiffies, false);
@@ -1336,14 +1326,17 @@ static unsigned long __part_start_io_acct(struct block_device *part, · 16 unmodified lines
13361326 return now;
13371327 }
13381328
1339−unsigned long part_start_io_acct(struct gendisk *disk, struct block_device **part,
1340− struct bio *bio)
1329+/**
1330+ * bio_start_io_acct - start I/O accounting for bio based drivers
1331+ * @bio: bio to start account for
1332+ *
1333+ * Returns the start time that should be passed back to bio_end_io_acct().
1334+ */
1335+unsigned long bio_start_io_acct(struct bio *bio)
13411336 {
1342− *part = disk_map_sector_rcu(disk, bio->bi_iter.bi_sector);
1343−
1344− return __part_start_io_acct(*part, bio_sectors(bio), bio_op(bio));
1337+ return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), bio_op(bio));
13451338 }
1346−EXPORT_SYMBOL_GPL(part_start_io_acct);
1339+EXPORT_SYMBOL_GPL(bio_start_io_acct);
13471340
13481341 unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
13491342 unsigned int op)
@@ -1366,12 +1359,12 @@ static void __part_end_io_acct(struct block_device *part, unsigned int op, · 16 unmodified lines
13661359 part_stat_unlock();
13671360 }
13681361
1369−void part_end_io_acct(struct block_device *part, struct bio *bio,
1370− unsigned long start_time)
1362+void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1363+ struct block_device *orig_bdev)
13711364 {
1372− __part_end_io_acct(part, bio_op(bio), start_time);
1365+ __part_end_io_acct(orig_bdev, bio_op(bio), start_time);
13731366 }
1374−EXPORT_SYMBOL_GPL(part_end_io_acct);
1367+EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped);
13751368
13761369 void disk_end_io_acct(struct gendisk *disk, unsigned int op,
13771370 unsigned long start_time)
block/blk-crypto-fallback.c+4−2
@@ -164,10 +164,12 @@ static struct bio *blk_crypto_clone_bio(struct bio *bio_src) · 163 unmodified lines
164164 struct bio_vec bv;
165165 struct bio *bio;
166166
167− bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL);
167+ bio = bio_kmalloc(GFP_NOIO, bio_segments(bio_src));
168168 if (!bio)
169169 return NULL;
170− bio->bi_disk = bio_src->bi_disk;
170+ bio->bi_bdev = bio_src->bi_bdev;
171+ if (bio_flagged(bio_src, BIO_REMAPPED))
172+ bio_set_flag(bio, BIO_REMAPPED);
171173 bio->bi_opf = bio_src->bi_opf;
172174 bio->bi_ioprio = bio_src->bi_ioprio;
173175 bio->bi_write_hint = bio_src->bi_write_hint;
block/blk-crypto.c+1−1
@@ -280,7 +280,7 @@ bool __blk_crypto_bio_prep(struct bio **bio_ptr) · 279 unmodified lines
280280 * Success if device supports the encryption context, or if we succeeded
281281 * in falling back to the crypto API.
282282 */
283− if (blk_ksm_crypto_cfg_supported(bio->bi_disk->queue->ksm,
283+ if (blk_ksm_crypto_cfg_supported(bio->bi_bdev->bd_disk->queue->ksm,
284284 &bc_key->crypto_cfg))
285285 return true;
286286
block/blk-exec.c+5−9
@@ -31,8 +31,7 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t error) · 30 unmodified lines
3131 }
3232
3333 /**
34− * blk_execute_rq_nowait - insert a request into queue for execution
35− * @q: queue to insert the request in
34+ * blk_execute_rq_nowait - insert a request to I/O scheduler for execution
3635 * @bd_disk: matching gendisk
3736 * @rq: request to insert
3837 * @at_head: insert request at head or tail of queue
@@ -45,9 +44,8 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t error) · 6 unmodified lines
4544 * Note:
4645 * This function will invoke @done directly if the queue is dead.
4746 */
48−void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
49− struct request *rq, int at_head,
50− rq_end_io_fn *done)
47+void blk_execute_rq_nowait(struct gendisk *bd_disk, struct request *rq,
48+ int at_head, rq_end_io_fn *done)
5149 {
5250 WARN_ON(irqs_disabled());
5351 WARN_ON(!blk_rq_is_passthrough(rq));
@@ -67,7 +65,6 @@ EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); · 13 unmodified lines
6765
6866 /**
6967 * blk_execute_rq - insert a request into queue for execution
70− * @q: queue to insert the request in
7168 * @bd_disk: matching gendisk
7269 * @rq: request to insert
7370 * @at_head: insert request at head or tail of queue
@@ -76,14 +73,13 @@ EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); · 2 unmodified lines
7673 * Insert a fully prepared request at the back of the I/O scheduler queue
7774 * for execution and wait for completion.
7875 */
79−void blk_execute_rq(struct request_queue *q, struct gendisk *bd_disk,
80− struct request *rq, int at_head)
76+void blk_execute_rq(struct gendisk *bd_disk, struct request *rq, int at_head)
8177 {
8278 DECLARE_COMPLETION_ONSTACK(wait);
8379 unsigned long hang_check;
8480
8581 rq->end_io_data = &wait;
86− blk_execute_rq_nowait(q, bd_disk, rq, at_head, blk_end_sync_rq);
82+ blk_execute_rq_nowait(bd_disk, rq, at_head, blk_end_sync_rq);
8783
8884 /* Prevent hang_check timer from firing at us during very long I/O */
8985 hang_check = sysctl_hung_task_timeout_secs;
block/blk-flush.c+6−11
@@ -432,23 +432,18 @@ void blk_insert_flush(struct request *rq) · 431 unmodified lines
432432 /**
433433 * blkdev_issue_flush - queue a flush
434434 * @bdev: blockdev to issue flush for
435− * @gfp_mask: memory allocation flags (for bio_alloc)
436435 *
437436 * Description:
438437 * Issue a flush for the block device in question.
439438 */
440−int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask)
439+int blkdev_issue_flush(struct block_device *bdev)
441440 {
442− struct bio *bio;
443− int ret = 0;
441+ struct bio bio;
444442
445− bio = bio_alloc(gfp_mask, 0);
446− bio_set_dev(bio, bdev);
447− bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
448−
449− ret = submit_bio_wait(bio);
450− bio_put(bio);
451− return ret;
443+ bio_init(&bio, NULL, 0);
444+ bio_set_dev(&bio, bdev);
445+ bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
446+ return submit_bio_wait(&bio);
452447 }
453448 EXPORT_SYMBOL(blkdev_issue_flush);
454449
block/blk-merge.c+8−9
@@ -298,14 +298,13 @@ split: · 297 unmodified lines
298298 * Split a bio into two bios, chain the two bios, submit the second half and
299299 * store a pointer to the first half in *@bio. If the second bio is still too
300300 * big it will be split by a recursive call to this function. Since this
301− * function may allocate a new bio from @bio->bi_disk->queue->bio_split, it is
302− * the responsibility of the caller to ensure that
303− * @bio->bi_disk->queue->bio_split is only released after processing of the
304− * split bio has finished.
301+ * function may allocate a new bio from q->bio_split, it is the responsibility
302+ * of the caller to ensure that q->bio_split is only released after processing
303+ * of the split bio has finished.
305304 */
306305 void __blk_queue_split(struct bio **bio, unsigned int *nr_segs)
307306 {
308− struct request_queue *q = (*bio)->bi_disk->queue;
307+ struct request_queue *q = (*bio)->bi_bdev->bd_disk->queue;
309308 struct bio *split = NULL;
310309
311310 switch (bio_op(*bio)) {
@@ -358,9 +357,9 @@ void __blk_queue_split(struct bio **bio, unsigned int *nr_segs) · 46 unmodified lines
358357 *
359358 * Split a bio into two bios, chains the two bios, submit the second half and
360359 * store a pointer to the first half in *@bio. Since this function may allocate
361− * a new bio from @bio->bi_disk->queue->bio_split, it is the responsibility of
362− * the caller to ensure that @bio->bi_disk->queue->bio_split is only released
363− * after processing of the split bio has finished.
360+ * a new bio from q->bio_split, it is the responsibility of the caller to ensure
361+ * that q->bio_split is only released after processing of the split bio has
362+ * finished.
364363 */
365364 void blk_queue_split(struct bio **bio)
366365 {
@@ -866,7 +865,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) · 499 unmodified lines
866865 return false;
867866
868867 /* must be same device */
869− if (rq->rq_disk != bio->bi_disk)
868+ if (rq->rq_disk != bio->bi_bdev->bd_disk)
870869 return false;
871870
872871 /* only merge integrity protected bio into ditto rq */
block/blk-mq.c+61−8
@@ -1646,6 +1646,42 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) · 1645 unmodified lines
16461646 }
16471647 EXPORT_SYMBOL(blk_mq_run_hw_queue);
16481648
1649+/*
1650+ * Is the request queue handled by an IO scheduler that does not respect
1651+ * hardware queues when dispatching?
1652+ */
1653+static bool blk_mq_has_sqsched(struct request_queue *q)
1654+{
1655+ struct elevator_queue *e = q->elevator;
1656+
1657+ if (e && e->type->ops.dispatch_request &&
1658+ !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE))
1659+ return true;
1660+ return false;
1661+}
1662+
1663+/*
1664+ * Return prefered queue to dispatch from (if any) for non-mq aware IO
1665+ * scheduler.
1666+ */
1667+static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
1668+{
1669+ struct blk_mq_hw_ctx *hctx;
1670+
1671+ /*
1672+ * If the IO scheduler does not respect hardware queues when
1673+ * dispatching, we just don't bother with multiple HW queues and
1674+ * dispatch from hctx for the current CPU since running multiple queues
1675+ * just causes lock contention inside the scheduler and pointless cache
1676+ * bouncing.
1677+ */
1678+ hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT,
1679+ raw_smp_processor_id());
1680+ if (!blk_mq_hctx_stopped(hctx))
1681+ return hctx;
1682+ return NULL;
1683+}
1684+
16491685 /**
16501686 * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
16511687 * @q: Pointer to the request queue to run.
@@ -1653,14 +1689,23 @@ EXPORT_SYMBOL(blk_mq_run_hw_queue); · 1 unmodified line
16531689 */
16541690 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
16551691 {
1656− struct blk_mq_hw_ctx *hctx;
1692+ struct blk_mq_hw_ctx *hctx, *sq_hctx;
16571693 int i;
16581694
1695+ sq_hctx = NULL;
1696+ if (blk_mq_has_sqsched(q))
1697+ sq_hctx = blk_mq_get_sq_hctx(q);
16591698 queue_for_each_hw_ctx(q, hctx, i) {
16601699 if (blk_mq_hctx_stopped(hctx))
16611700 continue;
1662−
1663− blk_mq_run_hw_queue(hctx, async);
1701+ /*
1702+ * Dispatch from this hctx either if there's no hctx preferred
1703+ * by IO scheduler or if it has requests that bypass the
1704+ * scheduler.
1705+ */
1706+ if (!sq_hctx || sq_hctx == hctx ||
1707+ !list_empty_careful(&hctx->dispatch))
1708+ blk_mq_run_hw_queue(hctx, async);
16641709 }
16651710 }
16661711 EXPORT_SYMBOL(blk_mq_run_hw_queues);
@@ -1672,14 +1717,23 @@ EXPORT_SYMBOL(blk_mq_run_hw_queues); · 5 unmodified lines
16721717 */
16731718 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
16741719 {
1675− struct blk_mq_hw_ctx *hctx;
1720+ struct blk_mq_hw_ctx *hctx, *sq_hctx;
16761721 int i;
16771722
1723+ sq_hctx = NULL;
1724+ if (blk_mq_has_sqsched(q))
1725+ sq_hctx = blk_mq_get_sq_hctx(q);
16781726 queue_for_each_hw_ctx(q, hctx, i) {
16791727 if (blk_mq_hctx_stopped(hctx))
16801728 continue;
1681−
1682− blk_mq_delay_run_hw_queue(hctx, msecs);
1729+ /*
1730+ * Dispatch from this hctx either if there's no hctx preferred
1731+ * by IO scheduler or if it has requests that bypass the
1732+ * scheduler.
1733+ */
1734+ if (!sq_hctx || sq_hctx == hctx ||
1735+ !list_empty_careful(&hctx->dispatch))
1736+ blk_mq_delay_run_hw_queue(hctx, msecs);
16831737 }
16841738 }
16851739 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
@@ -2128,7 +2182,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) · 442 unmodified lines
21282182 */
21292183 blk_qc_t blk_mq_submit_bio(struct bio *bio)
21302184 {
2131− struct request_queue *q = bio->bi_disk->queue;
2185+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
21322186 const int is_sync = op_is_sync(bio->bi_opf);
21332187 const int is_flush_fua = op_is_flush(bio->bi_opf);
21342188 struct blk_mq_alloc_data data = {
@@ -2653,7 +2707,6 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, · 518 unmodified lines
26532707 goto free_hctx;
26542708
26552709 atomic_set(&hctx->nr_active, 0);
2656− atomic_set(&hctx->elevator_queued, 0);
26572710 if (node == NUMA_NO_NODE)
26582711 node = set->numa_node;
26592712 hctx->numa_node = node;
block/blk-settings.c+39−2
@@ -60,6 +60,7 @@ void blk_set_default_limits(struct queue_limits *lim) · 59 unmodified lines
6060 lim->io_opt = 0;
6161 lim->misaligned = 0;
6262 lim->zoned = BLK_ZONED_NONE;
63+ lim->zone_write_granularity = 0;
6364 }
6465 EXPORT_SYMBOL(blk_set_default_limits);
6566
@@ -366,6 +367,28 @@ void blk_queue_physical_block_size(struct request_queue *q, unsigned int size) · 300 unmodified lines
366367 }
367368 EXPORT_SYMBOL(blk_queue_physical_block_size);
368369
370+/**
371+ * blk_queue_zone_write_granularity - set zone write granularity for the queue
372+ * @q: the request queue for the zoned device
373+ * @size: the zone write granularity size, in bytes
374+ *
375+ * Description:
376+ * This should be set to the lowest possible size allowing to write in
377+ * sequential zones of a zoned block device.
378+ */
379+void blk_queue_zone_write_granularity(struct request_queue *q,
380+ unsigned int size)
381+{
382+ if (WARN_ON_ONCE(!blk_queue_is_zoned(q)))
383+ return;
384+
385+ q->limits.zone_write_granularity = size;
386+
387+ if (q->limits.zone_write_granularity < q->limits.logical_block_size)
388+ q->limits.zone_write_granularity = q->limits.logical_block_size;
389+}
390+EXPORT_SYMBOL_GPL(blk_queue_zone_write_granularity);
391+
369392 /**
370393 * blk_queue_alignment_offset - set physical block alignment offset
371394 * @q: the request queue for the device
@@ -631,6 +654,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, · 259 unmodified lines
631654 t->discard_granularity;
632655 }
633656
657+ t->zone_write_granularity = max(t->zone_write_granularity,
658+ b->zone_write_granularity);
634659 t->zoned = max(t->zoned, b->zoned);
635660 return ret;
636661 }
@@ -847,6 +872,8 @@ EXPORT_SYMBOL_GPL(blk_queue_can_use_dma_map_merging); · 210 unmodified lines
847872 */
848873 void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model)
849874 {
875+ struct request_queue *q = disk->queue;
876+
850877 switch (model) {
851878 case BLK_ZONED_HM:
852879 /*
@@ -865,7 +892,7 @@ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) · 12 unmodified lines
865892 * we do nothing special as far as the block layer is concerned.
866893 */
867894 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) ||
868− disk_has_partitions(disk))
895+ !xa_empty(&disk->part_tbl))
869896 model = BLK_ZONED_NONE;
870897 break;
871898 case BLK_ZONED_NONE:
@@ -875,7 +902,17 @@ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) · 3 unmodified lines
875902 break;
876903 }
877904
878− disk->queue->limits.zoned = model;
905+ q->limits.zoned = model;
906+ if (model != BLK_ZONED_NONE) {
907+ /*
908+ * Set the zone write granularity to the device logical block
909+ * size by default. The driver can change this value if needed.
910+ */
911+ blk_queue_zone_write_granularity(q,
912+ queue_logical_block_size(q));
913+ } else {
914+ blk_queue_clear_zone_settings(q);
915+ }
879916 }
880917 EXPORT_SYMBOL_GPL(blk_queue_set_zoned);
881918
block/blk-sysfs.c+8−0
@@ -219,6 +219,12 @@ static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page) · 218 unmodified lines
219219 (unsigned long long)q->limits.max_write_zeroes_sectors << 9);
220220 }
221221
222+static ssize_t queue_zone_write_granularity_show(struct request_queue *q,
223+ char *page)
224+{
225+ return queue_var_show(queue_zone_write_granularity(q), page);
226+}
227+
222228 static ssize_t queue_zone_append_max_show(struct request_queue *q, char *page)
223229 {
224230 unsigned long long max_sectors = q->limits.max_zone_append_sectors;
@@ -585,6 +591,7 @@ QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data"); · 360 unmodified lines
585591 QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
586592 QUEUE_RO_ENTRY(queue_write_zeroes_max, "write_zeroes_max_bytes");
587593 QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");
594+QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");
588595
589596 QUEUE_RO_ENTRY(queue_zoned, "zoned");
590597 QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
@@ -639,6 +646,7 @@ static struct attribute *queue_attrs[] = { · 48 unmodified lines
639646 &queue_write_same_max_entry.attr,
640647 &queue_write_zeroes_max_entry.attr,
641648 &queue_zone_append_max_entry.attr,
649+ &queue_zone_write_granularity_entry.attr,
642650 &queue_nonrot_entry.attr,
643651 &queue_zoned_entry.attr,
644652 &queue_nr_zones_entry.attr,
block/blk-throttle.c+1−1
@@ -2178,7 +2178,7 @@ static inline void throtl_update_latency_buckets(struct throtl_data *td) · 2177 unmodified lines
21782178
21792179 bool blk_throtl_bio(struct bio *bio)
21802180 {
2181− struct request_queue *q = bio->bi_disk->queue;
2181+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
21822182 struct blkcg_gq *blkg = bio->bi_blkg;
21832183 struct throtl_qnode *qn = NULL;
21842184 struct throtl_grp *tg = blkg_to_tg(blkg);
block/blk-wbt.c+2−2
@@ -518,7 +518,7 @@ static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, · 517 unmodified lines
518518 rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb);
519519 }
520520
521−static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
521+static inline bool wbt_should_throttle(struct bio *bio)
522522 {
523523 switch (bio_op(bio)) {
524524 case REQ_OP_WRITE:
@@ -545,7 +545,7 @@ static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio) · 20 unmodified lines
545545
546546 if (bio_op(bio) == REQ_OP_READ) {
547547 flags = WBT_READ;
548− } else if (wbt_should_throttle(rwb, bio)) {
548+ } else if (wbt_should_throttle(bio)) {
549549 if (current_is_kswapd())
550550 flags |= WBT_KSWAPD;
551551 if (bio_op(bio) == REQ_OP_DISCARD)
block/blk-zoned.c+17−0
@@ -549,3 +549,20 @@ int blk_revalidate_disk_zones(struct gendisk *disk, · 548 unmodified lines
549549 return ret;
550550 }
551551 EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones);
552+
553+void blk_queue_clear_zone_settings(struct request_queue *q)
554+{
555+ blk_mq_freeze_queue(q);
556+
557+ blk_queue_free_zone_bitmaps(q);
558+ blk_queue_flag_clear(QUEUE_FLAG_ZONE_RESETALL, q);
559+ q->required_elevator_features &= ~ELEVATOR_F_ZBD_SEQ_WRITE;
560+ q->nr_zones = 0;
561+ q->max_open_zones = 0;
562+ q->max_active_zones = 0;
563+ q->limits.chunk_sectors = 0;
564+ q->limits.zone_write_granularity = 0;
565+ q->limits.max_zone_append_sectors = 0;
566+
567+ blk_mq_unfreeze_queue(q);
568+}
block/blk.h+7−5
@@ -55,6 +55,11 @@ void blk_free_flush_queue(struct blk_flush_queue *q); · 54 unmodified lines
5555
5656 void blk_freeze_queue(struct request_queue *q);
5757
58+#define BIO_INLINE_VECS 4
59+struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
60+ gfp_t gfp_mask);
61+void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);
62+
5863 static inline bool biovec_phys_mergeable(struct request_queue *q,
5964 struct bio_vec *vec1, struct bio_vec *vec2)
6065 {
@@ -202,8 +207,6 @@ static inline void elevator_exit(struct request_queue *q, · 141 unmodified lines
202207 __elevator_exit(q, e);
203208 }
204209
205−struct block_device *__disk_get_part(struct gendisk *disk, int partno);
206−
207210 ssize_t part_size_show(struct device *dev, struct device_attribute *attr,
208211 char *buf);
209212 ssize_t part_stat_show(struct device *dev, struct device_attribute *attr,
@@ -331,12 +334,12 @@ struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp); · 121 unmodified lines
331334
332335 #ifdef CONFIG_BLK_DEV_ZONED
333336 void blk_queue_free_zone_bitmaps(struct request_queue *q);
337+void blk_queue_clear_zone_settings(struct request_queue *q);
334338 #else
335339 static inline void blk_queue_free_zone_bitmaps(struct request_queue *q) {}
340+static inline void blk_queue_clear_zone_settings(struct request_queue *q) {}
336341 #endif
337342
338−struct block_device *disk_map_sector_rcu(struct gendisk *disk, sector_t sector);
339−
340343 int blk_alloc_devt(struct block_device *part, dev_t *devt);
341344 void blk_free_devt(dev_t devt);
342345 char *disk_name(struct gendisk *hd, int partno, char *buf);
@@ -349,7 +352,6 @@ int bdev_add_partition(struct block_device *bdev, int partno, · 6 unmodified lines
349352 int bdev_del_partition(struct block_device *bdev, int partno);
350353 int bdev_resize_partition(struct block_device *bdev, int partno,
351354 sector_t start, sector_t length);
352−int disk_expand_part_tbl(struct gendisk *disk, int target);
353355
354356 int bio_add_hw_page(struct request_queue *q, struct bio *bio,
355357 struct page *page, unsigned int len, unsigned int offset,
block/bounce.c+3−1
@@ -246,7 +246,9 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, · 245 unmodified lines
246246 bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
247247 if (!bio)
248248 return NULL;
249− bio->bi_disk = bio_src->bi_disk;
249+ bio->bi_bdev = bio_src->bi_bdev;
250+ if (bio_flagged(bio_src, BIO_REMAPPED))
251+ bio_set_flag(bio, BIO_REMAPPED);
250252 bio->bi_opf = bio_src->bi_opf;
251253 bio->bi_ioprio = bio_src->bi_ioprio;
252254 bio->bi_write_hint = bio_src->bi_write_hint;
block/bsg.c+4−2
@@ -157,8 +157,10 @@ static int bsg_sg_io(struct request_queue *q, fmode_t mode, void __user *uarg) · 156 unmodified lines
157157 return PTR_ERR(rq);
158158
159159 ret = q->bsg_dev.ops->fill_hdr(rq, &hdr, mode);
160− if (ret)
160+ if (ret) {
161+ blk_put_request(rq);
161162 return ret;
163+ }
162164
163165 rq->timeout = msecs_to_jiffies(hdr.timeout);
164166 if (!rq->timeout)
@@ -181,7 +183,7 @@ static int bsg_sg_io(struct request_queue *q, fmode_t mode, void __user *uarg) · 16 unmodified lines
181183
182184 bio = rq->bio;
183185
184− blk_execute_rq(q, NULL, rq, !(hdr.flags & BSG_FLAG_Q_AT_TAIL));
186+ blk_execute_rq(NULL, rq, !(hdr.flags & BSG_FLAG_Q_AT_TAIL));
185187 ret = rq->q->bsg_dev.ops->complete_rq(rq, &hdr);
186188 blk_rq_unmap_user(bio);
187189
block/genhd.c+49−257
@@ -162,15 +162,6 @@ static void part_in_flight_rw(struct block_device *part, · 161 unmodified lines
162162 inflight[1] = 0;
163163 }
164164
165−struct block_device *__disk_get_part(struct gendisk *disk, int partno)
166−{
167− struct disk_part_tbl *ptbl = rcu_dereference(disk->part_tbl);
168−
169− if (unlikely(partno < 0 || partno >= ptbl->len))
170− return NULL;
171− return rcu_dereference(ptbl->part[partno]);
172−}
173−
174165 /**
175166 * disk_part_iter_init - initialize partition iterator
176167 * @piter: iterator to initialize
@@ -185,26 +176,14 @@ struct block_device *__disk_get_part(struct gendisk *disk, int partno) · 8 unmodified lines
185176 void disk_part_iter_init(struct disk_part_iter *piter, struct gendisk *disk,
186177 unsigned int flags)
187178 {
188− struct disk_part_tbl *ptbl;
189−
190− rcu_read_lock();
191− ptbl = rcu_dereference(disk->part_tbl);
192−
193179 piter->disk = disk;
194180 piter->part = NULL;
195−
196− if (flags & DISK_PITER_REVERSE)
197− piter->idx = ptbl->len - 1;
198− else if (flags & (DISK_PITER_INCL_PART0 | DISK_PITER_INCL_EMPTY_PART0))
181+ if (flags & (DISK_PITER_INCL_PART0 | DISK_PITER_INCL_EMPTY_PART0))
199182 piter->idx = 0;
200183 else
201184 piter->idx = 1;
202−
203185 piter->flags = flags;
204−
205− rcu_read_unlock();
206186 }
207−EXPORT_SYMBOL_GPL(disk_part_iter_init);
208187
209188 /**
210189 * disk_part_iter_next - proceed iterator to the next partition and return it
@@ -217,57 +196,30 @@ EXPORT_SYMBOL_GPL(disk_part_iter_init); · 6 unmodified lines
217196 */
218197 struct block_device *disk_part_iter_next(struct disk_part_iter *piter)
219198 {
220− struct disk_part_tbl *ptbl;
221− int inc, end;
199+ struct block_device *part;
200+ unsigned long idx;
222201
223202 /* put the last partition */
224203 disk_part_iter_exit(piter);
225204
226− /* get part_tbl */
227205 rcu_read_lock();
228− ptbl = rcu_dereference(piter->disk->part_tbl);
229−
230− /* determine iteration parameters */
231− if (piter->flags & DISK_PITER_REVERSE) {
232− inc = -1;
233− if (piter->flags & (DISK_PITER_INCL_PART0 |
234− DISK_PITER_INCL_EMPTY_PART0))
235− end = -1;
236− else
237− end = 0;
238− } else {
239− inc = 1;
240− end = ptbl->len;
241− }
242−
243− /* iterate to the next partition */
244− for (; piter->idx != end; piter->idx += inc) {
245− struct block_device *part;
246−
247− part = rcu_dereference(ptbl->part[piter->idx]);
248− if (!part)
249− continue;
250− piter->part = bdgrab(part);
251− if (!piter->part)
252− continue;
206+ xa_for_each_start(&piter->disk->part_tbl, idx, part, piter->idx) {
253207 if (!bdev_nr_sectors(part) &&
254208 !(piter->flags & DISK_PITER_INCL_EMPTY) &&
255209 !(piter->flags & DISK_PITER_INCL_EMPTY_PART0 &&
256− piter->idx == 0)) {
257− bdput(piter->part);
258− piter->part = NULL;
210+ piter->idx == 0))
259211 continue;
260− }
261212
262− piter->idx += inc;
213+ piter->part = bdgrab(part);
214+ if (!piter->part)
215+ continue;
216+ piter->idx = idx + 1;
263217 break;
264218 }
265−
266219 rcu_read_unlock();
267220
268221 return piter->part;
269222 }
270−EXPORT_SYMBOL_GPL(disk_part_iter_next);
271223
272224 /**
273225 * disk_part_iter_exit - finish up partition iteration
@@ -284,91 +236,6 @@ void disk_part_iter_exit(struct disk_part_iter *piter) · 10 unmodified lines
284236 bdput(piter->part);
285237 piter->part = NULL;
286238 }
287−EXPORT_SYMBOL_GPL(disk_part_iter_exit);
288−
289−static inline int sector_in_part(struct block_device *part, sector_t sector)
290−{
291− return part->bd_start_sect <= sector &&
292− sector < part->bd_start_sect + bdev_nr_sectors(part);
293−}
294−
295−/**
296− * disk_map_sector_rcu - map sector to partition
297− * @disk: gendisk of interest
298− * @sector: sector to map
299− *
300− * Find out which partition @sector maps to on @disk. This is
301− * primarily used for stats accounting.
302− *
303− * CONTEXT:
304− * RCU read locked.
305− *
306− * RETURNS:
307− * Found partition on success, part0 is returned if no partition matches
308− * or the matched partition is being deleted.
309− */
310−struct block_device *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
311−{
312− struct disk_part_tbl *ptbl;
313− struct block_device *part;
314− int i;
315−
316− rcu_read_lock();
317− ptbl = rcu_dereference(disk->part_tbl);
318−
319− part = rcu_dereference(ptbl->last_lookup);
320− if (part && sector_in_part(part, sector))
321− goto out_unlock;
322−
323− for (i = 1; i < ptbl->len; i++) {
324− part = rcu_dereference(ptbl->part[i]);
325− if (part && sector_in_part(part, sector)) {
326− rcu_assign_pointer(ptbl->last_lookup, part);
327− goto out_unlock;
328− }
329− }
330−
331− part = disk->part0;
332−out_unlock:
333− rcu_read_unlock();
334− return part;
335−}
336−
337−/**
338− * disk_has_partitions
339− * @disk: gendisk of interest
340− *
341− * Walk through the partition table and check if valid partition exists.
342− *
343− * CONTEXT:
344− * Don't care.
345− *
346− * RETURNS:
347− * True if the gendisk has at least one valid non-zero size partition.
348− * Otherwise false.
349− */
350−bool disk_has_partitions(struct gendisk *disk)
351−{
352− struct disk_part_tbl *ptbl;
353− int i;
354− bool ret = false;
355−
356− rcu_read_lock();
357− ptbl = rcu_dereference(disk->part_tbl);
358−
359− /* Iterate partitions skipping the whole device at index 0 */
360− for (i = 1; i < ptbl->len; i++) {
361− if (rcu_dereference(ptbl->part[i])) {
362− ret = true;
363− break;
364− }
365− }
366−
367− rcu_read_unlock();
368−
369− return ret;
370−}
371−EXPORT_SYMBOL_GPL(disk_has_partitions);
372239
373240 /*
374241 * Can be deleted altogether. Later.
@@ -604,6 +471,18 @@ static char *bdevt_str(dev_t devt, char *buf) · 229 unmodified lines
604471 return buf;
605472 }
606473
474+void disk_uevent(struct gendisk *disk, enum kobject_action action)
475+{
476+ struct disk_part_iter piter;
477+ struct block_device *part;
478+
479+ disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
480+ while ((part = disk_part_iter_next(&piter)))
481+ kobject_uevent(bdev_kobj(part), action);
482+ disk_part_iter_exit(&piter);
483+}
484+EXPORT_SYMBOL_GPL(disk_uevent);
485+
607486 static void disk_scan_partitions(struct gendisk *disk)
608487 {
609488 struct block_device *bdev;
@@ -621,8 +500,6 @@ static void register_disk(struct device *parent, struct gendisk *disk, · 11 unmodified lines
621500 const struct attribute_group **groups)
622501 {
623502 struct device *ddev = disk_to_dev(disk);
624− struct disk_part_iter piter;
625− struct block_device *part;
626503 int err;
627504
628505 ddev->parent = parent;
@@ -665,15 +542,9 @@ static void register_disk(struct device *parent, struct gendisk *disk, · 36 unmodified lines
665542
666543 disk_scan_partitions(disk);
667544
668− /* announce disk after possible partitions are created */
545+ /* announce the disk and partitions after all partitions are created */
669546 dev_set_uevent_suppress(ddev, 0);
670− kobject_uevent(&ddev->kobj, KOBJ_ADD);
671−
672− /* announce possible partitions */
673− disk_part_iter_init(&piter, disk, 0);
674− while ((part = disk_part_iter_next(&piter)))
675− kobject_uevent(bdev_kobj(part), KOBJ_ADD);
676− disk_part_iter_exit(&piter);
547+ disk_uevent(disk, KOBJ_ADD);
677548
678549 if (disk->queue->backing_dev_info->dev) {
679550 err = sysfs_create_link(&ddev->kobj,
@@ -829,8 +700,7 @@ void del_gendisk(struct gendisk *disk) · 149 unmodified lines
829700 down_write(&bdev_lookup_sem);
830701
831702 /* invalidate stuff */
832− disk_part_iter_init(&piter, disk,
833− DISK_PITER_INCL_EMPTY | DISK_PITER_REVERSE);
703+ disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY);
834704 while ((part = disk_part_iter_next(&piter))) {
835705 invalidate_partition(part);
836706 delete_partition(part);
@@ -929,7 +799,7 @@ struct block_device *bdget_disk(struct gendisk *disk, int partno) · 92 unmodified lines
929799 struct block_device *bdev = NULL;
930800
931801 rcu_read_lock();
932− bdev = __disk_get_part(disk, partno);
802+ bdev = xa_load(&disk->part_tbl, partno);
933803 if (bdev && !bdgrab(bdev))
934804 bdev = NULL;
935805 rcu_read_unlock();
@@ -1319,83 +1189,6 @@ static const struct attribute_group *disk_attr_groups[] = { · 383 unmodified lines
13191189 NULL
13201190 };
13211191
1322−/**
1323− * disk_replace_part_tbl - replace disk->part_tbl in RCU-safe way
1324− * @disk: disk to replace part_tbl for
1325− * @new_ptbl: new part_tbl to install
1326− *
1327− * Replace disk->part_tbl with @new_ptbl in RCU-safe way. The
1328− * original ptbl is freed using RCU callback.
1329− *
1330− * LOCKING:
1331− * Matching bd_mutex locked or the caller is the only user of @disk.
1332− */
1333−static void disk_replace_part_tbl(struct gendisk *disk,
1334− struct disk_part_tbl *new_ptbl)
1335−{
1336− struct disk_part_tbl *old_ptbl =
1337− rcu_dereference_protected(disk->part_tbl, 1);
1338−
1339− rcu_assign_pointer(disk->part_tbl, new_ptbl);
1340−
1341− if (old_ptbl) {
1342− rcu_assign_pointer(old_ptbl->last_lookup, NULL);
1343− kfree_rcu(old_ptbl, rcu_head);
1344− }
1345−}
1346−
1347−/**
1348− * disk_expand_part_tbl - expand disk->part_tbl
1349− * @disk: disk to expand part_tbl for
1350− * @partno: expand such that this partno can fit in
1351− *
1352− * Expand disk->part_tbl such that @partno can fit in. disk->part_tbl
1353− * uses RCU to allow unlocked dereferencing for stats and other stuff.
1354− *
1355− * LOCKING:
1356− * Matching bd_mutex locked or the caller is the only user of @disk.
1357− * Might sleep.
1358− *
1359− * RETURNS:
1360− * 0 on success, -errno on failure.
1361− */
1362−int disk_expand_part_tbl(struct gendisk *disk, int partno)
1363−{
1364− struct disk_part_tbl *old_ptbl =
1365− rcu_dereference_protected(disk->part_tbl, 1);
1366− struct disk_part_tbl *new_ptbl;
1367− int len = old_ptbl ? old_ptbl->len : 0;
1368− int i, target;
1369−
1370− /*
1371− * check for int overflow, since we can get here from blkpg_ioctl()
1372− * with a user passed 'partno'.
1373− */
1374− target = partno + 1;
1375− if (target < 0)
1376− return -EINVAL;
1377−
1378− /* disk_max_parts() is zero during initialization, ignore if so */
1379− if (disk_max_parts(disk) && target > disk_max_parts(disk))
1380− return -EINVAL;
1381−
1382− if (target <= len)
1383− return 0;
1384−
1385− new_ptbl = kzalloc_node(struct_size(new_ptbl, part, target), GFP_KERNEL,
1386− disk->node_id);
1387− if (!new_ptbl)
1388− return -ENOMEM;
1389−
1390− new_ptbl->len = target;
1391−
1392− for (i = 0; i < len; i++)
1393− rcu_assign_pointer(new_ptbl->part[i], old_ptbl->part[i]);
1394−
1395− disk_replace_part_tbl(disk, new_ptbl);
1396− return 0;
1397−}
1398−
13991192 /**
14001193 * disk_release - releases all allocated resources of the gendisk
14011194 * @dev: the device representing this disk
@@ -1419,7 +1212,7 @@ static void disk_release(struct device *dev) · 17 unmodified lines
14191212 blk_free_devt(dev->devt);
14201213 disk_release_events(disk);
14211214 kfree(disk->random);
1422− disk_replace_part_tbl(disk, NULL);
1215+ xa_destroy(&disk->part_tbl);
14231216 bdput(disk->part0);
14241217 if (disk->queue)
14251218 blk_put_queue(disk->queue);
@@ -1572,7 +1365,6 @@ dev_t blk_lookup_devt(const char *name, int partno) · 146 unmodified lines
15721365 struct gendisk *__alloc_disk_node(int minors, int node_id)
15731366 {
15741367 struct gendisk *disk;
1575− struct disk_part_tbl *ptbl;
15761368
15771369 if (minors > DISK_MAX_PARTS) {
15781370 printk(KERN_ERR
@@ -1590,11 +1382,9 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) · 11 unmodified lines
15901382 goto out_free_disk;
15911383
15921384 disk->node_id = node_id;
1593− if (disk_expand_part_tbl(disk, 0))
1594− goto out_bdput;
1595−
1596− ptbl = rcu_dereference_protected(disk->part_tbl, 1);
1597− rcu_assign_pointer(ptbl->part[0], disk->part0);
1385+ xa_init(&disk->part_tbl);
1386+ if (xa_insert(&disk->part_tbl, 0, disk->part0, GFP_KERNEL))
1387+ goto out_destroy_part_tbl;
15981388
15991389 disk->minors = minors;
16001390 rand_initialize_disk(disk);
@@ -1603,7 +1393,8 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) · 2 unmodified lines
16031393 device_initialize(disk_to_dev(disk));
16041394 return disk;
16051395
1606−out_bdput:
1396+out_destroy_part_tbl:
1397+ xa_destroy(&disk->part_tbl);
16071398 bdput(disk->part0);
16081399 out_free_disk:
16091400 kfree(disk);
@@ -1638,31 +1429,32 @@ static void set_disk_ro_uevent(struct gendisk *gd, int ro) · 28 unmodified lines
16381429 kobject_uevent_env(&disk_to_dev(gd)->kobj, KOBJ_CHANGE, envp);
16391430 }
16401431
1641−void set_disk_ro(struct gendisk *disk, int flag)
1432+/**
1433+ * set_disk_ro - set a gendisk read-only
1434+ * @disk: gendisk to operate on
1435+ * @read_only: %true to set the disk read-only, %false set the disk read/write
1436+ *
1437+ * This function is used to indicate whether a given disk device should have its
1438+ * read-only flag set. set_disk_ro() is typically used by device drivers to
1439+ * indicate whether the underlying physical device is write-protected.
1440+ */
1441+void set_disk_ro(struct gendisk *disk, bool read_only)
16421442 {
1643− struct disk_part_iter piter;
1644− struct block_device *part;
1645−
1646− if (disk->part0->bd_read_only != flag) {
1647− set_disk_ro_uevent(disk, flag);
1648− disk->part0->bd_read_only = flag;
1443+ if (read_only) {
1444+ if (test_and_set_bit(GD_READ_ONLY, &disk->state))
1445+ return;
1446+ } else {
1447+ if (!test_and_clear_bit(GD_READ_ONLY, &disk->state))
1448+ return;
16491449 }
1650−
1651− disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY);
1652− while ((part = disk_part_iter_next(&piter)))
1653− part->bd_read_only = flag;
1654− disk_part_iter_exit(&piter);
1450+ set_disk_ro_uevent(disk, read_only);
16551451 }
1656−
16571452 EXPORT_SYMBOL(set_disk_ro);
16581453
16591454 int bdev_read_only(struct block_device *bdev)
16601455 {
1661− if (!bdev)
1662− return 0;
1663− return bdev->bd_read_only;
1456+ return bdev->bd_read_only || get_disk_ro(bdev->bd_disk);
16641457 }
1665−
16661458 EXPORT_SYMBOL(bdev_read_only);
16671459
16681460 /*
block/kyber-iosched.c+1−0
@@ -1029,6 +1029,7 @@ static struct elevator_type kyber_sched = { · 1028 unmodified lines
10291029 #endif
10301030 .elevator_attrs = kyber_sched_attrs,
10311031 .elevator_name = "kyber",
1032+ .elevator_features = ELEVATOR_F_MQ_AWARE,
10321033 .elevator_owner = THIS_MODULE,
10331034 };
10341035
block/mq-deadline.c+0−6
@@ -386,8 +386,6 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) · 385 unmodified lines
386386 spin_lock(&dd->lock);
387387 rq = __dd_dispatch_request(dd);
388388 spin_unlock(&dd->lock);
389− if (rq)
390− atomic_dec(&rq->mq_hctx->elevator_queued);
391389
392390 return rq;
393391 }
@@ -535,7 +533,6 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, · 141 unmodified lines
535533 rq = list_first_entry(list, struct request, queuelist);
536534 list_del_init(&rq->queuelist);
537535 dd_insert_request(hctx, rq, at_head);
538− atomic_inc(&hctx->elevator_queued);
539536 }
540537 spin_unlock(&dd->lock);
541538 }
@@ -582,9 +579,6 @@ static bool dd_has_work(struct blk_mq_hw_ctx *hctx) · 40 unmodified lines
582579 {
583580 struct deadline_data *dd = hctx->queue->elevator->elevator_data;
584581
585− if (!atomic_read(&hctx->elevator_queued))
586− return false;
587−
588582 return !list_empty_careful(&dd->dispatch) ||
589583 !list_empty_careful(&dd->fifo_list[0]) ||
590584 !list_empty_careful(&dd->fifo_list[1]);
block/partitions/core.c+8−28
@@ -197,7 +197,7 @@ static ssize_t part_start_show(struct device *dev, · 196 unmodified lines
197197 static ssize_t part_ro_show(struct device *dev,
198198 struct device_attribute *attr, char *buf)
199199 {
200− return sprintf(buf, "%d\n", dev_to_bdev(dev)->bd_read_only);
200+ return sprintf(buf, "%d\n", bdev_read_only(dev_to_bdev(dev)));
201201 }
202202
203203 static ssize_t part_alignment_offset_show(struct device *dev,
@@ -289,13 +289,7 @@ struct device_type part_type = { · 85 unmodified lines
289289 */
290290 void delete_partition(struct block_device *part)
291291 {
292− struct gendisk *disk = part->bd_disk;
293− struct disk_part_tbl *ptbl =
294− rcu_dereference_protected(disk->part_tbl, 1);
295−
296− rcu_assign_pointer(ptbl->part[part->bd_partno], NULL);
297− rcu_assign_pointer(ptbl->last_lookup, NULL);
298−
292+ xa_erase(&part->bd_disk->part_tbl, part->bd_partno);
299293 kobject_put(part->bd_holder_dir);
300294 device_del(&part->bd_device);
301295
@@ -327,7 +321,6 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, · 25 unmodified lines
327321 struct device *ddev = disk_to_dev(disk);
328322 struct device *pdev;
329323 struct block_device *bdev;
330− struct disk_part_tbl *ptbl;
331324 const char *dname;
332325 int err;
333326
@@ -343,18 +336,13 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, · 9 unmodified lines
343336 case BLK_ZONED_HA:
344337 pr_info("%s: disabling host aware zoned block device support due to partitions\n",
345338 disk->disk_name);
346− disk->queue->limits.zoned = BLK_ZONED_NONE;
339+ blk_queue_set_zoned(disk, BLK_ZONED_NONE);
347340 break;
348341 case BLK_ZONED_NONE:
349342 break;
350343 }
351344
352− err = disk_expand_part_tbl(disk, partno);
353− if (err)
354− return ERR_PTR(err);
355− ptbl = rcu_dereference_protected(disk->part_tbl, 1);
356−
357− if (ptbl->part[partno])
345+ if (xa_load(&disk->part_tbl, partno))
358346 return ERR_PTR(-EBUSY);
359347
360348 bdev = bdev_alloc(disk, partno);
@@ -363,7 +351,6 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, · 2 unmodified lines
363351
364352 bdev->bd_start_sect = start;
365353 bdev_set_nr_sectors(bdev, len);
366− bdev->bd_read_only = get_disk_ro(disk);
367354
368355 if (info) {
369356 err = -ENOMEM;
@@ -408,8 +395,10 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, · 38 unmodified lines
408395 }
409396
410397 /* everything is up and running, commence */
398+ err = xa_insert(&disk->part_tbl, partno, bdev, GFP_KERNEL);
399+ if (err)
400+ goto out_del;
411401 bdev_add(bdev, devt);
412− rcu_assign_pointer(ptbl->part[partno], bdev);
413402
414403 /* suppress uevent if the disk suppresses it */
415404 if (!dev_get_uevent_suppress(ddev))
@@ -615,7 +604,7 @@ static bool blk_add_partition(struct gendisk *disk, struct block_device *bdev, · 199 unmodified lines
615604 int blk_add_partitions(struct gendisk *disk, struct block_device *bdev)
616605 {
617606 struct parsed_partitions *state;
618− int ret = -EAGAIN, p, highest;
607+ int ret = -EAGAIN, p;
619608
620609 if (!disk_part_scan_enabled(disk))
621610 return 0;
@@ -663,15 +652,6 @@ int blk_add_partitions(struct gendisk *disk, struct block_device *bdev) · 41 unmodified lines
663652 /* tell userspace that the media / partition table may have changed */
664653 kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE);
665654
666− /*
667− * Detect the highest partition number and preallocate disk->part_tbl.
668− * This is an optimization and not strictly necessary.
669− */
670− for (p = 1, highest = 0; p < state->limit; p++)
671− if (state->parts[p].size)
672− highest = p;
673− disk_expand_part_tbl(disk, highest);
674−
675655 for (p = 1; p < state->limit; p++)
676656 if (!blk_add_partition(disk, bdev, state, p))
677657 goto out_free_state;
block/scsi_ioctl.c+3−3
@@ -357,7 +357,7 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk, · 356 unmodified lines
357357 * (if he doesn't check that is his problem).
358358 * N.B. a non-zero SCSI status is _not_ necessarily an error.
359359 */
360− blk_execute_rq(q, bd_disk, rq, at_head);
360+ blk_execute_rq(bd_disk, rq, at_head);
361361
362362 hdr->duration = jiffies_to_msecs(jiffies - start_time);
363363
@@ -493,7 +493,7 @@ int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode, · 129 unmodified lines
493493 goto error;
494494 }
495495
496− blk_execute_rq(q, disk, rq, 0);
496+ blk_execute_rq(disk, rq, 0);
497497
498498 err = req->result & 0xff; /* only 8 bit SCSI status */
499499 if (err) {
@@ -532,7 +532,7 @@ static int __blk_send_generic(struct request_queue *q, struct gendisk *bd_disk, · 32 unmodified lines
532532 scsi_req(rq)->cmd[0] = cmd;
533533 scsi_req(rq)->cmd[4] = data;
534534 scsi_req(rq)->cmd_len = 6;
535− blk_execute_rq(q, bd_disk, rq, 0);
535+ blk_execute_rq(bd_disk, rq, 0);
536536 err = scsi_req(rq)->result ? -EIO : 0;
537537 blk_put_request(rq);
538538
drivers/block/brd.c+2−6
@@ -284,15 +284,11 @@ out: · 283 unmodified lines
284284
285285 static blk_qc_t brd_submit_bio(struct bio *bio)
286286 {
287− struct brd_device *brd = bio->bi_disk->private_data;
287+ struct brd_device *brd = bio->bi_bdev->bd_disk->private_data;
288+ sector_t sector = bio->bi_iter.bi_sector;
288289 struct bio_vec bvec;
289− sector_t sector;
290290 struct bvec_iter iter;
291291
292− sector = bio->bi_iter.bi_sector;
293− if (bio_end_sector(bio) > get_capacity(bio->bi_disk))
294− goto io_error;
295−
296292 bio_for_each_segment(bvec, bio, iter) {
297293 unsigned int len = bvec.bv_len;
298294 int err;
drivers/block/drbd/drbd_actlog.c+1−1
@@ -138,7 +138,7 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, · 137 unmodified lines
138138 op_flags |= REQ_FUA | REQ_PREFLUSH;
139139 op_flags |= REQ_SYNC;
140140
141− bio = bio_alloc_drbd(GFP_NOIO);
141+ bio = bio_alloc_bioset(GFP_NOIO, 1, &drbd_md_io_bio_set);
142142 bio_set_dev(bio, bdev->md_bdev);
143143 bio->bi_iter.bi_sector = sector;
144144 err = -EIO;
drivers/block/drbd/drbd_bitmap.c+1−1
@@ -976,7 +976,7 @@ static void drbd_bm_endio(struct bio *bio) · 975 unmodified lines
976976
977977 static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_hold(local)
978978 {
979− struct bio *bio = bio_alloc_drbd(GFP_NOIO);
979+ struct bio *bio = bio_alloc_bioset(GFP_NOIO, 1, &drbd_md_io_bio_set);
980980 struct drbd_device *device = ctx->device;
981981 struct drbd_bitmap *b = device->bitmap;
982982 struct page *page;
drivers/block/drbd/drbd_int.h+2−4
@@ -1422,8 +1422,6 @@ extern mempool_t drbd_md_io_page_pool; · 1421 unmodified lines
14221422 /* We also need to make sure we get a bio
14231423 * when we need it for housekeeping purposes */
14241424 extern struct bio_set drbd_md_io_bio_set;
1425−/* to allocate from that set */
1426−extern struct bio *bio_alloc_drbd(gfp_t gfp_mask);
14271425
14281426 /* And a bio_set for cloning */
14291427 extern struct bio_set drbd_io_bio_set;
@@ -1579,8 +1577,8 @@ static inline void drbd_submit_bio_noacct(struct drbd_device *device, · 149 unmodified lines
15791577 int fault_type, struct bio *bio)
15801578 {
15811579 __release(local);
1582− if (!bio->bi_disk) {
1583− drbd_err(device, "drbd_submit_bio_noacct: bio->bi_disk == NULL\n");
1580+ if (!bio->bi_bdev) {
1581+ drbd_err(device, "drbd_submit_bio_noacct: bio->bi_bdev == NULL\n");
15841582 bio->bi_status = BLK_STS_IOERR;
15851583 bio_endio(bio);
15861584 return;
drivers/block/drbd/drbd_main.c+0−13
@@ -138,19 +138,6 @@ static const struct block_device_operations drbd_ops = { · 137 unmodified lines
138138 .release = drbd_release,
139139 };
140140
141−struct bio *bio_alloc_drbd(gfp_t gfp_mask)
142−{
143− struct bio *bio;
144−
145− if (!bioset_initialized(&drbd_md_io_bio_set))
146− return bio_alloc(gfp_mask, 1);
147−
148− bio = bio_alloc_bioset(gfp_mask, 1, &drbd_md_io_bio_set);
149− if (!bio)
150− return NULL;
151− return bio;
152−}
153−
154141 #ifdef __CHECKER__
155142 /* When checking with sparse, and this is an inline function, sparse will
156143 give tons of false positives. When this is a real functions sparse works.
drivers/block/drbd/drbd_req.c+5−2
@@ -30,7 +30,10 @@ static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio · 29 unmodified lines
3030 return NULL;
3131 memset(req, 0, sizeof(*req));
3232
33− drbd_req_make_private_bio(req, bio_src);
33+ req->private_bio = bio_clone_fast(bio_src, GFP_NOIO, &drbd_io_bio_set);
34+ req->private_bio->bi_private = req;
35+ req->private_bio->bi_end_io = drbd_request_endio;
36+
3437 req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0)
3538 | (bio_op(bio_src) == REQ_OP_WRITE_SAME ? RQ_WSAME : 0)
3639 | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_ZEROES : 0)
@@ -1595,7 +1598,7 @@ void do_submit(struct work_struct *ws) · 1558 unmodified lines
15951598
15961599 blk_qc_t drbd_submit_bio(struct bio *bio)
15971600 {
1598− struct drbd_device *device = bio->bi_disk->private_data;
1601+ struct drbd_device *device = bio->bi_bdev->bd_disk->private_data;
15991602 unsigned long start_jif;
16001603
16011604 blk_queue_split(&bio);
drivers/block/drbd/drbd_req.h+0−12
@@ -256,18 +256,6 @@ enum drbd_req_state_bits { · 255 unmodified lines
256256 #define MR_WRITE 1
257257 #define MR_READ 2
258258
259−static inline void drbd_req_make_private_bio(struct drbd_request *req, struct bio *bio_src)
260−{
261− struct bio *bio;
262− bio = bio_clone_fast(bio_src, GFP_NOIO, &drbd_io_bio_set);
263−
264− req->private_bio = bio;
265−
266− bio->bi_private = req;
267− bio->bi_end_io = drbd_request_endio;
268− bio->bi_next = NULL;
269−}
270−
271259 /* Short lived temporary struct on the stack.
272260 * We could squirrel the error to be returned into
273261 * bio->bi_iter.bi_size, or similar. But that would be too ugly. */
drivers/block/drbd/drbd_worker.c+4−1
@@ -1523,8 +1523,11 @@ int w_restart_disk_io(struct drbd_work *w, int cancel) · 1522 unmodified lines
15231523 if (bio_data_dir(req->master_bio) == WRITE && req->rq_state & RQ_IN_ACT_LOG)
15241524 drbd_al_begin_io(device, &req->i);
15251525
1526− drbd_req_make_private_bio(req, req->master_bio);
1526+ req->private_bio = bio_clone_fast(req->master_bio, GFP_NOIO,
1527+ &drbd_io_bio_set);
15271528 bio_set_dev(req->private_bio, device->ldev->backing_bdev);
1529+ req->private_bio->bi_private = req;
1530+ req->private_bio->bi_end_io = drbd_request_endio;
15281531 submit_bio_noacct(req->private_bio);
15291532
15301533 return 0;
drivers/block/mtip32xx/mtip32xx.c+1−1
@@ -1015,7 +1015,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, · 1014 unmodified lines
10151015 rq->timeout = timeout;
10161016
10171017 /* insert request and run queue */
1018− blk_execute_rq(rq->q, NULL, rq, true);
1018+ blk_execute_rq(NULL, rq, true);
10191019
10201020 if (int_cmd->status) {
10211021 dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n",
drivers/block/null_blk/main.c+1−1
@@ -1420,7 +1420,7 @@ static blk_qc_t null_submit_bio(struct bio *bio) · 1419 unmodified lines
14201420 {
14211421 sector_t sector = bio->bi_iter.bi_sector;
14221422 sector_t nr_sectors = bio_sectors(bio);
1423− struct nullb *nullb = bio->bi_disk->private_data;
1423+ struct nullb *nullb = bio->bi_bdev->bd_disk->private_data;
14241424 struct nullb_queue *nq = nullb_to_queue(nullb);
14251425 struct nullb_cmd *cmd;
14261426
drivers/block/null_blk/zoned.c+4−4
@@ -148,10 +148,6 @@ int null_init_zoned_dev(struct nullb_device *dev, struct request_queue *q) · 147 unmodified lines
148148 sector += dev->zone_size_sects;
149149 }
150150
151− q->limits.zoned = BLK_ZONED_HM;
152− blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
153− blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE);
154−
155151 return 0;
156152 }
157153
@@ -160,6 +156,10 @@ int null_register_zoned_dev(struct nullb *nullb) · 2 unmodified lines
160156 struct nullb_device *dev = nullb->dev;
161157 struct request_queue *q = nullb->q;
162158
159+ blk_queue_set_zoned(nullb->disk, BLK_ZONED_HM);
160+ blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
161+ blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE);
162+
163163 if (queue_is_mq(q)) {
164164 int ret = blk_revalidate_disk_zones(nullb->disk, NULL);
165165
drivers/block/paride/pd.c+1−1
@@ -781,7 +781,7 @@ static int pd_special_command(struct pd_unit *disk, · 780 unmodified lines
781781 req = blk_mq_rq_to_pdu(rq);
782782
783783 req->func = func;
784− blk_execute_rq(disk->gd->queue, disk->gd, rq, 0);
784+ blk_execute_rq(disk->gd, rq, 0);
785785 blk_put_request(rq);
786786 return 0;
787787 }
drivers/block/pktcdvd.c+3−3
@@ -722,7 +722,7 @@ static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command * · 721 unmodified lines
722722 if (cgc->quiet)
723723 rq->rq_flags |= RQF_QUIET;
724724
725− blk_execute_rq(rq->q, pd->bdev->bd_disk, rq, 0);
725+ blk_execute_rq(pd->bdev->bd_disk, rq, 0);
726726 if (scsi_req(rq)->result)
727727 ret = -EIO;
728728 out:
@@ -2374,7 +2374,7 @@ static blk_qc_t pkt_submit_bio(struct bio *bio) · 1645 unmodified lines
23742374
23752375 blk_queue_split(&bio);
23762376
2377− pd = bio->bi_disk->queue->queuedata;
2377+ pd = bio->bi_bdev->bd_disk->queue->queuedata;
23782378 if (!pd) {
23792379 pr_err("%s incorrect request queue\n", bio_devname(bio, b));
23802380 goto end_io;
@@ -2418,7 +2418,7 @@ static blk_qc_t pkt_submit_bio(struct bio *bio) · 37 unmodified lines
24182418 split = bio;
24192419 }
24202420
2421− pkt_make_request_write(bio->bi_disk->queue, split);
2421+ pkt_make_request_write(bio->bi_bdev->bd_disk->queue, split);
24222422 } while (split != bio);
24232423
24242424 return BLK_QC_T_NONE;
drivers/block/ps3vram.c+1−1
@@ -581,7 +581,7 @@ out: · 580 unmodified lines
581581
582582 static blk_qc_t ps3vram_submit_bio(struct bio *bio)
583583 {
584− struct ps3_system_bus_device *dev = bio->bi_disk->private_data;
584+ struct ps3_system_bus_device *dev = bio->bi_bdev->bd_disk->private_data;
585585 struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev);
586586 int busy;
587587
drivers/block/rbd.c+0−19
@@ -692,29 +692,10 @@ static void rbd_release(struct gendisk *disk, fmode_t mode) · 691 unmodified lines
692692 put_device(&rbd_dev->dev);
693693 }
694694
695−static int rbd_set_read_only(struct block_device *bdev, bool ro)
696−{
697− struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
698−
699− /*
700− * Both images mapped read-only and snapshots can't be marked
701− * read-write.
702− */
703− if (!ro) {
704− if (rbd_is_ro(rbd_dev))
705− return -EROFS;
706−
707− rbd_assert(!rbd_is_snap(rbd_dev));
708− }
709−
710− return 0;
711−}
712−
713695 static const struct block_device_operations rbd_bd_ops = {
714696 .owner = THIS_MODULE,
715697 .open = rbd_open,
716698 .release = rbd_release,
717− .set_read_only = rbd_set_read_only,
718699 };
719700
720701 /*
drivers/block/rsxx/dev.c+1−1
@@ -122,7 +122,7 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card, · 121 unmodified lines
122122
123123 static blk_qc_t rsxx_submit_bio(struct bio *bio)
124124 {
125− struct rsxx_cardinfo *card = bio->bi_disk->private_data;
125+ struct rsxx_cardinfo *card = bio->bi_bdev->bd_disk->private_data;
126126 struct rsxx_bio_meta *bio_meta;
127127 blk_status_t st = BLK_STS_IOERR;
128128
drivers/block/sx8.c+2−2
@@ -539,7 +539,7 @@ static int carm_array_info (struct carm_host *host, unsigned int array_idx) · 538 unmodified lines
539539 spin_unlock_irq(&host->lock);
540540
541541 DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag);
542− blk_execute_rq_nowait(host->oob_q, NULL, rq, true, NULL);
542+ blk_execute_rq_nowait(NULL, rq, true, NULL);
543543
544544 return 0;
545545
@@ -578,7 +578,7 @@ static int carm_send_special (struct carm_host *host, carm_sspc_t func) · 32 unmodified lines
578578 crq->msg_bucket = (u32) rc;
579579
580580 DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag);
581− blk_execute_rq_nowait(host->oob_q, NULL, rq, true, NULL);
581+ blk_execute_rq_nowait(NULL, rq, true, NULL);
582582
583583 return 0;
584584 }
drivers/block/umem.c+1−1
@@ -521,7 +521,7 @@ static int mm_check_plugged(struct cardinfo *card) · 520 unmodified lines
521521
522522 static blk_qc_t mm_submit_bio(struct bio *bio)
523523 {
524− struct cardinfo *card = bio->bi_disk->private_data;
524+ struct cardinfo *card = bio->bi_bdev->bd_disk->private_data;
525525
526526 pr_debug("mm_make_request %llu %u\n",
527527 (unsigned long long)bio->bi_iter.bi_sector,
drivers/block/virtio_blk.c+1−1
@@ -320,7 +320,7 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str) · 319 unmodified lines
320320 if (err)
321321 goto out;
322322
323− blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);
323+ blk_execute_rq(vblk->disk, req, false);
324324 err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req)));
325325 out:
326326 blk_put_request(req);
drivers/block/zram/zram_drv.c+1−1
@@ -1596,7 +1596,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) · 1595 unmodified lines
15961596 */
15971597 static blk_qc_t zram_submit_bio(struct bio *bio)
15981598 {
1599− struct zram *zram = bio->bi_disk->private_data;
1599+ struct zram *zram = bio->bi_bdev->bd_disk->private_data;
16001600
16011601 if (!valid_io_request(zram, bio->bi_iter.bi_sector,
16021602 bio->bi_iter.bi_size)) {
drivers/cdrom/cdrom.c+1−1
@@ -2214,7 +2214,7 @@ static int cdrom_read_cdda_bpc(struct cdrom_device_info *cdi, __u8 __user *ubuf, · 2213 unmodified lines
22142214 rq->timeout = 60 * HZ;
22152215 bio = rq->bio;
22162216
2217− blk_execute_rq(q, cdi->disk, rq, 0);
2217+ blk_execute_rq(cdi->disk, rq, 0);
22182218 if (scsi_req(rq)->result) {
22192219 struct scsi_sense_hdr sshdr;
22202220
drivers/ide/ide-atapi.c+1−1
@@ -107,7 +107,7 @@ int ide_queue_pc_tail(ide_drive_t *drive, struct gendisk *disk, · 106 unmodified lines
107107 memcpy(scsi_req(rq)->cmd, pc->c, 12);
108108 if (drive->media == ide_tape)
109109 scsi_req(rq)->cmd[13] = REQ_IDETAPE_PC1;
110− blk_execute_rq(drive->queue, disk, rq, 0);
110+ blk_execute_rq(disk, rq, 0);
111111 error = scsi_req(rq)->result ? -EIO : 0;
112112 put_req:
113113 blk_put_request(rq);
drivers/ide/ide-cd.c+1−1
@@ -467,7 +467,7 @@ int ide_cd_queue_pc(ide_drive_t *drive, const unsigned char *cmd, · 466 unmodified lines
467467 }
468468 }
469469
470− blk_execute_rq(drive->queue, info->disk, rq, 0);
470+ blk_execute_rq(info->disk, rq, 0);
471471 error = scsi_req(rq)->result ? -EIO : 0;
472472
473473 if (buffer)
drivers/ide/ide-cd_ioctl.c+1−1
@@ -299,7 +299,7 @@ int ide_cdrom_reset(struct cdrom_device_info *cdi) · 298 unmodified lines
299299 rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
300300 ide_req(rq)->type = ATA_PRIV_MISC;
301301 rq->rq_flags = RQF_QUIET;
302− blk_execute_rq(drive->queue, cd->disk, rq, 0);
302+ blk_execute_rq(cd->disk, rq, 0);
303303 ret = scsi_req(rq)->result ? -EIO : 0;
304304 blk_put_request(rq);
305305 /*
drivers/ide/ide-devsets.c+1−1
@@ -173,7 +173,7 @@ int ide_devset_execute(ide_drive_t *drive, const struct ide_devset *setting, · 172 unmodified lines
173173 *(int *)&scsi_req(rq)->cmd[1] = arg;
174174 ide_req(rq)->special = setting->set;
175175
176− blk_execute_rq(q, NULL, rq, 0);
176+ blk_execute_rq(NULL, rq, 0);
177177 ret = scsi_req(rq)->result;
178178 blk_put_request(rq);
179179
drivers/ide/ide-disk.c+1−1
@@ -482,7 +482,7 @@ static int set_multcount(ide_drive_t *drive, int arg) · 481 unmodified lines
482482
483483 drive->mult_req = arg;
484484 drive->special_flags |= IDE_SFLAG_SET_MULTMODE;
485− blk_execute_rq(drive->queue, NULL, rq, 0);
485+ blk_execute_rq(NULL, rq, 0);
486486 blk_put_request(rq);
487487
488488 return (drive->mult_count == arg) ? 0 : -EIO;
drivers/ide/ide-ioctls.c+2−2
@@ -137,7 +137,7 @@ static int ide_cmd_ioctl(ide_drive_t *drive, void __user *argp) · 136 unmodified lines
137137
138138 rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
139139 ide_req(rq)->type = ATA_PRIV_TASKFILE;
140− blk_execute_rq(drive->queue, NULL, rq, 0);
140+ blk_execute_rq(NULL, rq, 0);
141141 err = scsi_req(rq)->result ? -EIO : 0;
142142 blk_put_request(rq);
143143
@@ -235,7 +235,7 @@ static int generic_drive_reset(ide_drive_t *drive) · 91 unmodified lines
235235 ide_req(rq)->type = ATA_PRIV_MISC;
236236 scsi_req(rq)->cmd_len = 1;
237237 scsi_req(rq)->cmd[0] = REQ_DRIVE_RESET;
238− blk_execute_rq(drive->queue, NULL, rq, 1);
238+ blk_execute_rq(NULL, rq, 1);
239239 ret = scsi_req(rq)->result;
240240 blk_put_request(rq);
241241 return ret;
drivers/ide/ide-park.c+1−1
@@ -37,7 +37,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout) · 36 unmodified lines
3737 scsi_req(rq)->cmd_len = 1;
3838 ide_req(rq)->type = ATA_PRIV_MISC;
3939 ide_req(rq)->special = &timeout;
40− blk_execute_rq(q, NULL, rq, 1);
40+ blk_execute_rq(NULL, rq, 1);
4141 rc = scsi_req(rq)->result ? -EIO : 0;
4242 blk_put_request(rq);
4343 if (rc)
drivers/ide/ide-pm.c+2−2
@@ -27,7 +27,7 @@ int generic_ide_suspend(struct device *dev, pm_message_t mesg) · 26 unmodified lines
2727 mesg.event = PM_EVENT_FREEZE;
2828 rqpm.pm_state = mesg.event;
2929
30− blk_execute_rq(drive->queue, NULL, rq, 0);
30+ blk_execute_rq(NULL, rq, 0);
3131 ret = scsi_req(rq)->result ? -EIO : 0;
3232 blk_put_request(rq);
3333
@@ -50,7 +50,7 @@ static int ide_pm_execute_rq(struct request *rq) · 16 unmodified lines
5050 blk_mq_end_request(rq, BLK_STS_OK);
5151 return -ENXIO;
5252 }
53− blk_execute_rq(q, NULL, rq, true);
53+ blk_execute_rq(NULL, rq, true);
5454
5555 return scsi_req(rq)->result ? -EIO : 0;
5656 }
drivers/ide/ide-tape.c+1−1
@@ -868,7 +868,7 @@ static int idetape_queue_rw_tail(ide_drive_t *drive, int cmd, int size) · 867 unmodified lines
868868 goto out_put;
869869 }
870870
871− blk_execute_rq(drive->queue, tape->disk, rq, 0);
871+ blk_execute_rq(tape->disk, rq, 0);
872872
873873 /* calculate the number of transferred bytes and update buffer state */
874874 size -= scsi_req(rq)->resid_len;
drivers/ide/ide-taskfile.c+1−1
@@ -443,7 +443,7 @@ int ide_raw_taskfile(ide_drive_t *drive, struct ide_cmd *cmd, u8 *buf, · 442 unmodified lines
443443 ide_req(rq)->special = cmd;
444444 cmd->rq = rq;
445445
446− blk_execute_rq(drive->queue, NULL, rq, 0);
446+ blk_execute_rq(NULL, rq, 0);
447447 error = scsi_req(rq)->result ? -EIO : 0;
448448 put_req:
449449 blk_put_request(rq);
drivers/lightnvm/pblk-init.c+1−1
@@ -49,7 +49,7 @@ struct bio_set pblk_bio_set; · 48 unmodified lines
4949
5050 static blk_qc_t pblk_submit_bio(struct bio *bio)
5151 {
52− struct pblk *pblk = bio->bi_disk->queue->queuedata;
52+ struct pblk *pblk = bio->bi_bdev->bd_disk->queue->queuedata;
5353
5454 if (bio_op(bio) == REQ_OP_DISCARD) {
5555 pblk_discard(pblk, bio);
drivers/md/bcache/debug.c+1−1
@@ -114,7 +114,7 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio) · 113 unmodified lines
114114 check = bio_kmalloc(GFP_NOIO, bio_segments(bio));
115115 if (!check)
116116 return;
117− check->bi_disk = bio->bi_disk;
117+ bio_set_dev(check, bio->bi_bdev);
118118 check->bi_opf = REQ_OP_READ;
119119 check->bi_iter.bi_sector = bio->bi_iter.bi_sector;
120120 check->bi_iter.bi_size = bio->bi_iter.bi_size;
drivers/md/bcache/request.c+24−15
@@ -475,7 +475,7 @@ struct search { · 474 unmodified lines
475475 unsigned int read_dirty_data:1;
476476 unsigned int cache_missed:1;
477477
478− struct block_device *part;
478+ struct block_device *orig_bdev;
479479 unsigned long start_time;
480480
481481 struct btree_op op;
@@ -670,8 +670,8 @@ static void bio_complete(struct search *s) · 188 unmodified lines
670670 {
671671 if (s->orig_bio) {
672672 /* Count on bcache device */
673− part_end_io_acct(s->part, s->orig_bio, s->start_time);
674−
673+ bio_end_io_acct_remapped(s->orig_bio, s->start_time,
674+ s->orig_bdev);
675675 trace_bcache_request_end(s->d, s->orig_bio);
676676 s->orig_bio->bi_status = s->iop.status;
677677 bio_endio(s->orig_bio);
@@ -714,7 +714,8 @@ static void search_free(struct closure *cl) · 36 unmodified lines
714714 }
715715
716716 static inline struct search *search_alloc(struct bio *bio,
717− struct bcache_device *d)
717+ struct bcache_device *d, struct block_device *orig_bdev,
718+ unsigned long start_time)
718719 {
719720 struct search *s;
720721
@@ -732,7 +733,8 @@ static inline struct search *search_alloc(struct bio *bio, · 11 unmodified lines
732733 s->write = op_is_write(bio_op(bio));
733734 s->read_dirty_data = 0;
734735 /* Count on the bcache device */
735− s->start_time = part_start_io_acct(d->disk, &s->part, bio);
736+ s->orig_bdev = orig_bdev;
737+ s->start_time = start_time;
736738 s->iop.c = d->c;
737739 s->iop.bio = NULL;
738740 s->iop.inode = d->id;
@@ -894,7 +896,8 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s, · 155 unmodified lines
894896 !(bio->bi_opf & (REQ_META|REQ_PRIO)) &&
895897 s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA)
896898 reada = min_t(sector_t, dc->readahead >> 9,
897− get_capacity(bio->bi_disk) - bio_end_sector(bio));
899+ get_capacity(bio->bi_bdev->bd_disk) -
900+ bio_end_sector(bio));
898901
899902 s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);
900903
@@ -1073,7 +1076,7 @@ struct detached_dev_io_private { · 172 unmodified lines
10731076 unsigned long start_time;
10741077 bio_end_io_t *bi_end_io;
10751078 void *bi_private;
1076− struct block_device *part;
1079+ struct block_device *orig_bdev;
10771080 };
10781081
10791082 static void detached_dev_end_io(struct bio *bio)
@@ -1085,7 +1088,7 @@ static void detached_dev_end_io(struct bio *bio) · 5 unmodified lines
10851088 bio->bi_private = ddip->bi_private;
10861089
10871090 /* Count on the bcache device */
1088− part_end_io_acct(ddip->part, bio, ddip->start_time);
1091+ bio_end_io_acct_remapped(bio, ddip->start_time, ddip->orig_bdev);
10891092
10901093 if (bio->bi_status) {
10911094 struct cached_dev *dc = container_of(ddip->d,
@@ -1098,7 +1101,8 @@ static void detached_dev_end_io(struct bio *bio) · 6 unmodified lines
10981101 bio->bi_end_io(bio);
10991102 }
11001103
1101−static void detached_dev_do_request(struct bcache_device *d, struct bio *bio)
1104+static void detached_dev_do_request(struct bcache_device *d, struct bio *bio,
1105+ struct block_device *orig_bdev, unsigned long start_time)
11021106 {
11031107 struct detached_dev_io_private *ddip;
11041108 struct cached_dev *dc = container_of(d, struct cached_dev, disk);
@@ -1111,7 +1115,8 @@ static void detached_dev_do_request(struct bcache_device *d, struct bio *bio) · 6 unmodified lines
11111115 ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO);
11121116 ddip->d = d;
11131117 /* Count on the bcache device */
1114− ddip->start_time = part_start_io_acct(d->disk, &ddip->part, bio);
1118+ ddip->orig_bdev = orig_bdev;
1119+ ddip->start_time = start_time;
11151120 ddip->bi_end_io = bio->bi_end_io;
11161121 ddip->bi_private = bio->bi_private;
11171122 bio->bi_end_io = detached_dev_end_io;
@@ -1167,8 +1172,10 @@ static void quit_max_writeback_rate(struct cache_set *c, · 49 unmodified lines
11671172 blk_qc_t cached_dev_submit_bio(struct bio *bio)
11681173 {
11691174 struct search *s;
1170− struct bcache_device *d = bio->bi_disk->private_data;
1175+ struct block_device *orig_bdev = bio->bi_bdev;
1176+ struct bcache_device *d = orig_bdev->bd_disk->private_data;
11711177 struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1178+ unsigned long start_time;
11721179 int rw = bio_data_dir(bio);
11731180
11741181 if (unlikely((d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags)) ||
@@ -1193,11 +1200,13 @@ blk_qc_t cached_dev_submit_bio(struct bio *bio) · 18 unmodified lines
11931200 }
11941201 }
11951202
1203+ start_time = bio_start_io_acct(bio);
1204+
11961205 bio_set_dev(bio, dc->bdev);
11971206 bio->bi_iter.bi_sector += dc->sb.data_offset;
11981207
11991208 if (cached_dev_get(dc)) {
1200− s = search_alloc(bio, d);
1209+ s = search_alloc(bio, d, orig_bdev, start_time);
12011210 trace_bcache_request_start(s->d, bio);
12021211
12031212 if (!bio->bi_iter.bi_size) {
@@ -1218,7 +1227,7 @@ blk_qc_t cached_dev_submit_bio(struct bio *bio) · 14 unmodified lines
12181227 }
12191228 } else
12201229 /* I/O request sent to backing device */
1221− detached_dev_do_request(d, bio);
1230+ detached_dev_do_request(d, bio, orig_bdev, start_time);
12221231
12231232 return BLK_QC_T_NONE;
12241233 }
@@ -1274,7 +1283,7 @@ blk_qc_t flash_dev_submit_bio(struct bio *bio) · 49 unmodified lines
12741283 {
12751284 struct search *s;
12761285 struct closure *cl;
1277− struct bcache_device *d = bio->bi_disk->private_data;
1286+ struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;
12781287
12791288 if (unlikely(d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags))) {
12801289 bio->bi_status = BLK_STS_IOERR;
@@ -1282,7 +1291,7 @@ blk_qc_t flash_dev_submit_bio(struct bio *bio) · 1 unmodified line
12821291 return BLK_QC_T_NONE;
12831292 }
12841293
1285− s = search_alloc(bio, d);
1294+ s = search_alloc(bio, d, bio->bi_bdev, bio_start_io_acct(bio));
12861295 cl = &s->cl;
12871296 bio = &s->bio.bio;
12881297
drivers/md/bcache/super.c+1−1
@@ -1939,7 +1939,7 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb) · 1938 unmodified lines
19391939 goto err;
19401940
19411941 if (bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio),
1942− BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER))
1942+ BIOSET_NEED_RESCUER))
19431943 goto err;
19441944
19451945 c->uuids = alloc_meta_bucket_pages(GFP_KERNEL, sb);
drivers/md/dm-bio-record.h+3−6
@@ -18,8 +18,7 @@ · 17 unmodified lines
1818 */
1919
2020 struct dm_bio_details {
21− struct gendisk *bi_disk;
22− u8 bi_partno;
21+ struct block_device *bi_bdev;
2322 int __bi_remaining;
2423 unsigned long bi_flags;
2524 struct bvec_iter bi_iter;
@@ -31,8 +30,7 @@ struct dm_bio_details { · 5 unmodified lines
3130
3231 static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio)
3332 {
34− bd->bi_disk = bio->bi_disk;
35− bd->bi_partno = bio->bi_partno;
33+ bd->bi_bdev = bio->bi_bdev;
3634 bd->bi_flags = bio->bi_flags;
3735 bd->bi_iter = bio->bi_iter;
3836 bd->__bi_remaining = atomic_read(&bio->__bi_remaining);
@@ -44,8 +42,7 @@ static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio) · 5 unmodified lines
4442
4543 static inline void dm_bio_restore(struct dm_bio_details *bd, struct bio *bio)
4644 {
47− bio->bi_disk = bd->bi_disk;
48− bio->bi_partno = bd->bi_partno;
45+ bio->bi_bdev = bd->bi_bdev;
4946 bio->bi_flags = bd->bi_flags;
5047 bio->bi_iter = bd->bi_iter;
5148 atomic_set(&bio->__bi_remaining, bd->__bi_remaining);
drivers/md/dm-cache-metadata.c+1−1
@@ -449,7 +449,7 @@ static int __check_incompat_features(struct cache_disk_superblock *disk_super, · 448 unmodified lines
449449 /*
450450 * Check for read-only metadata to skip the following RDWR checks.
451451 */
452− if (get_disk_ro(cmd->bdev->bd_disk))
452+ if (bdev_read_only(cmd->bdev))
453453 return 0;
454454
455455 features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;
drivers/md/dm-clone-target.c+1−13
@@ -85,12 +85,6 @@ struct clone { · 84 unmodified lines
8585
8686 struct dm_clone_metadata *cmd;
8787
88− /*
89− * bio used to flush the destination device, before committing the
90− * metadata.
91− */
92− struct bio flush_bio;
93−
9488 /* Region hydration hash table */
9589 struct hash_table_bucket *ht;
9690
@@ -1155,11 +1149,7 @@ static int commit_metadata(struct clone *clone, bool *dest_dev_flushed) · 1058 unmodified lines
11551149 goto out;
11561150 }
11571151
1158− bio_reset(&clone->flush_bio);
1159− bio_set_dev(&clone->flush_bio, clone->dest_dev->bdev);
1160− clone->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
1161−
1162− r = submit_bio_wait(&clone->flush_bio);
1152+ r = blkdev_issue_flush(clone->dest_dev->bdev);
11631153 if (unlikely(r)) {
11641154 __metadata_operation_failed(clone, "flush destination device", r);
11651155 goto out;
@@ -1886,7 +1876,6 @@ static int clone_ctr(struct dm_target *ti, unsigned int argc, char **argv) · 720 unmodified lines
18861876 bio_list_init(&clone->deferred_flush_completions);
18871877 clone->hydration_offset = 0;
18881878 atomic_set(&clone->hydrations_in_flight, 0);
1889− bio_init(&clone->flush_bio, NULL, 0);
18901879
18911880 clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
18921881 if (!clone->wq) {
@@ -1958,7 +1947,6 @@ static void clone_dtr(struct dm_target *ti) · 65 unmodified lines
19581947 struct clone *clone = ti->private;
19591948
19601949 mutex_destroy(&clone->commit_lock);
1961− bio_uninit(&clone->flush_bio);
19621950
19631951 for (i = 0; i < clone->nr_ctr_args; i++)
19641952 kfree(clone->ctr_args[i]);
drivers/md/dm-raid1.c+5−5
@@ -145,7 +145,7 @@ static void dispatch_bios(void *context, struct bio_list *bio_list) · 144 unmodified lines
145145
146146 struct dm_raid1_bio_record {
147147 struct mirror *m;
148− /* if details->bi_disk == NULL, details were not saved */
148+ /* if details->bi_bdev == NULL, details were not saved */
149149 struct dm_bio_details details;
150150 region_t write_region;
151151 };
@@ -1190,7 +1190,7 @@ static int mirror_map(struct dm_target *ti, struct bio *bio) · 1038 unmodified lines
11901190 struct dm_raid1_bio_record *bio_record =
11911191 dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record));
11921192
1193− bio_record->details.bi_disk = NULL;
1193+ bio_record->details.bi_bdev = NULL;
11941194
11951195 if (rw == WRITE) {
11961196 /* Save region for mirror_end_io() handler */
@@ -1257,7 +1257,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, · 60 unmodified lines
12571257 goto out;
12581258
12591259 if (unlikely(*error)) {
1260− if (!bio_record->details.bi_disk) {
1260+ if (!bio_record->details.bi_bdev) {
12611261 /*
12621262 * There wasn't enough memory to record necessary
12631263 * information for a retry or there was no other
@@ -1282,7 +1282,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, · 18 unmodified lines
12821282 bd = &bio_record->details;
12831283
12841284 dm_bio_restore(bd, bio);
1285− bio_record->details.bi_disk = NULL;
1285+ bio_record->details.bi_bdev = NULL;
12861286 bio->bi_status = 0;
12871287
12881288 queue_bio(ms, bio, rw);
@@ -1292,7 +1292,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, · 3 unmodified lines
12921292 }
12931293
12941294 out:
1295− bio_record->details.bi_disk = NULL;
1295+ bio_record->details.bi_bdev = NULL;
12961296
12971297 return DM_ENDIO_DONE;
12981298 }
drivers/md/dm-thin-metadata.c+1−1
@@ -636,7 +636,7 @@ static int __check_incompat_features(struct thin_disk_superblock *disk_super, · 635 unmodified lines
636636 /*
637637 * Check for read-only metadata to skip the following RDWR checks.
638638 */
639− if (get_disk_ro(pmd->bdev->bd_disk))
639+ if (bdev_read_only(pmd->bdev))
640640 return 0;
641641
642642 features = le32_to_cpu(disk_super->compat_ro_flags) & ~THIN_FEATURE_COMPAT_RO_SUPP;
drivers/md/dm-zoned-metadata.c+3−3
@@ -819,7 +819,7 @@ static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set) · 818 unmodified lines
819819 ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,
820820 mblk->page);
821821 if (ret == 0)
822− ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
822+ ret = blkdev_issue_flush(dev->bdev);
823823
824824 return ret;
825825 }
@@ -862,7 +862,7 @@ static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd, · 36 unmodified lines
862862
863863 /* Flush drive cache (this will also sync data) */
864864 if (ret == 0)
865− ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
865+ ret = blkdev_issue_flush(dev->bdev);
866866
867867 return ret;
868868 }
@@ -933,7 +933,7 @@ int dmz_flush_metadata(struct dmz_metadata *zmd) · 64 unmodified lines
933933
934934 /* If there are no dirty metadata blocks, just flush the device cache */
935935 if (list_empty(&write_list)) {
936− ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
936+ ret = blkdev_issue_flush(dev->bdev);
937937 goto err;
938938 }
939939
drivers/md/dm.c+7−7
@@ -977,16 +977,17 @@ static void clone_endio(struct bio *bio) · 976 unmodified lines
977977 struct mapped_device *md = tio->io->md;
978978 dm_endio_fn endio = tio->ti->type->end_io;
979979 struct bio *orig_bio = io->orig_bio;
980+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
980981
981982 if (unlikely(error == BLK_STS_TARGET)) {
982983 if (bio_op(bio) == REQ_OP_DISCARD &&
983− !bio->bi_disk->queue->limits.max_discard_sectors)
984+ !q->limits.max_discard_sectors)
984985 disable_discard(md);
985986 else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
986− !bio->bi_disk->queue->limits.max_write_same_sectors)
987+ !q->limits.max_write_same_sectors)
987988 disable_write_same(md);
988989 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
989− !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
990+ !q->limits.max_write_zeroes_sectors)
990991 disable_write_zeroes(md);
991992 }
992993
@@ -996,7 +997,7 @@ static void clone_endio(struct bio *bio) · 3 unmodified lines
996997 */
997998 if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
998999 sector_t written_sector = bio->bi_iter.bi_sector;
999− struct request_queue *q = orig_bio->bi_disk->queue;
1000+ struct request_queue *q = orig_bio->bi_bdev->bd_disk->queue;
10001001 u64 mask = (u64)blk_queue_zone_sectors(q) - 1;
10011002
10021003 orig_bio->bi_iter.bi_sector += written_sector & mask;
@@ -1422,8 +1423,7 @@ static int __send_empty_flush(struct clone_info *ci) · 419 unmodified lines
14221423 */
14231424 bio_init(&flush_bio, NULL, 0);
14241425 flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1425− flush_bio.bi_disk = ci->io->md->disk;
1426− bio_associate_blkg(&flush_bio);
1426+ bio_set_dev(&flush_bio, ci->io->md->disk->part0);
14271427
14281428 ci->bio = &flush_bio;
14291429 ci->sector_count = 0;
@@ -1626,7 +1626,7 @@ static blk_qc_t __split_and_process_bio(struct mapped_device *md, · 196 unmodified lines
16261626
16271627 static blk_qc_t dm_submit_bio(struct bio *bio)
16281628 {
1629− struct mapped_device *md = bio->bi_disk->private_data;
1629+ struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
16301630 blk_qc_t ret = BLK_QC_T_NONE;
16311631 int srcu_idx;
16321632 struct dm_table *map;
drivers/md/md-linear.c+1−1
@@ -252,7 +252,7 @@ static bool linear_make_request(struct mddev *mddev, struct bio *bio) · 251 unmodified lines
252252 start_sector + data_offset;
253253
254254 if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
255− !blk_queue_discard(bio->bi_disk->queue))) {
255+ !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) {
256256 /* Just ignore it */
257257 bio_endio(bio);
258258 } else {
drivers/md/md.c+29−44
@@ -340,24 +340,6 @@ static int start_readonly; · 339 unmodified lines
340340 */
341341 static bool create_on_open = true;
342342
343−struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs,
344− struct mddev *mddev)
345−{
346− if (!mddev || !bioset_initialized(&mddev->bio_set))
347− return bio_alloc(gfp_mask, nr_iovecs);
348−
349− return bio_alloc_bioset(gfp_mask, nr_iovecs, &mddev->bio_set);
350−}
351−EXPORT_SYMBOL_GPL(bio_alloc_mddev);
352−
353−static struct bio *md_bio_alloc_sync(struct mddev *mddev)
354−{
355− if (!mddev || !bioset_initialized(&mddev->sync_set))
356− return bio_alloc(GFP_NOIO, 1);
357−
358− return bio_alloc_bioset(GFP_NOIO, 1, &mddev->sync_set);
359−}
360−
361343 /*
362344 * We have a system wide 'event count' that is incremented
363345 * on any 'interesting' event, and readers of /proc/mdstat
@@ -463,8 +445,8 @@ struct md_io { · 99 unmodified lines
463445 struct mddev *mddev;
464446 bio_end_io_t *orig_bi_end_io;
465447 void *orig_bi_private;
448+ struct block_device *orig_bi_bdev;
466449 unsigned long start_time;
467− struct block_device *part;
468450 };
469451
470452 static void md_end_io(struct bio *bio)
@@ -472,7 +454,7 @@ static void md_end_io(struct bio *bio) · 1 unmodified line
472454 struct md_io *md_io = bio->bi_private;
473455 struct mddev *mddev = md_io->mddev;
474456
475− part_end_io_acct(md_io->part, bio, md_io->start_time);
457+ bio_end_io_acct_remapped(bio, md_io->start_time, md_io->orig_bi_bdev);
476458
477459 bio->bi_end_io = md_io->orig_bi_end_io;
478460 bio->bi_private = md_io->orig_bi_private;
@@ -486,7 +468,7 @@ static void md_end_io(struct bio *bio) · 7 unmodified lines
486468 static blk_qc_t md_submit_bio(struct bio *bio)
487469 {
488470 const int rw = bio_data_dir(bio);
489− struct mddev *mddev = bio->bi_disk->private_data;
471+ struct mddev *mddev = bio->bi_bdev->bd_disk->private_data;
490472
491473 if (mddev == NULL || mddev->pers == NULL) {
492474 bio_io_error(bio);
@@ -514,12 +496,12 @@ static blk_qc_t md_submit_bio(struct bio *bio) · 21 unmodified lines
514496 md_io->mddev = mddev;
515497 md_io->orig_bi_end_io = bio->bi_end_io;
516498 md_io->orig_bi_private = bio->bi_private;
499+ md_io->orig_bi_bdev = bio->bi_bdev;
517500
518501 bio->bi_end_io = md_end_io;
519502 bio->bi_private = md_io;
520503
521− md_io->start_time = part_start_io_acct(mddev->gendisk,
522− &md_io->part, bio);
504+ md_io->start_time = bio_start_io_acct(bio);
523505 }
524506
525507 /* bio could be mergeable after passing to underlayer */
@@ -613,7 +595,7 @@ static void submit_flushes(struct work_struct *ws) · 87 unmodified lines
613595 atomic_inc(&rdev->nr_pending);
614596 atomic_inc(&rdev->nr_pending);
615597 rcu_read_unlock();
616− bi = bio_alloc_mddev(GFP_NOIO, 0, mddev);
598+ bi = bio_alloc_bioset(GFP_NOIO, 0, &mddev->bio_set);
617599 bi->bi_end_io = md_end_flush;
618600 bi->bi_private = rdev;
619601 bio_set_dev(bi, rdev->bdev);
@@ -999,7 +981,7 @@ void md_super_write(struct mddev *mddev, struct md_rdev *rdev, · 379 unmodified lines
999981 if (test_bit(Faulty, &rdev->flags))
1000982 return;
1001983
1002− bio = md_bio_alloc_sync(mddev);
984+ bio = bio_alloc_bioset(GFP_NOIO, 1, &mddev->sync_set);
1003985
1004986 atomic_inc(&rdev->nr_pending);
1005987
@@ -1031,29 +1013,29 @@ int md_super_wait(struct mddev *mddev) · 25 unmodified lines
10311013 int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
10321014 struct page *page, int op, int op_flags, bool metadata_op)
10331015 {
1034− struct bio *bio = md_bio_alloc_sync(rdev->mddev);
1035− int ret;
1016+ struct bio bio;
1017+ struct bio_vec bvec;
1018+
1019+ bio_init(&bio, &bvec, 1);
10361020
10371021 if (metadata_op && rdev->meta_bdev)
1038− bio_set_dev(bio, rdev->meta_bdev);
1022+ bio_set_dev(&bio, rdev->meta_bdev);
10391023 else
1040− bio_set_dev(bio, rdev->bdev);
1041− bio_set_op_attrs(bio, op, op_flags);
1024+ bio_set_dev(&bio, rdev->bdev);
1025+ bio.bi_opf = op | op_flags;
10421026 if (metadata_op)
1043− bio->bi_iter.bi_sector = sector + rdev->sb_start;
1027+ bio.bi_iter.bi_sector = sector + rdev->sb_start;
10441028 else if (rdev->mddev->reshape_position != MaxSector &&
10451029 (rdev->mddev->reshape_backwards ==
10461030 (sector >= rdev->mddev->reshape_position)))
1047− bio->bi_iter.bi_sector = sector + rdev->new_data_offset;
1031+ bio.bi_iter.bi_sector = sector + rdev->new_data_offset;
10481032 else
1049− bio->bi_iter.bi_sector = sector + rdev->data_offset;
1050− bio_add_page(bio, page, size, 0);
1033+ bio.bi_iter.bi_sector = sector + rdev->data_offset;
1034+ bio_add_page(&bio, page, size, 0);
10511035
1052− submit_bio_wait(bio);
1036+ submit_bio_wait(&bio);
10531037
1054− ret = !bio->bi_status;
1055− bio_put(bio);
1056− return ret;
1038+ return !bio.bi_status;
10571039 }
10581040 EXPORT_SYMBOL_GPL(sync_page_io);
10591041
@@ -2417,6 +2399,12 @@ int md_integrity_add_rdev(struct md_rdev *rdev, struct mddev *mddev) · 1357 unmodified lines
24172399 }
24182400 EXPORT_SYMBOL(md_integrity_add_rdev);
24192401
2402+static bool rdev_read_only(struct md_rdev *rdev)
2403+{
2404+ return bdev_read_only(rdev->bdev) ||
2405+ (rdev->meta_bdev && bdev_read_only(rdev->meta_bdev));
2406+}
2407+
24202408 static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev)
24212409 {
24222410 char b[BDEVNAME_SIZE];
@@ -2426,8 +2414,7 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev) · 3 unmodified lines
24262414 if (find_rdev(mddev, rdev->bdev->bd_dev))
24272415 return -EEXIST;
24282416
2429− if ((bdev_read_only(rdev->bdev) || bdev_read_only(rdev->meta_bdev)) &&
2430− mddev->pers)
2417+ if (rdev_read_only(rdev) && mddev->pers)
24312418 return -EROFS;
24322419
24332420 /* make sure rdev->sectors exceeds mddev->dev_sectors */
@@ -5861,9 +5848,7 @@ int md_run(struct mddev *mddev) · 3427 unmodified lines
58615848 continue;
58625849 sync_blockdev(rdev->bdev);
58635850 invalidate_bdev(rdev->bdev);
5864− if (mddev->ro != 1 &&
5865− (bdev_read_only(rdev->bdev) ||
5866− bdev_read_only(rdev->meta_bdev))) {
5851+ if (mddev->ro != 1 && rdev_read_only(rdev)) {
58675852 mddev->ro = 1;
58685853 if (mddev->gendisk)
58695854 set_disk_ro(mddev->gendisk, 1);
@@ -6158,7 +6143,7 @@ static int restart_array(struct mddev *mddev) · 288 unmodified lines
61586143 if (test_bit(Journal, &rdev->flags) &&
61596144 !test_bit(Faulty, &rdev->flags))
61606145 has_journal = true;
6161− if (bdev_read_only(rdev->bdev))
6146+ if (rdev_read_only(rdev))
61626147 has_readonly = true;
61636148 }
61646149 rcu_read_unlock();
drivers/md/md.h+3−5
@@ -556,7 +556,7 @@ static inline void md_sync_acct(struct block_device *bdev, unsigned long nr_sect · 555 unmodified lines
556556
557557 static inline void md_sync_acct_bio(struct bio *bio, unsigned long nr_sectors)
558558 {
559− atomic_add(nr_sectors, &bio->bi_disk->sync_io);
559+ md_sync_acct(bio->bi_bdev, nr_sectors);
560560 }
561561
562562 struct md_personality
@@ -742,8 +742,6 @@ extern void md_rdev_clear(struct md_rdev *rdev); · 179 unmodified lines
742742 extern void md_handle_request(struct mddev *mddev, struct bio *bio);
743743 extern void mddev_suspend(struct mddev *mddev);
744744 extern void mddev_resume(struct mddev *mddev);
745−extern struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs,
746− struct mddev *mddev);
747745
748746 extern void md_reload_sb(struct mddev *mddev, int raid_disk);
749747 extern void md_update_sb(struct mddev *mddev, int force);
@@ -793,14 +791,14 @@ static inline void mddev_clear_unsupported_flags(struct mddev *mddev, · 43 unmodified lines
793791 static inline void mddev_check_writesame(struct mddev *mddev, struct bio *bio)
794792 {
795793 if (bio_op(bio) == REQ_OP_WRITE_SAME &&
796− !bio->bi_disk->queue->limits.max_write_same_sectors)
794+ !bio->bi_bdev->bd_disk->queue->limits.max_write_same_sectors)
797795 mddev->queue->limits.max_write_same_sectors = 0;
798796 }
799797
800798 static inline void mddev_check_write_zeroes(struct mddev *mddev, struct bio *bio)
801799 {
802800 if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
803− !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
801+ !bio->bi_bdev->bd_disk->queue->limits.max_write_zeroes_sectors)
804802 mddev->queue->limits.max_write_zeroes_sectors = 0;
805803 }
806804
drivers/md/raid1.c+4−4
@@ -794,13 +794,13 @@ static void flush_bio_list(struct r1conf *conf, struct bio *bio) · 793 unmodified lines
794794
795795 while (bio) { /* submit pending writes */
796796 struct bio *next = bio->bi_next;
797− struct md_rdev *rdev = (void *)bio->bi_disk;
797+ struct md_rdev *rdev = (void *)bio->bi_bdev;
798798 bio->bi_next = NULL;
799799 bio_set_dev(bio, rdev->bdev);
800800 if (test_bit(Faulty, &rdev->flags)) {
801801 bio_io_error(bio);
802802 } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
803− !blk_queue_discard(bio->bi_disk->queue)))
803+ !blk_queue_discard(bio->bi_bdev->bd_disk->queue)))
804804 /* Just ignore it */
805805 bio_endio(bio);
806806 else
@@ -1104,7 +1104,7 @@ static void alloc_behind_master_bio(struct r1bio *r1_bio, · 297 unmodified lines
11041104 int i = 0;
11051105 struct bio *behind_bio = NULL;
11061106
1107− behind_bio = bio_alloc_mddev(GFP_NOIO, vcnt, r1_bio->mddev);
1107+ behind_bio = bio_alloc_bioset(GFP_NOIO, vcnt, &r1_bio->mddev->bio_set);
11081108 if (!behind_bio)
11091109 return;
11101110
@@ -1520,7 +1520,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, · 409 unmodified lines
15201520 trace_block_bio_remap(mbio, disk_devt(mddev->gendisk),
15211521 r1_bio->sector);
15221522 /* flush_pending_writes() needs access to the rdev so...*/
1523− mbio->bi_disk = (void *)conf->mirrors[i].rdev;
1523+ mbio->bi_bdev = (void *)conf->mirrors[i].rdev;
15241524
15251525 cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug));
15261526 if (cb)
drivers/md/raid10.c+7−11
@@ -882,13 +882,13 @@ static void flush_pending_writes(struct r10conf *conf) · 881 unmodified lines
882882
883883 while (bio) { /* submit pending writes */
884884 struct bio *next = bio->bi_next;
885− struct md_rdev *rdev = (void*)bio->bi_disk;
885+ struct md_rdev *rdev = (void*)bio->bi_bdev;
886886 bio->bi_next = NULL;
887887 bio_set_dev(bio, rdev->bdev);
888888 if (test_bit(Faulty, &rdev->flags)) {
889889 bio_io_error(bio);
890890 } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
891− !blk_queue_discard(bio->bi_disk->queue)))
891+ !blk_queue_discard(bio->bi_bdev->bd_disk->queue)))
892892 /* Just ignore it */
893893 bio_endio(bio);
894894 else
@@ -1075,13 +1075,13 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) · 180 unmodified lines
10751075
10761076 while (bio) { /* submit pending writes */
10771077 struct bio *next = bio->bi_next;
1078− struct md_rdev *rdev = (void*)bio->bi_disk;
1078+ struct md_rdev *rdev = (void*)bio->bi_bdev;
10791079 bio->bi_next = NULL;
10801080 bio_set_dev(bio, rdev->bdev);
10811081 if (test_bit(Faulty, &rdev->flags)) {
10821082 bio_io_error(bio);
10831083 } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
1084− !blk_queue_discard(bio->bi_disk->queue)))
1084+ !blk_queue_discard(bio->bi_bdev->bd_disk->queue)))
10851085 /* Just ignore it */
10861086 bio_endio(bio);
10871087 else
@@ -1253,7 +1253,7 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, · 165 unmodified lines
12531253 trace_block_bio_remap(mbio, disk_devt(conf->mddev->gendisk),
12541254 r10_bio->sector);
12551255 /* flush_pending_writes() needs access to the rdev so...*/
1256− mbio->bi_disk = (void *)rdev;
1256+ mbio->bi_bdev = (void *)rdev;
12571257
12581258 atomic_inc(&r10_bio->remaining);
12591259
@@ -3003,7 +3003,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 1743 unmodified lines
30033003
30043004 /* Again, very different code for resync and recovery.
30053005 * Both must result in an r10bio with a list of bios that
3006− * have bi_end_io, bi_sector, bi_disk set,
3006+ * have bi_end_io, bi_sector, bi_bdev set,
30073007 * and bi_private set to the r10bio.
30083008 * For recovery, we may actually create several r10bios
30093009 * with 2 bios in each, that correspond to the bios in the main one.
@@ -4531,7 +4531,7 @@ read_more: · 1521 unmodified lines
45314531 return sectors_done;
45324532 }
45334533
4534− read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);
4534+ read_bio = bio_alloc_bioset(GFP_KERNEL, RESYNC_PAGES, &mddev->bio_set);
45354535
45364536 bio_set_dev(read_bio, rdev->bdev);
45374537 read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
@@ -4539,10 +4539,6 @@ read_more: · 1 unmodified line
45394539 read_bio->bi_private = r10_bio;
45404540 read_bio->bi_end_io = end_reshape_read;
45414541 bio_set_op_attrs(read_bio, REQ_OP_READ, 0);
4542− read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
4543− read_bio->bi_status = 0;
4544− read_bio->bi_vcnt = 0;
4545− read_bio->bi_iter.bi_size = 0;
45464542 r10_bio->master_bio = read_bio;
45474543 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;
45484544
drivers/md/raid5-ppl.c+1−1
@@ -1037,7 +1037,7 @@ static int ppl_recover(struct ppl_log *log, struct ppl_header *pplhdr, · 1036 unmodified lines
10371037 }
10381038
10391039 /* flush the disk cache after recovery if necessary */
1040− ret = blkdev_issue_flush(rdev->bdev, GFP_KERNEL);
1040+ ret = blkdev_issue_flush(rdev->bdev);
10411041 out:
10421042 __free_page(page);
10431043 return ret;
drivers/md/raid5.c+46−64
@@ -5310,7 +5310,7 @@ static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) · 5309 unmodified lines
53105310 unsigned int chunk_sectors;
53115311 unsigned int bio_sectors = bio_sectors(bio);
53125312
5313− WARN_ON_ONCE(bio->bi_partno);
5313+ WARN_ON_ONCE(bio->bi_bdev->bd_partno);
53145314
53155315 chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);
53165316 return chunk_sectors >=
@@ -5393,90 +5393,72 @@ static void raid5_align_endio(struct bio *bi) · 76 unmodified lines
53935393 static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
53945394 {
53955395 struct r5conf *conf = mddev->private;
5396− int dd_idx;
5397− struct bio* align_bi;
5396+ struct bio *align_bio;
53985397 struct md_rdev *rdev;
5399− sector_t end_sector;
5398+ sector_t sector, end_sector, first_bad;
5399+ int bad_sectors, dd_idx;
54005400
54015401 if (!in_chunk_boundary(mddev, raid_bio)) {
54025402 pr_debug("%s: non aligned\n", __func__);
54035403 return 0;
54045404 }
5405− /*
5406− * use bio_clone_fast to make a copy of the bio
5407− */
5408− align_bi = bio_clone_fast(raid_bio, GFP_NOIO, &mddev->bio_set);
5409− if (!align_bi)
5410− return 0;
5411− /*
5412− * set bi_end_io to a new function, and set bi_private to the
5413− * original bio.
5414− */
5415− align_bi->bi_end_io = raid5_align_endio;
5416− align_bi->bi_private = raid_bio;
5417− /*
5418− * compute position
5419− */
5420− align_bi->bi_iter.bi_sector =
5421− raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
5422− 0, &dd_idx, NULL);
54235405
5424− end_sector = bio_end_sector(align_bi);
5406+ sector = raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 0,
5407+ &dd_idx, NULL);
5408+ end_sector = bio_end_sector(raid_bio);
5409+
54255410 rcu_read_lock();
5411+ if (r5c_big_stripe_cached(conf, sector))
5412+ goto out_rcu_unlock;
5413+
54265414 rdev = rcu_dereference(conf->disks[dd_idx].replacement);
54275415 if (!rdev || test_bit(Faulty, &rdev->flags) ||
54285416 rdev->recovery_offset < end_sector) {
54295417 rdev = rcu_dereference(conf->disks[dd_idx].rdev);
5430− if (rdev &&
5431− (test_bit(Faulty, &rdev->flags) ||
5418+ if (!rdev)
5419+ goto out_rcu_unlock;
5420+ if (test_bit(Faulty, &rdev->flags) ||
54325421 !(test_bit(In_sync, &rdev->flags) ||
5433− rdev->recovery_offset >= end_sector)))
5434− rdev = NULL;
5422+ rdev->recovery_offset >= end_sector))
5423+ goto out_rcu_unlock;
54355424 }
54365425
5437− if (r5c_big_stripe_cached(conf, align_bi->bi_iter.bi_sector)) {
5438− rcu_read_unlock();
5439− bio_put(align_bi);
5426+ atomic_inc(&rdev->nr_pending);
5427+ rcu_read_unlock();
5428+
5429+ align_bio = bio_clone_fast(raid_bio, GFP_NOIO, &mddev->bio_set);
5430+ bio_set_dev(align_bio, rdev->bdev);
5431+ align_bio->bi_end_io = raid5_align_endio;
5432+ align_bio->bi_private = raid_bio;
5433+ align_bio->bi_iter.bi_sector = sector;
5434+
5435+ raid_bio->bi_next = (void *)rdev;
5436+
5437+ if (is_badblock(rdev, sector, bio_sectors(align_bio), &first_bad,
5438+ &bad_sectors)) {
5439+ bio_put(align_bio);
5440+ rdev_dec_pending(rdev, mddev);
54405441 return 0;
54415442 }
54425443
5443− if (rdev) {
5444− sector_t first_bad;
5445− int bad_sectors;
5446−
5447− atomic_inc(&rdev->nr_pending);
5448− rcu_read_unlock();
5449− raid_bio->bi_next = (void*)rdev;
5450− bio_set_dev(align_bi, rdev->bdev);
5451−
5452− if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
5453− bio_sectors(align_bi),
5454− &first_bad, &bad_sectors)) {
5455− bio_put(align_bi);
5456− rdev_dec_pending(rdev, mddev);
5457− return 0;
5458− }
5444+ /* No reshape active, so we can trust rdev->data_offset */
5445+ align_bio->bi_iter.bi_sector += rdev->data_offset;
54595446
5460− /* No reshape active, so we can trust rdev->data_offset */
5461− align_bi->bi_iter.bi_sector += rdev->data_offset;
5447+ spin_lock_irq(&conf->device_lock);
5448+ wait_event_lock_irq(conf->wait_for_quiescent, conf->quiesce == 0,
5449+ conf->device_lock);
5450+ atomic_inc(&conf->active_aligned_reads);
5451+ spin_unlock_irq(&conf->device_lock);
54625452
5463− spin_lock_irq(&conf->device_lock);
5464− wait_event_lock_irq(conf->wait_for_quiescent,
5465− conf->quiesce == 0,
5466− conf->device_lock);
5467− atomic_inc(&conf->active_aligned_reads);
5468− spin_unlock_irq(&conf->device_lock);
5453+ if (mddev->gendisk)
5454+ trace_block_bio_remap(align_bio, disk_devt(mddev->gendisk),
5455+ raid_bio->bi_iter.bi_sector);
5456+ submit_bio_noacct(align_bio);
5457+ return 1;
54695458
5470− if (mddev->gendisk)
5471− trace_block_bio_remap(align_bi, disk_devt(mddev->gendisk),
5472− raid_bio->bi_iter.bi_sector);
5473− submit_bio_noacct(align_bi);
5474− return 1;
5475− } else {
5476− rcu_read_unlock();
5477− bio_put(align_bi);
5478− return 0;
5479− }
5459+out_rcu_unlock:
5460+ rcu_read_unlock();
5461+ return 0;
54805462 }
54815463
54825464 static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio)
drivers/mmc/core/block.c+5−5
@@ -253,7 +253,7 @@ static ssize_t power_ro_lock_store(struct device *dev, · 252 unmodified lines
253253 goto out_put;
254254 }
255255 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP;
256− blk_execute_rq(mq->queue, NULL, req, 0);
256+ blk_execute_rq(NULL, req, 0);
257257 ret = req_to_mmc_queue_req(req)->drv_op_result;
258258 blk_put_request(req);
259259
@@ -629,7 +629,7 @@ static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md, · 369 unmodified lines
629629 rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
630630 req_to_mmc_queue_req(req)->drv_op_data = idatas;
631631 req_to_mmc_queue_req(req)->ioc_count = 1;
632− blk_execute_rq(mq->queue, NULL, req, 0);
632+ blk_execute_rq(NULL, req, 0);
633633 ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
634634 err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata);
635635 blk_put_request(req);
@@ -698,7 +698,7 @@ static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md, · 62 unmodified lines
698698 rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
699699 req_to_mmc_queue_req(req)->drv_op_data = idata;
700700 req_to_mmc_queue_req(req)->ioc_count = num_of_cmds;
701− blk_execute_rq(mq->queue, NULL, req, 0);
701+ blk_execute_rq(NULL, req, 0);
702702 ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
703703
704704 /* copy to user if data and response */
@@ -2722,7 +2722,7 @@ static int mmc_dbg_card_status_get(void *data, u64 *val) · 2017 unmodified lines
27222722 if (IS_ERR(req))
27232723 return PTR_ERR(req);
27242724 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
2725− blk_execute_rq(mq->queue, NULL, req, 0);
2725+ blk_execute_rq(NULL, req, 0);
27262726 ret = req_to_mmc_queue_req(req)->drv_op_result;
27272727 if (ret >= 0) {
27282728 *val = ret;
@@ -2761,7 +2761,7 @@ static int mmc_ext_csd_open(struct inode *inode, struct file *filp) · 32 unmodified lines
27612761 }
27622762 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;
27632763 req_to_mmc_queue_req(req)->drv_op_data = &ext_csd;
2764− blk_execute_rq(mq->queue, NULL, req, 0);
2764+ blk_execute_rq(NULL, req, 0);
27652765 err = req_to_mmc_queue_req(req)->drv_op_result;
27662766 blk_put_request(req);
27672767 if (err) {
drivers/nvdimm/blk.c+2−2
@@ -165,7 +165,7 @@ static int nsblk_do_bvec(struct nd_namespace_blk *nsblk, · 164 unmodified lines
165165 static blk_qc_t nd_blk_submit_bio(struct bio *bio)
166166 {
167167 struct bio_integrity_payload *bip;
168− struct nd_namespace_blk *nsblk = bio->bi_disk->private_data;
168+ struct nd_namespace_blk *nsblk = bio->bi_bdev->bd_disk->private_data;
169169 struct bvec_iter iter;
170170 unsigned long start;
171171 struct bio_vec bvec;
@@ -177,7 +177,7 @@ static blk_qc_t nd_blk_submit_bio(struct bio *bio) · 5 unmodified lines
177177
178178 bip = bio_integrity(bio);
179179 rw = bio_data_dir(bio);
180− do_acct = blk_queue_io_stat(bio->bi_disk->queue);
180+ do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
181181 if (do_acct)
182182 start = bio_start_io_acct(bio);
183183 bio_for_each_segment(bvec, bio, iter) {
drivers/nvdimm/btt.c+2−2
@@ -1442,7 +1442,7 @@ static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip, · 1441 unmodified lines
14421442 static blk_qc_t btt_submit_bio(struct bio *bio)
14431443 {
14441444 struct bio_integrity_payload *bip = bio_integrity(bio);
1445− struct btt *btt = bio->bi_disk->private_data;
1445+ struct btt *btt = bio->bi_bdev->bd_disk->private_data;
14461446 struct bvec_iter iter;
14471447 unsigned long start;
14481448 struct bio_vec bvec;
@@ -1452,7 +1452,7 @@ static blk_qc_t btt_submit_bio(struct bio *bio) · 3 unmodified lines
14521452 if (!bio_integrity_prep(bio))
14531453 return BLK_QC_T_NONE;
14541454
1455− do_acct = blk_queue_io_stat(bio->bi_disk->queue);
1455+ do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
14561456 if (do_acct)
14571457 start = bio_start_io_acct(bio);
14581458 bio_for_each_segment(bvec, bio, iter) {
drivers/nvdimm/pmem.c+2−2
@@ -196,13 +196,13 @@ static blk_qc_t pmem_submit_bio(struct bio *bio) · 195 unmodified lines
196196 unsigned long start;
197197 struct bio_vec bvec;
198198 struct bvec_iter iter;
199− struct pmem_device *pmem = bio->bi_disk->private_data;
199+ struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data;
200200 struct nd_region *nd_region = to_region(pmem);
201201
202202 if (bio->bi_opf & REQ_PREFLUSH)
203203 ret = nvdimm_flush(nd_region, bio);
204204
205− do_acct = blk_queue_io_stat(bio->bi_disk->queue);
205+ do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
206206 if (do_acct)
207207 start = bio_start_io_acct(bio);
208208 bio_for_each_segment(bvec, bio, iter) {
drivers/nvme/host/core.c+16−15
@@ -925,7 +925,7 @@ static void nvme_execute_rq_polled(struct request_queue *q, · 924 unmodified lines
925925
926926 rq->cmd_flags |= REQ_HIPRI;
927927 rq->end_io_data = &wait;
928− blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
928+ blk_execute_rq_nowait(bd_disk, rq, at_head, nvme_end_sync_rq);
929929
930930 while (!completion_done(&wait)) {
931931 blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
@@ -964,7 +964,7 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, · 32 unmodified lines
964964 if (poll)
965965 nvme_execute_rq_polled(req->q, NULL, req, at_head);
966966 else
967− blk_execute_rq(req->q, NULL, req, at_head);
967+ blk_execute_rq(NULL, req, at_head);
968968 if (result)
969969 *result = nvme_req(req)->result;
970970 if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
@@ -1101,7 +1101,7 @@ void nvme_execute_passthru_rq(struct request *rq) · 130 unmodified lines
11011101 u32 effects;
11021102
11031103 effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1104− blk_execute_rq(rq->q, disk, rq, 0);
1104+ blk_execute_rq(disk, rq, 0);
11051105 nvme_passthru_end(ctrl, effects);
11061106 }
11071107 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
@@ -1113,7 +1113,7 @@ static int nvme_submit_user_cmd(struct request_queue *q, · 5 unmodified lines
11131113 {
11141114 bool write = nvme_is_write(cmd);
11151115 struct nvme_ns *ns = q->queuedata;
1116− struct gendisk *disk = ns ? ns->disk : NULL;
1116+ struct block_device *bdev = ns ? ns->disk->part0 : NULL;
11171117 struct request *req;
11181118 struct bio *bio = NULL;
11191119 void *meta = NULL;
@@ -1133,8 +1133,9 @@ static int nvme_submit_user_cmd(struct request_queue *q, · 13 unmodified lines
11331133 if (ret)
11341134 goto out;
11351135 bio = req->bio;
1136− bio->bi_disk = disk;
1137− if (disk && meta_buffer && meta_len) {
1136+ if (bdev)
1137+ bio_set_dev(bio, bdev);
1138+ if (bdev && meta_buffer && meta_len) {
11381139 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
11391140 meta_seed, write);
11401141 if (IS_ERR(meta)) {
@@ -1202,7 +1203,7 @@ static int nvme_keep_alive(struct nvme_ctrl *ctrl) · 61 unmodified lines
12021203 rq->timeout = ctrl->kato * HZ;
12031204 rq->end_io_data = ctrl;
12041205
1205− blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
1206+ blk_execute_rq_nowait(NULL, rq, 0, nvme_keep_alive_end_io);
12061207
12071208 return 0;
12081209 }
@@ -2125,9 +2126,8 @@ static void nvme_update_disk_info(struct gendisk *disk, · 916 unmodified lines
21252126 nvme_config_discard(disk, ns);
21262127 nvme_config_write_zeroes(disk, ns);
21272128
2128− if ((id->nsattr & NVME_NS_ATTR_RO) ||
2129− test_bit(NVME_NS_FORCE_RO, &ns->flags))
2130− set_disk_ro(disk, true);
2129+ set_disk_ro(disk, (id->nsattr & NVME_NS_ATTR_RO) ||
2130+ test_bit(NVME_NS_FORCE_RO, &ns->flags));
21312131 }
21322132
21332133 static inline bool nvme_first_scan(struct gendisk *disk)
@@ -2176,17 +2176,18 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id) · 42 unmodified lines
21762176 ns->lba_shift = id->lbaf[lbaf].ds;
21772177 nvme_set_queue_limits(ns->ctrl, ns->queue);
21782178
2179+ ret = nvme_configure_metadata(ns, id);
2180+ if (ret)
2181+ goto out_unfreeze;
2182+ nvme_set_chunk_sectors(ns, id);
2183+ nvme_update_disk_info(ns->disk, ns, id);
2184+
21792185 if (ns->head->ids.csi == NVME_CSI_ZNS) {
21802186 ret = nvme_update_zone_info(ns, lbaf);
21812187 if (ret)
21822188 goto out_unfreeze;
21832189 }
21842190
2185− ret = nvme_configure_metadata(ns, id);
2186− if (ret)
2187− goto out_unfreeze;
2188− nvme_set_chunk_sectors(ns, id);
2189− nvme_update_disk_info(ns->disk, ns, id);
21902191 blk_mq_unfreeze_queue(ns->disk->queue);
21912192
21922193 if (blk_queue_is_zoned(ns->queue)) {
drivers/nvme/host/lightnvm.c+3−4
@@ -695,7 +695,7 @@ static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd, · 694 unmodified lines
695695
696696 rq->end_io_data = rqd;
697697
698− blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io);
698+ blk_execute_rq_nowait(NULL, rq, 0, nvme_nvm_end_io);
699699
700700 return 0;
701701
@@ -757,7 +757,6 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q, · 55 unmodified lines
757757 {
758758 bool write = nvme_is_write((struct nvme_command *)vcmd);
759759 struct nvm_dev *dev = ns->ndev;
760− struct gendisk *disk = ns->disk;
761760 struct request *rq;
762761 struct bio *bio = NULL;
763762 __le64 *ppa_list = NULL;
@@ -817,10 +816,10 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q, · 53 unmodified lines
817816 vcmd->ph_rw.metadata = cpu_to_le64(metadata_dma);
818817 }
819818
820− bio->bi_disk = disk;
819+ bio_set_dev(bio, ns->disk->part0);
821820 }
822821
823− blk_execute_rq(q, NULL, rq, 0);
822+ blk_execute_rq(NULL, rq, 0);
824823
825824 if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
826825 ret = -EINTR;
drivers/nvme/host/multipath.c+3−3
@@ -296,7 +296,7 @@ static bool nvme_available_path(struct nvme_ns_head *head) · 295 unmodified lines
296296
297297 blk_qc_t nvme_ns_head_submit_bio(struct bio *bio)
298298 {
299− struct nvme_ns_head *head = bio->bi_disk->private_data;
299+ struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;
300300 struct device *dev = disk_to_dev(head->disk);
301301 struct nvme_ns *ns;
302302 blk_qc_t ret = BLK_QC_T_NONE;
@@ -312,7 +312,7 @@ blk_qc_t nvme_ns_head_submit_bio(struct bio *bio) · 9 unmodified lines
312312 srcu_idx = srcu_read_lock(&head->srcu);
313313 ns = nvme_find_path(head);
314314 if (likely(ns)) {
315− bio->bi_disk = ns->disk;
315+ bio_set_dev(bio, ns->disk->part0);
316316 bio->bi_opf |= REQ_NVME_MPATH;
317317 trace_block_bio_remap(bio, disk_devt(ns->head->disk),
318318 bio->bi_iter.bi_sector);
@@ -352,7 +352,7 @@ static void nvme_requeue_work(struct work_struct *work) · 33 unmodified lines
352352 * Reset disk to the mpath node and resubmit to select a new
353353 * path.
354354 */
355− bio->bi_disk = head->disk;
355+ bio_set_dev(bio, head->disk->part0);
356356 submit_bio_noacct(bio);
357357 }
358358 }
drivers/nvme/host/pci.c+2−2
@@ -1357,7 +1357,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) · 1356 unmodified lines
13571357 }
13581358
13591359 abort_req->end_io_data = NULL;
1360− blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
1360+ blk_execute_rq_nowait(NULL, abort_req, 0, abort_endio);
13611361
13621362 /*
13631363 * The aborted req will be completed on receiving the abort req.
@@ -2281,7 +2281,7 @@ static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode) · 917 unmodified lines
22812281 req->end_io_data = nvmeq;
22822282
22832283 init_completion(&nvmeq->delete_done);
2284− blk_execute_rq_nowait(q, NULL, req, false,
2284+ blk_execute_rq_nowait(NULL, req, false,
22852285 opcode == nvme_admin_delete_cq ?
22862286 nvme_del_cq_end : nvme_del_queue_end);
22872287 return 0;
drivers/nvme/host/rdma.c+1−1
@@ -1468,7 +1468,7 @@ static int nvme_rdma_map_sg_pi(struct nvme_rdma_queue *queue, · 1467 unmodified lines
14681468 if (unlikely(nr))
14691469 goto mr_put;
14701470
1471− nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_disk), c,
1471+ nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_bdev->bd_disk), c,
14721472 req->mr->sig_attrs, ns->pi_type);
14731473 nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask);
14741474
drivers/nvme/host/zns.c+3−8
@@ -9,13 +9,7 @@ · 8 unmodified lines
99
1010 int nvme_revalidate_zones(struct nvme_ns *ns)
1111 {
12− struct request_queue *q = ns->queue;
13− int ret;
14−
15− ret = blk_revalidate_disk_zones(ns->disk, NULL);
16− if (!ret)
17− blk_queue_max_zone_append_sectors(q, ns->ctrl->max_zone_append);
18− return ret;
12+ return blk_revalidate_disk_zones(ns->disk, NULL);
1913 }
2014
2115 static int nvme_set_max_append(struct nvme_ctrl *ctrl)
@@ -109,10 +103,11 @@ int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf) · 87 unmodified lines
109103 goto free_data;
110104 }
111105
112− q->limits.zoned = BLK_ZONED_HM;
106+ blk_queue_set_zoned(ns->disk, BLK_ZONED_HM);
113107 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
114108 blk_queue_max_open_zones(q, le32_to_cpu(id->mor) + 1);
115109 blk_queue_max_active_zones(q, le32_to_cpu(id->mar) + 1);
110+ blk_queue_max_zone_append_sectors(q, ns->ctrl->max_zone_append);
116111 free_data:
117112 kfree(id);
118113 return status;
drivers/nvme/target/io-cmd-bdev.c+1−1
@@ -333,7 +333,7 @@ static void nvmet_bdev_execute_flush(struct nvmet_req *req) · 332 unmodified lines
333333
334334 u16 nvmet_bdev_flush(struct nvmet_req *req)
335335 {
336− if (blkdev_issue_flush(req->ns->bdev, GFP_KERNEL))
336+ if (blkdev_issue_flush(req->ns->bdev))
337337 return NVME_SC_INTERNAL | NVME_SC_DNR;
338338 return 0;
339339 }
drivers/nvme/target/passthru.c+1−1
@@ -275,7 +275,7 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req) · 274 unmodified lines
275275 schedule_work(&req->p.work);
276276 } else {
277277 rq->end_io_data = req;
278− blk_execute_rq_nowait(rq->q, ns ? ns->disk : NULL, rq, 0,
278+ blk_execute_rq_nowait(ns ? ns->disk : NULL, rq, 0,
279279 nvmet_passthru_req_done);
280280 }
281281
drivers/s390/block/dasd.c+5−21
@@ -428,23 +428,15 @@ static int dasd_state_unfmt_to_basic(struct dasd_device *device) · 427 unmodified lines
428428 static int
429429 dasd_state_ready_to_online(struct dasd_device * device)
430430 {
431− struct gendisk *disk;
432− struct disk_part_iter piter;
433− struct block_device *part;
434−
435431 device->state = DASD_STATE_ONLINE;
436432 if (device->block) {
437433 dasd_schedule_block_bh(device->block);
438434 if ((device->features & DASD_FEATURE_USERAW)) {
439− disk = device->block->gdp;
440− kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE);
435+ kobject_uevent(&disk_to_dev(device->block->gdp)->kobj,
436+ KOBJ_CHANGE);
441437 return 0;
442438 }
443− disk = device->block->bdev->bd_disk;
444− disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
445− while ((part = disk_part_iter_next(&piter)))
446− kobject_uevent(bdev_kobj(part), KOBJ_CHANGE);
447− disk_part_iter_exit(&piter);
439+ disk_uevent(device->block->bdev->bd_disk, KOBJ_CHANGE);
448440 }
449441 return 0;
450442 }
@@ -455,9 +447,6 @@ dasd_state_ready_to_online(struct dasd_device * device) · 4 unmodified lines
455447 static int dasd_state_online_to_ready(struct dasd_device *device)
456448 {
457449 int rc;
458− struct gendisk *disk;
459− struct disk_part_iter piter;
460− struct block_device *part;
461450
462451 if (device->discipline->online_to_ready) {
463452 rc = device->discipline->online_to_ready(device);
@@ -466,13 +455,8 @@ static int dasd_state_online_to_ready(struct dasd_device *device) · 2 unmodified lines
466455 }
467456
468457 device->state = DASD_STATE_READY;
469− if (device->block && !(device->features & DASD_FEATURE_USERAW)) {
470− disk = device->block->bdev->bd_disk;
471− disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
472− while ((part = disk_part_iter_next(&piter)))
473− kobject_uevent(bdev_kobj(part), KOBJ_CHANGE);
474− disk_part_iter_exit(&piter);
475− }
458+ if (device->block && !(device->features & DASD_FEATURE_USERAW))
459+ disk_uevent(device->block->bdev->bd_disk, KOBJ_CHANGE);
476460 return 0;
477461 }
478462
drivers/s390/block/dcssblk.c+1−5
@@ -879,17 +879,13 @@ dcssblk_submit_bio(struct bio *bio) · 878 unmodified lines
879879 blk_queue_split(&bio);
880880
881881 bytes_done = 0;
882− dev_info = bio->bi_disk->private_data;
882+ dev_info = bio->bi_bdev->bd_disk->private_data;
883883 if (dev_info == NULL)
884884 goto fail;
885885 if ((bio->bi_iter.bi_sector & 7) != 0 ||
886886 (bio->bi_iter.bi_size & 4095) != 0)
887887 /* Request is not page-aligned. */
888888 goto fail;
889− if (bio_end_sector(bio) > get_capacity(bio->bi_disk)) {
890− /* Request beyond end of DCSS segment. */
891− goto fail;
892− }
893889 /* verify data transfer direction */
894890 if (dev_info->is_shared) {
895891 switch (dev_info->segment_type) {
drivers/s390/block/xpram.c+1−1
@@ -184,7 +184,7 @@ static unsigned long xpram_highest_page_index(void) · 183 unmodified lines
184184 */
185185 static blk_qc_t xpram_submit_bio(struct bio *bio)
186186 {
187− xpram_device_t *xdev = bio->bi_disk->private_data;
187+ xpram_device_t *xdev = bio->bi_bdev->bd_disk->private_data;
188188 struct bio_vec bvec;
189189 struct bvec_iter iter;
190190 unsigned int index;
drivers/scsi/scsi_error.c+1−1
@@ -2007,7 +2007,7 @@ static void scsi_eh_lock_door(struct scsi_device *sdev) · 2006 unmodified lines
20072007 req->timeout = 10 * HZ;
20082008 rq->retries = 5;
20092009
2010− blk_execute_rq_nowait(req->q, NULL, req, 1, eh_lock_door_done);
2010+ blk_execute_rq_nowait(NULL, req, 1, eh_lock_door_done);
20112011 }
20122012
20132013 /**
drivers/scsi/scsi_lib.c+1−1
@@ -269,7 +269,7 @@ int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, · 268 unmodified lines
269269 /*
270270 * head injection *required* here otherwise quiesce won't work
271271 */
272− blk_execute_rq(req->q, NULL, req, 1);
272+ blk_execute_rq(NULL, req, 1);
273273
274274 /*
275275 * Some devices (USB mass-storage in particular) may transfer
drivers/scsi/sd_zbc.c+39−4
@@ -665,12 +665,28 @@ static int sd_zbc_init_disk(struct scsi_disk *sdkp) · 664 unmodified lines
665665 return 0;
666666 }
667667
668−void sd_zbc_release_disk(struct scsi_disk *sdkp)
668+static void sd_zbc_clear_zone_info(struct scsi_disk *sdkp)
669669 {
670+ /* Serialize against revalidate zones */
671+ mutex_lock(&sdkp->rev_mutex);
672+
670673 kvfree(sdkp->zones_wp_offset);
671674 sdkp->zones_wp_offset = NULL;
672675 kfree(sdkp->zone_wp_update_buf);
673676 sdkp->zone_wp_update_buf = NULL;
677+
678+ sdkp->nr_zones = 0;
679+ sdkp->rev_nr_zones = 0;
680+ sdkp->zone_blocks = 0;
681+ sdkp->rev_zone_blocks = 0;
682+
683+ mutex_unlock(&sdkp->rev_mutex);
684+}
685+
686+void sd_zbc_release_disk(struct scsi_disk *sdkp)
687+{
688+ if (sd_is_zoned(sdkp))
689+ sd_zbc_clear_zone_info(sdkp);
674690 }
675691
676692 static void sd_zbc_revalidate_zones_cb(struct gendisk *disk)
@@ -769,6 +785,21 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buf) · 92 unmodified lines
769785 */
770786 return 0;
771787
788+ /* READ16/WRITE16 is mandatory for ZBC disks */
789+ sdkp->device->use_16_for_rw = 1;
790+ sdkp->device->use_10_for_rw = 0;
791+
792+ if (!blk_queue_is_zoned(q)) {
793+ /*
794+ * This can happen for a host aware disk with partitions.
795+ * The block device zone information was already cleared
796+ * by blk_queue_set_zoned(). Only clear the scsi disk zone
797+ * information and exit early.
798+ */
799+ sd_zbc_clear_zone_info(sdkp);
800+ return 0;
801+ }
802+
772803 /* Check zoned block device characteristics (unconstrained reads) */
773804 ret = sd_zbc_check_zoned_characteristics(sdkp, buf);
774805 if (ret)
@@ -789,9 +820,13 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buf) · 14 unmodified lines
789820 blk_queue_max_active_zones(q, 0);
790821 nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks);
791822
792− /* READ16/WRITE16 is mandatory for ZBC disks */
793− sdkp->device->use_16_for_rw = 1;
794− sdkp->device->use_10_for_rw = 0;
823+ /*
824+ * Per ZBC and ZAC specifications, writes in sequential write required
825+ * zones of host-managed devices must be aligned to the device physical
826+ * block size.
827+ */
828+ if (blk_queue_zoned_model(q) == BLK_ZONED_HM)
829+ blk_queue_zone_write_granularity(q, sdkp->physical_block_size);
795830
796831 sdkp->rev_nr_zones = nr_zones;
797832 sdkp->rev_zone_blocks = zone_blocks;
drivers/scsi/sg.c+1−2
@@ -829,8 +829,7 @@ sg_common_write(Sg_fd * sfp, Sg_request * srp, · 828 unmodified lines
829829
830830 srp->rq->timeout = timeout;
831831 kref_get(&sfp->f_ref); /* sg_rq_end_io() does kref_put(). */
832− blk_execute_rq_nowait(sdp->device->request_queue, sdp->disk,
833− srp->rq, at_head, sg_rq_end_io);
832+ blk_execute_rq_nowait(sdp->disk, srp->rq, at_head, sg_rq_end_io);
834833 return 0;
835834 }
836835
drivers/scsi/st.c+1−1
@@ -585,7 +585,7 @@ static int st_scsi_execute(struct st_request *SRpnt, const unsigned char *cmd, · 584 unmodified lines
585585 rq->retries = retries;
586586 req->end_io_data = SRpnt;
587587
588− blk_execute_rq_nowait(req->q, NULL, req, 1, st_scsi_execute_end);
588+ blk_execute_rq_nowait(NULL, req, 1, st_scsi_execute_end);
589589 return 0;
590590 }
591591
drivers/target/target_core_file.c+6−14
@@ -241,6 +241,7 @@ struct target_core_file_cmd { · 240 unmodified lines
241241 unsigned long len;
242242 struct se_cmd *cmd;
243243 struct kiocb iocb;
244+ struct bio_vec bvecs[];
244245 };
245246
246247 static void cmd_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
@@ -268,29 +269,22 @@ fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, · 21 unmodified lines
268269 struct target_core_file_cmd *aio_cmd;
269270 struct iov_iter iter = {};
270271 struct scatterlist *sg;
271− struct bio_vec *bvec;
272272 ssize_t len = 0;
273273 int ret = 0, i;
274274
275− aio_cmd = kmalloc(sizeof(struct target_core_file_cmd), GFP_KERNEL);
275+ aio_cmd = kmalloc(struct_size(aio_cmd, bvecs, sgl_nents), GFP_KERNEL);
276276 if (!aio_cmd)
277277 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
278278
279− bvec = kcalloc(sgl_nents, sizeof(struct bio_vec), GFP_KERNEL);
280− if (!bvec) {
281− kfree(aio_cmd);
282− return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
283− }
284−
285279 for_each_sg(sgl, sg, sgl_nents, i) {
286− bvec[i].bv_page = sg_page(sg);
287− bvec[i].bv_len = sg->length;
288− bvec[i].bv_offset = sg->offset;
280+ aio_cmd->bvecs[i].bv_page = sg_page(sg);
281+ aio_cmd->bvecs[i].bv_len = sg->length;
282+ aio_cmd->bvecs[i].bv_offset = sg->offset;
289283
290284 len += sg->length;
291285 }
292286
293− iov_iter_bvec(&iter, is_write, bvec, sgl_nents, len);
287+ iov_iter_bvec(&iter, is_write, aio_cmd->bvecs, sgl_nents, len);
294288
295289 aio_cmd->cmd = cmd;
296290 aio_cmd->len = len;
@@ -307,8 +301,6 @@ fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, · 10 unmodified lines
307301 else
308302 ret = call_read_iter(file, &aio_cmd->iocb, &iter);
309303
310− kfree(bvec);
311−
312304 if (ret != -EIOCBQUEUED)
313305 cmd_rw_aio_complete(&aio_cmd->iocb, ret, 0);
314306
drivers/target/target_core_pscsi.c+1−2
@@ -1000,8 +1000,7 @@ pscsi_execute_cmd(struct se_cmd *cmd) · 999 unmodified lines
10001000 req->timeout = PS_TIMEOUT_OTHER;
10011001 scsi_req(req)->retries = PS_RETRY;
10021002
1003− blk_execute_rq_nowait(pdv->pdv_sd->request_queue, NULL, req,
1004− (cmd->sam_task_attr == TCM_HEAD_TAG),
1003+ blk_execute_rq_nowait(NULL, req, (cmd->sam_task_attr == TCM_HEAD_TAG),
10051004 pscsi_req_done);
10061005
10071006 return 0;
fs/block_dev.c+9−11
@@ -126,7 +126,6 @@ int truncate_bdev_range(struct block_device *bdev, fmode_t mode, · 125 unmodified lines
126126 bd_abort_claiming(bdev, truncate_bdev_range);
127127 return 0;
128128 }
129−EXPORT_SYMBOL(truncate_bdev_range);
130129
131130 static void set_init_blocksize(struct block_device *bdev)
132131 {
@@ -424,7 +423,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) · 291 unmodified lines
424423 dio->size += bio->bi_iter.bi_size;
425424 pos += bio->bi_iter.bi_size;
426425
427− nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
426+ nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES);
428427 if (!nr_pages) {
429428 bool polled = false;
430429
@@ -489,9 +488,10 @@ blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) · 58 unmodified lines
489488 {
490489 int nr_pages;
491490
492− nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1);
493− if (!nr_pages)
491+ if (!iov_iter_count(iter))
494492 return 0;
493+
494+ nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES + 1);
495495 if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)
496496 return __blkdev_direct_IO_simple(iocb, iter, nr_pages);
497497
@@ -688,7 +688,7 @@ int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync) · 190 unmodified lines
688688 * i_mutex and doing so causes performance issues with concurrent
689689 * O_SYNC writers to a block device.
690690 */
691− error = blkdev_issue_flush(bdev, GFP_KERNEL);
691+ error = blkdev_issue_flush(bdev);
692692 if (error == -EOPNOTSUPP)
693693 error = 0;
694694
@@ -1808,13 +1808,11 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, · 1113 unmodified lines
18081808 return error;
18091809
18101810 /*
1811− * Invalidate again; if someone wandered in and dirtied a page,
1812− * the caller will be given -EBUSY. The third argument is
1813− * inclusive, so the rounding here is safe.
1811+ * Invalidate the page cache again; if someone wandered in and dirtied
1812+ * a page, we just discard it - userspace has no way of knowing whether
1813+ * the write happened before or after discard completing...
18141814 */
1815− return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
1816− start >> PAGE_SHIFT,
1817− end >> PAGE_SHIFT);
1815+ return truncate_bdev_range(bdev, file->f_mode, start, end);
18181816 }
18191817
18201818 const struct file_operations def_blk_fops = {
fs/btrfs/check-integrity.c+5−5
@@ -2674,7 +2674,7 @@ static void __btrfsic_submit_bio(struct bio *bio) · 2673 unmodified lines
26742674 mutex_lock(&btrfsic_mutex);
26752675 /* since btrfsic_submit_bio() is also called before
26762676 * btrfsic_mount(), this might return NULL */
2677− dev_state = btrfsic_dev_state_lookup(bio_dev(bio) + bio->bi_partno);
2677+ dev_state = btrfsic_dev_state_lookup(bio->bi_bdev->bd_dev);
26782678 if (NULL != dev_state &&
26792679 (bio_op(bio) == REQ_OP_WRITE) && bio_has_data(bio)) {
26802680 unsigned int i = 0;
@@ -2690,9 +2690,9 @@ static void __btrfsic_submit_bio(struct bio *bio) · 9 unmodified lines
26902690 bio_is_patched = 0;
26912691 if (dev_state->state->print_mask &
26922692 BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
2693− pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_disk=%p)\n",
2693+ pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_bdev=%p)\n",
26942694 bio_op(bio), bio->bi_opf, segs,
2695− bio->bi_iter.bi_sector, dev_bytenr, bio->bi_disk);
2695+ bio->bi_iter.bi_sector, dev_bytenr, bio->bi_bdev);
26962696
26972697 mapped_datav = kmalloc_array(segs,
26982698 sizeof(*mapped_datav), GFP_NOFS);
@@ -2721,8 +2721,8 @@ static void __btrfsic_submit_bio(struct bio *bio) · 22 unmodified lines
27212721 } else if (NULL != dev_state && (bio->bi_opf & REQ_PREFLUSH)) {
27222722 if (dev_state->state->print_mask &
27232723 BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
2724− pr_info("submit_bio(rw=%d,0x%x FLUSH, disk=%p)\n",
2725− bio_op(bio), bio->bi_opf, bio->bi_disk);
2724+ pr_info("submit_bio(rw=%d,0x%x FLUSH, bdev=%p)\n",
2725+ bio_op(bio), bio->bi_opf, bio->bi_bdev);
27262726 if (!dev_state->dummy_block_for_bio_bh_flush.is_iodone) {
27272727 if ((dev_state->state->print_mask &
27282728 (BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH |
fs/btrfs/raid56.c+2−5
@@ -1104,8 +1104,7 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio, · 1103 unmodified lines
11041104 * devices or if they are not contiguous
11051105 */
11061106 if (last_end == disk_start && !last->bi_status &&
1107− last->bi_disk == stripe->dev->bdev->bd_disk &&
1108− last->bi_partno == stripe->dev->bdev->bd_partno) {
1107+ last->bi_bdev == stripe->dev->bdev) {
11091108 ret = bio_add_page(last, page, PAGE_SIZE, 0);
11101109 if (ret == PAGE_SIZE)
11111110 return 0;
@@ -1356,9 +1355,7 @@ static int find_bio_stripe(struct btrfs_raid_bio *rbio, · 244 unmodified lines
13561355 for (i = 0; i < rbio->bbio->num_stripes; i++) {
13571356 stripe = &rbio->bbio->stripes[i];
13581357 if (in_range(physical, stripe->physical, rbio->stripe_len) &&
1359− stripe->dev->bdev &&
1360− bio->bi_disk == stripe->dev->bdev->bd_disk &&
1361− bio->bi_partno == stripe->dev->bdev->bd_partno) {
1358+ stripe->dev->bdev && bio->bi_bdev == stripe->dev->bdev) {
13621359 return i;
13631360 }
13641361 }
fs/btrfs/scrub.c+1−1
@@ -1728,7 +1728,7 @@ static void scrub_wr_submit(struct scrub_ctx *sctx) · 1727 unmodified lines
17281728
17291729 sbio = sctx->wr_curr_bio;
17301730 sctx->wr_curr_bio = NULL;
1731− WARN_ON(!sbio->bio->bi_disk);
1731+ WARN_ON(!sbio->bio->bi_bdev);
17321732 scrub_pending_bio_inc(sctx);
17331733 /* process all writes in a single worker thread. Then the block layer
17341734 * orders the requests before sending them to the driver which
fs/btrfs/volumes.c+1−1
@@ -421,7 +421,7 @@ static struct btrfs_device *__alloc_device(struct btrfs_fs_info *fs_info) · 420 unmodified lines
421421 * Preallocate a bio that's always going to be used for flushing device
422422 * barriers and matches the device lifespan
423423 */
424− dev->flush_bio = bio_alloc_bioset(GFP_KERNEL, 0, NULL);
424+ dev->flush_bio = bio_kmalloc(GFP_KERNEL, 0);
425425 if (!dev->flush_bio) {
426426 kfree(dev);
427427 return ERR_PTR(-ENOMEM);
fs/btrfs/zoned.c+2−2
@@ -1275,8 +1275,8 @@ void btrfs_record_physical_zoned(struct inode *inode, u64 file_offset, · 1274 unmodified lines
12751275 return;
12761276
12771277 ordered->physical = physical;
1278− ordered->disk = bio->bi_disk;
1279− ordered->partno = bio->bi_partno;
1278+ ordered->disk = bio->bi_bdev->bd_disk;
1279+ ordered->partno = bio->bi_bdev->bd_partno;
12801280
12811281 btrfs_put_ordered_extent(ordered);
12821282 }
fs/direct-io.c+3−1
@@ -426,6 +426,8 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) · 425 unmodified lines
426426 unsigned long flags;
427427
428428 bio->bi_private = dio;
429+ /* don't account direct I/O as memory stall */
430+ bio_clear_flag(bio, BIO_WORKINGSET);
429431
430432 spin_lock_irqsave(&dio->bio_lock, flags);
431433 dio->refcount++;
@@ -434,7 +436,7 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) · 2 unmodified lines
434436 if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty)
435437 bio_set_pages_dirty(bio);
436438
437− dio->bio_disk = bio->bi_disk;
439+ dio->bio_disk = bio->bi_bdev->bd_disk;
438440
439441 if (sdio->submit_io) {
440442 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio);
fs/exfat/file.c+1−1
@@ -361,7 +361,7 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) · 360 unmodified lines
361361 if (err)
362362 return err;
363363
364− return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
364+ return blkdev_issue_flush(inode->i_sb->s_bdev);
365365 }
366366
367367 const struct file_operations exfat_file_operations = {
fs/ext4/fast_commit.c+2−2
@@ -1076,7 +1076,7 @@ static int ext4_fc_perform_commit(journal_t *journal) · 1075 unmodified lines
10761076 * flush before we start writing fast commit blocks.
10771077 */
10781078 if (journal->j_fs_dev != journal->j_dev)
1079− blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);
1079+ blkdev_issue_flush(journal->j_fs_dev);
10801080
10811081 blk_start_plug(&plug);
10821082 if (sbi->s_fc_bytes == 0) {
@@ -1535,7 +1535,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl) · 452 unmodified lines
15351535 out:
15361536 iput(inode);
15371537 if (!ret)
1538− blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);
1538+ blkdev_issue_flush(sb->s_bdev);
15391539
15401540 return 0;
15411541 }
fs/ext4/fsync.c+1−1
@@ -174,7 +174,7 @@ int ext4_sync_file(struct file *file, loff_t start, loff_t end, int datasync) · 173 unmodified lines
174174 ret = ext4_fsync_journal(inode, datasync, &needs_barrier);
175175
176176 if (needs_barrier) {
177− err = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
177+ err = blkdev_issue_flush(inode->i_sb->s_bdev);
178178 if (!ret)
179179 ret = err;
180180 }
fs/ext4/ialloc.c+1−1
@@ -1583,7 +1583,7 @@ int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, · 1582 unmodified lines
15831583 if (ret < 0)
15841584 goto err_out;
15851585 if (barrier)
1586− blkdev_issue_flush(sb->s_bdev, GFP_NOFS);
1586+ blkdev_issue_flush(sb->s_bdev);
15871587
15881588 skip_zeroout:
15891589 ext4_lock_group(sb, group);
fs/ext4/super.c+1−1
@@ -5709,7 +5709,7 @@ static int ext4_sync_fs(struct super_block *sb, int wait) · 5708 unmodified lines
57095709 needs_barrier = true;
57105710 if (needs_barrier) {
57115711 int err;
5712− err = blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);
5712+ err = blkdev_issue_flush(sb->s_bdev);
57135713 if (!ret)
57145714 ret = err;
57155715 }
fs/f2fs/data.c+5−35
@@ -49,27 +49,6 @@ void f2fs_destroy_bioset(void) · 48 unmodified lines
4949 bioset_exit(&f2fs_bioset);
5050 }
5151
52−static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask,
53− unsigned int nr_iovecs)
54−{
55− return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset);
56−}
57−
58−struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio)
59−{
60− if (noio) {
61− /* No failure on bio allocation */
62− return __f2fs_bio_alloc(GFP_NOIO, npages);
63− }
64−
65− if (time_to_inject(sbi, FAULT_ALLOC_BIO)) {
66− f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO);
67− return NULL;
68− }
69−
70− return __f2fs_bio_alloc(GFP_KERNEL, npages);
71−}
72−
7352 static bool __is_cp_guaranteed(struct page *page)
7453 {
7554 struct address_space *mapping = page->mapping;
@@ -398,22 +377,12 @@ int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) · 322 unmodified lines
398377 return 0;
399378 }
400379
401−/*
402− * Return true, if pre_bio's bdev is same as its target device.
403− */
404−static bool __same_bdev(struct f2fs_sb_info *sbi,
405− block_t blk_addr, struct bio *bio)
406−{
407− struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
408− return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
409−}
410−
411380 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
412381 {
413382 struct f2fs_sb_info *sbi = fio->sbi;
414383 struct bio *bio;
415384
416− bio = f2fs_bio_alloc(sbi, npages, true);
385+ bio = bio_alloc_bioset(GFP_NOIO, npages, &f2fs_bioset);
417386
418387 f2fs_target_device(sbi, fio->new_blkaddr, bio);
419388 if (is_read_io(fio->op)) {
@@ -711,7 +680,7 @@ static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, · 291 unmodified lines
711680 return false;
712681 if (last_blkaddr + 1 != cur_blkaddr)
713682 return false;
714− return __same_bdev(sbi, cur_blkaddr, bio);
683+ return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL);
715684 }
716685
717686 static bool io_type_is_mergeable(struct f2fs_bio_info *io,
@@ -999,8 +968,9 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, · 281 unmodified lines
999968 struct bio_post_read_ctx *ctx;
1000969 unsigned int post_read_steps = 0;
1001970
1002− bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES),
1003− for_write);
971+ bio = bio_alloc_bioset(for_write ? GFP_NOIO : GFP_KERNEL,
972+ min_t(int, nr_pages, BIO_MAX_PAGES),
973+ &f2fs_bioset);
1004974 if (!bio)
1005975 return ERR_PTR(-ENOMEM);
1006976
fs/f2fs/f2fs.h+0−2
@@ -43,7 +43,6 @@ enum { · 42 unmodified lines
4343 FAULT_KVMALLOC,
4444 FAULT_PAGE_ALLOC,
4545 FAULT_PAGE_GET,
46− FAULT_ALLOC_BIO,
4746 FAULT_ALLOC_NID,
4847 FAULT_ORPHAN,
4948 FAULT_BLOCK,
@@ -3482,7 +3481,6 @@ void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi); · 3432 unmodified lines
34823481 */
34833482 int __init f2fs_init_bioset(void);
34843483 void f2fs_destroy_bioset(void);
3485−struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio);
34863484 int f2fs_init_bio_entry_cache(void);
34873485 void f2fs_destroy_bio_entry_cache(void);
34883486 void f2fs_submit_bio(struct f2fs_sb_info *sbi,
fs/f2fs/segment.c+1−11
@@ -563,17 +563,7 @@ do_sync: · 562 unmodified lines
563563 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
564564 struct block_device *bdev)
565565 {
566− struct bio *bio;
567− int ret;
568−
569− bio = f2fs_bio_alloc(sbi, 0, false);
570− if (!bio)
571− return -ENOMEM;
572−
573− bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH;
574− bio_set_dev(bio, bdev);
575− ret = submit_bio_wait(bio);
576− bio_put(bio);
566+ int ret = blkdev_issue_flush(bdev);
577567
578568 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
579569 test_opt(sbi, FLUSH_MERGE), ret);
fs/f2fs/super.c+0−1
@@ -46,7 +46,6 @@ const char *f2fs_fault_name[FAULT_MAX] = { · 45 unmodified lines
4646 [FAULT_KVMALLOC] = "kvmalloc",
4747 [FAULT_PAGE_ALLOC] = "page alloc",
4848 [FAULT_PAGE_GET] = "page get",
49− [FAULT_ALLOC_BIO] = "alloc bio",
5049 [FAULT_ALLOC_NID] = "alloc nid",
5150 [FAULT_ORPHAN] = "orphan",
5251 [FAULT_BLOCK] = "no more block",
fs/fat/file.c+1−1
@@ -195,7 +195,7 @@ int fat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) · 194 unmodified lines
195195 if (err)
196196 return err;
197197
198− return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
198+ return blkdev_issue_flush(inode->i_sb->s_bdev);
199199 }
200200
201201
fs/hfsplus/inode.c+1−1
@@ -340,7 +340,7 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, · 339 unmodified lines
340340 }
341341
342342 if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags))
343− blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
343+ blkdev_issue_flush(inode->i_sb->s_bdev);
344344
345345 inode_unlock(inode);
346346
fs/hfsplus/super.c+1−1
@@ -239,7 +239,7 @@ out: · 238 unmodified lines
239239 mutex_unlock(&sbi->vh_mutex);
240240
241241 if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags))
242− blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);
242+ blkdev_issue_flush(sb->s_bdev);
243243
244244 return error;
245245 }
fs/iomap/direct-io.c+4−5
@@ -279,11 +279,8 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, · 278 unmodified lines
279279 orig_count = iov_iter_count(dio->submit.iter);
280280 iov_iter_truncate(dio->submit.iter, length);
281281
282− nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES);
283− if (nr_pages <= 0) {
284− ret = nr_pages;
282+ if (!iov_iter_count(dio->submit.iter))
285283 goto out;
286− }
287284
288285 if (need_zeroout) {
289286 /* zero out from the start of the block to the write offset */
@@ -299,6 +296,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, · 9 unmodified lines
299296 */
300297 bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua);
301298
299+ nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_PAGES);
302300 do {
303301 size_t n;
304302 if (dio->error) {
@@ -339,7 +337,8 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, · 34 unmodified lines
339337 dio->size += n;
340338 copied += n;
341339
342− nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES);
340+ nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter,
341+ BIO_MAX_PAGES);
343342 iomap_dio_submit_bio(dio, iomap, bio, pos);
344343 pos += n;
345344 } while (nr_pages);
fs/jbd2/checkpoint.c+1−1
@@ -416,7 +416,7 @@ int jbd2_cleanup_journal_tail(journal_t *journal) · 415 unmodified lines
416416 * jbd2_cleanup_journal_tail() doesn't get called all that often.
417417 */
418418 if (journal->j_flags & JBD2_BARRIER)
419− blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);
419+ blkdev_issue_flush(journal->j_fs_dev);
420420
421421 return __jbd2_update_log_tail(journal, first_tid, blocknr);
422422 }
fs/jbd2/commit.c+2−2
@@ -825,7 +825,7 @@ start_journal_io: · 824 unmodified lines
825825 if (commit_transaction->t_need_data_flush &&
826826 (journal->j_fs_dev != journal->j_dev) &&
827827 (journal->j_flags & JBD2_BARRIER))
828− blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);
828+ blkdev_issue_flush(journal->j_fs_dev);
829829
830830 /* Done it all: now write the commit record asynchronously. */
831831 if (jbd2_has_feature_async_commit(journal)) {
@@ -932,7 +932,7 @@ start_journal_io: · 100 unmodified lines
932932 stats.run.rs_blocks_logged++;
933933 if (jbd2_has_feature_async_commit(journal) &&
934934 journal->j_flags & JBD2_BARRIER) {
935− blkdev_issue_flush(journal->j_dev, GFP_NOFS);
935+ blkdev_issue_flush(journal->j_dev);
936936 }
937937
938938 if (err)
fs/jbd2/recovery.c+1−1
@@ -326,7 +326,7 @@ int jbd2_journal_recover(journal_t *journal) · 325 unmodified lines
326326 err = err2;
327327 /* Make sure all replayed data is on permanent storage */
328328 if (journal->j_flags & JBD2_BARRIER) {
329− err2 = blkdev_issue_flush(journal->j_fs_dev, GFP_KERNEL);
329+ err2 = blkdev_issue_flush(journal->j_fs_dev);
330330 if (!err)
331331 err = err2;
332332 }
fs/libfs.c+1−1
@@ -1117,7 +1117,7 @@ int generic_file_fsync(struct file *file, loff_t start, loff_t end, · 1116 unmodified lines
11171117 err = __generic_file_fsync(file, start, end, datasync);
11181118 if (err)
11191119 return err;
1120− return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
1120+ return blkdev_issue_flush(inode->i_sb->s_bdev);
11211121 }
11221122 EXPORT_SYMBOL(generic_file_fsync);
11231123
fs/nfs/blocklayout/blocklayout.c+0−5
@@ -123,11 +123,6 @@ bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector, · 122 unmodified lines
123123
124124 npg = min(npg, BIO_MAX_PAGES);
125125 bio = bio_alloc(GFP_NOIO, npg);
126− if (!bio && (current->flags & PF_MEMALLOC)) {
127− while (!bio && (npg /= 2))
128− bio = bio_alloc(GFP_NOIO, npg);
129− }
130−
131126 if (bio) {
132127 bio->bi_iter.bi_sector = disk_sector;
133128 bio_set_dev(bio, bdev);
fs/nfsd/blocklayout.c+1−1
@@ -254,7 +254,7 @@ again: · 253 unmodified lines
254254 req->cmd[4] = bufflen & 0xff;
255255 req->cmd_len = COMMAND_SIZE(INQUIRY);
256256
257− blk_execute_rq(rq->q, NULL, rq, 1);
257+ blk_execute_rq(NULL, rq, 1);
258258 if (req->result) {
259259 pr_err("pNFS: INQUIRY 0x83 failed with: %x\n",
260260 req->result);
fs/nilfs2/segbuf.c+0−4
@@ -386,10 +386,6 @@ static struct bio *nilfs_alloc_seg_bio(struct the_nilfs *nilfs, sector_t start, · 385 unmodified lines
386386 struct bio *bio;
387387
388388 bio = bio_alloc(GFP_NOIO, nr_vecs);
389− if (bio == NULL) {
390− while (!bio && (nr_vecs >>= 1))
391− bio = bio_alloc(GFP_NOIO, nr_vecs);
392− }
393389 if (likely(bio)) {
394390 bio_set_dev(bio, nilfs->ns_bdev);
395391 bio->bi_iter.bi_sector =
fs/nilfs2/the_nilfs.h+1−1
@@ -375,7 +375,7 @@ static inline int nilfs_flush_device(struct the_nilfs *nilfs) · 374 unmodified lines
375375 */
376376 smp_wmb();
377377
378− err = blkdev_issue_flush(nilfs->ns_bdev, GFP_KERNEL);
378+ err = blkdev_issue_flush(nilfs->ns_bdev);
379379 if (err != -EIO)
380380 err = 0;
381381 return err;
fs/ocfs2/file.c+1−1
@@ -194,7 +194,7 @@ static int ocfs2_sync_file(struct file *file, loff_t start, loff_t end, · 193 unmodified lines
194194 needs_barrier = true;
195195 err = jbd2_complete_transaction(journal, commit_tid);
196196 if (needs_barrier) {
197− ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
197+ ret = blkdev_issue_flush(inode->i_sb->s_bdev);
198198 if (!err)
199199 err = ret;
200200 }
fs/reiserfs/file.c+1−1
@@ -159,7 +159,7 @@ static int reiserfs_sync_file(struct file *filp, loff_t start, loff_t end, · 158 unmodified lines
159159 barrier_done = reiserfs_commit_for_inode(inode);
160160 reiserfs_write_unlock(inode->i_sb);
161161 if (barrier_done != 1 && reiserfs_barrier_flush(inode->i_sb))
162− blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
162+ blkdev_issue_flush(inode->i_sb->s_bdev);
163163 inode_unlock(inode);
164164 if (barrier_done < 0)
165165 return barrier_done;
fs/splice.c+6−3
@@ -662,12 +662,14 @@ iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, · 661 unmodified lines
662662
663663 /* build the vector */
664664 left = sd.total_len;
665− for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) {
665+ for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
666666 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
667667 size_t this_len = buf->len;
668668
669− if (this_len > left)
670− this_len = left;
669+ /* zero-length bvecs are not supported, skip them */
670+ if (!this_len)
671+ continue;
672+ this_len = min(this_len, left);
671673
672674 ret = pipe_buf_confirm(pipe, buf);
673675 if (unlikely(ret)) {
@@ -680,6 +682,7 @@ iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, · 6 unmodified lines
680682 array[n].bv_len = this_len;
681683 array[n].bv_offset = buf->offset;
682684 left -= this_len;
685+ n++;
683686 }
684687
685688 iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left);
fs/super.c+2−1
@@ -865,7 +865,8 @@ int reconfigure_super(struct fs_context *fc) · 864 unmodified lines
865865
866866 if (fc->sb_flags_mask & SB_RDONLY) {
867867 #ifdef CONFIG_BLOCK
868− if (!(fc->sb_flags & SB_RDONLY) && bdev_read_only(sb->s_bdev))
868+ if (!(fc->sb_flags & SB_RDONLY) && sb->s_bdev &&
869+ bdev_read_only(sb->s_bdev))
869870 return -EACCES;
870871 #endif
871872
fs/xfs/xfs_super.c+1−1
@@ -343,7 +343,7 @@ void · 342 unmodified lines
343343 xfs_blkdev_issue_flush(
344344 xfs_buftarg_t *buftarg)
345345 {
346− blkdev_issue_flush(buftarg->bt_bdev, GFP_NOFS);
346+ blkdev_issue_flush(buftarg->bt_bdev);
347347 }
348348
349349 STATIC void
fs/zonefs/super.c+6−7
@@ -541,7 +541,7 @@ static int zonefs_file_fsync(struct file *file, loff_t start, loff_t end, · 540 unmodified lines
541541 if (ZONEFS_I(inode)->i_ztype == ZONEFS_ZTYPE_CNV)
542542 ret = file_write_and_wait_range(file, start, end);
543543 if (!ret)
544− ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
544+ ret = blkdev_issue_flush(inode->i_sb->s_bdev);
545545
546546 if (ret)
547547 zonefs_io_error(inode, true);
@@ -678,7 +678,7 @@ static ssize_t zonefs_file_dio_append(struct kiocb *iocb, struct iov_iter *from) · 130 unmodified lines
678678 if (!nr_pages)
679679 return 0;
680680
681− bio = bio_alloc_bioset(GFP_NOFS, nr_pages, &fs_bio_set);
681+ bio = bio_alloc(GFP_NOFS, nr_pages);
682682 if (!bio)
683683 return -ENOMEM;
684684
@@ -1581,12 +1581,11 @@ static int zonefs_fill_super(struct super_block *sb, void *data, int silent) · 896 unmodified lines
15811581 sb->s_time_gran = 1;
15821582
15831583 /*
1584− * The block size is set to the device physical sector size to ensure
1585− * that write operations on 512e devices (512B logical block and 4KB
1586− * physical block) are always aligned to the device physical blocks,
1587− * as mandated by the ZBC/ZAC specifications.
1584+ * The block size is set to the device zone write granularity to ensure
1585+ * that write operations are always aligned according to the device
1586+ * interface constraints.
15881587 */
1589− sb_set_blocksize(sb, bdev_physical_block_size(sb->s_bdev));
1588+ sb_set_blocksize(sb, bdev_zone_write_granularity(sb->s_bdev));
15901589 sbi->s_zone_sectors_shift = ilog2(bdev_zone_sectors(sb->s_bdev));
15911590 sbi->s_uid = GLOBAL_ROOT_UID;
15921591 sbi->s_gid = GLOBAL_ROOT_GID;
include/linux/bio.h+28−27
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010 #include <linux/ioprio.h>
1111 /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
1212 #include <linux/blk_types.h>
13+#include <linux/uio.h>
1314
1415 #define BIO_DEBUG
1516
@@ -328,7 +329,6 @@ struct bio_integrity_payload { · 312 unmodified lines
328329
329330 struct bvec_iter bip_iter;
330331
331− unsigned short bip_slab; /* slab the bip came from */
332332 unsigned short bip_vcnt; /* # of integrity bio_vecs */
333333 unsigned short bip_max_vcnt; /* integrity bio_vec slots */
334334 unsigned short bip_flags; /* control flags */
@@ -406,7 +406,9 @@ extern void bioset_exit(struct bio_set *); · 71 unmodified lines
406406 extern int biovec_init_pool(mempool_t *pool, int pool_entries);
407407 extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);
408408
409−extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);
409+struct bio *bio_alloc_bioset(gfp_t gfp, unsigned short nr_iovecs,
410+ struct bio_set *bs);
411+struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs);
410412 extern void bio_put(struct bio *);
411413
412414 extern void __bio_clone_fast(struct bio *, struct bio *);
@@ -414,16 +416,11 @@ extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *); · 1 unmodified line
414416
415417 extern struct bio_set fs_bio_set;
416418
417−static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
419+static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned short nr_iovecs)
418420 {
419421 return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set);
420422 }
421423
422−static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
423−{
424− return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
425−}
426−
427424 extern blk_qc_t submit_bio(struct bio *);
428425
429426 extern void bio_endio(struct bio *);
@@ -441,6 +438,18 @@ static inline void bio_wouldblock_error(struct bio *bio) · 11 unmodified lines
441438 bio_endio(bio);
442439 }
443440
441+/*
442+ * Calculate number of bvec segments that should be allocated to fit data
443+ * pointed by @iter. If @iter is backed by bvec it's going to be reused
444+ * instead of allocating a new one.
445+ */
446+static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
447+{
448+ if (iov_iter_is_bvec(iter))
449+ return 0;
450+ return iov_iter_npages(iter, max_segs);
451+}
452+
444453 struct request_queue;
445454
446455 extern int submit_bio_wait(struct bio *bio);
@@ -480,29 +489,26 @@ static inline void zero_fill_bio(struct bio *bio) · 33 unmodified lines
480489 zero_fill_bio_iter(bio, bio->bi_iter);
481490 }
482491
483−extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
484−extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
485−extern unsigned int bvec_nr_vecs(unsigned short idx);
486492 extern const char *bio_devname(struct bio *bio, char *buffer);
487493
488−#define bio_set_dev(bio, bdev) \
489−do { \
490− if ((bio)->bi_disk != (bdev)->bd_disk) \
491− bio_clear_flag(bio, BIO_THROTTLED);\
492− (bio)->bi_disk = (bdev)->bd_disk; \
493− (bio)->bi_partno = (bdev)->bd_partno; \
494− bio_associate_blkg(bio); \
494+#define bio_set_dev(bio, bdev) \
495+do { \
496+ bio_clear_flag(bio, BIO_REMAPPED); \
497+ if ((bio)->bi_bdev != (bdev)) \
498+ bio_clear_flag(bio, BIO_THROTTLED); \
499+ (bio)->bi_bdev = (bdev); \
500+ bio_associate_blkg(bio); \
495501 } while (0)
496502
497503 #define bio_copy_dev(dst, src) \
498504 do { \
499− (dst)->bi_disk = (src)->bi_disk; \
500− (dst)->bi_partno = (src)->bi_partno; \
505+ bio_clear_flag(dst, BIO_REMAPPED); \
506+ (dst)->bi_bdev = (src)->bi_bdev; \
501507 bio_clone_blkg_association(dst, src); \
502508 } while (0)
503509
504510 #define bio_dev(bio) \
505− disk_devt((bio)->bi_disk)
511+ disk_devt((bio)->bi_bdev->bd_disk)
506512
507513 #ifdef CONFIG_BLK_CGROUP
508514 void bio_associate_blkg(struct bio *bio);
@@ -705,6 +711,7 @@ struct bio_set { · 196 unmodified lines
705711 mempool_t bvec_integrity_pool;
706712 #endif
707713
714+ unsigned int back_pad;
708715 /*
709716 * Deadlock avoidance for stacking block drivers: see comments in
710717 * bio_alloc_bioset() for details
@@ -715,12 +722,6 @@ struct bio_set { · 4 unmodified lines
715722 struct workqueue_struct *rescue_workqueue;
716723 };
717724
718−struct biovec_slab {
719− int nr_vecs;
720− char *name;
721− struct kmem_cache *slab;
722−};
723−
724725 static inline bool bioset_initialized(struct bio_set *bs)
725726 {
726727 return bs->bio_slab != NULL;
include/linux/blk-mq.h+2−6
@@ -140,10 +140,6 @@ struct blk_mq_hw_ctx { · 139 unmodified lines
140140 * shared across request queues.
141141 */
142142 atomic_t nr_active;
143− /**
144− * @elevator_queued: Number of queued requests on hctx.
145− */
146− atomic_t elevator_queued;
147143
148144 /** @cpuhp_online: List to store request if CPU is going to die */
149145 struct hlist_node cpuhp_online;
@@ -602,8 +598,8 @@ static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio, · 452 unmodified lines
602598 rq->bio = rq->biotail = bio;
603599 rq->ioprio = bio_prio(bio);
604600
605− if (bio->bi_disk)
606− rq->rq_disk = bio->bi_disk;
601+ if (bio->bi_bdev)
602+ rq->rq_disk = bio->bi_bdev->bd_disk;
607603 }
608604
609605 blk_qc_t blk_mq_submit_bio(struct bio *bio);
include/linux/blk_types.h+3−30
@@ -222,16 +222,15 @@ static inline void bio_issue_init(struct bio_issue *issue, · 221 unmodified lines
222222 */
223223 struct bio {
224224 struct bio *bi_next; /* request queue link */
225− struct gendisk *bi_disk;
225+ struct block_device *bi_bdev;
226226 unsigned int bi_opf; /* bottom bits req flags,
227227 * top bits REQ_OP. Use
228228 * accessors.
229229 */
230− unsigned short bi_flags; /* status, etc and bvec pool number */
230+ unsigned short bi_flags; /* BIO_* below */
231231 unsigned short bi_ioprio;
232232 unsigned short bi_write_hint;
233233 blk_status_t bi_status;
234− u8 bi_partno;
235234 atomic_t __bi_remaining;
236235
237236 struct bvec_iter bi_iter;
@@ -304,36 +303,10 @@ enum { · 66 unmodified lines
304303 * of this bio. */
305304 BIO_CGROUP_ACCT, /* has been accounted to a cgroup */
306305 BIO_TRACKED, /* set if bio goes through the rq_qos path */
306+ BIO_REMAPPED,
307307 BIO_FLAG_LAST
308308 };
309309
310−/* See BVEC_POOL_OFFSET below before adding new flags */
311−
312−/*
313− * We support 6 different bvec pools, the last one is magic in that it
314− * is backed by a mempool.
315− */
316−#define BVEC_POOL_NR 6
317−#define BVEC_POOL_MAX (BVEC_POOL_NR - 1)
318−
319−/*
320− * Top 3 bits of bio flags indicate the pool the bvecs came from. We add
321− * 1 to the actual index so that 0 indicates that there are no bvecs to be
322− * freed.
323− */
324−#define BVEC_POOL_BITS (3)
325−#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS)
326−#define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET)
327−#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)
328−# error "BVEC_POOL_BITS is too small"
329−#endif
330−
331−/*
332− * Flags starting here get preserved by bio_reset() - this includes
333− * only BVEC_POOL_IDX()
334− */
335−#define BIO_RESET_BITS BVEC_POOL_OFFSET
336−
337310 typedef __u32 __bitwise blk_mq_req_flags_t;
338311
339312 /*
include/linux/blkdev.h+25−28
@@ -337,6 +337,7 @@ struct queue_limits { · 336 unmodified lines
337337 unsigned int max_zone_append_sectors;
338338 unsigned int discard_granularity;
339339 unsigned int discard_alignment;
340+ unsigned int zone_write_granularity;
340341
341342 unsigned short max_segments;
342343 unsigned short max_integrity_segments;
@@ -948,9 +949,8 @@ extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, uns · 605 unmodified lines
948949 extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
949950 struct rq_map_data *, const struct iov_iter *,
950951 gfp_t);
951−extern void blk_execute_rq(struct request_queue *, struct gendisk *,
952− struct request *, int);
953−extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
952+extern void blk_execute_rq(struct gendisk *, struct request *, int);
953+extern void blk_execute_rq_nowait(struct gendisk *,
954954 struct request *, int, rq_end_io_fn *);
955955
956956 /* Helper to convert REQ_OP_XXX to its string format XXX */
@@ -1161,6 +1161,8 @@ extern void blk_queue_logical_block_size(struct request_queue *, unsigned int); · 204 unmodified lines
11611161 extern void blk_queue_max_zone_append_sectors(struct request_queue *q,
11621162 unsigned int max_zone_append_sectors);
11631163 extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1164+void blk_queue_zone_write_granularity(struct request_queue *q,
1165+ unsigned int size);
11641166 extern void blk_queue_alignment_offset(struct request_queue *q,
11651167 unsigned int alignment);
11661168 void blk_queue_update_readahead(struct request_queue *q);
@@ -1289,7 +1291,7 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) · 122 unmodified lines
12891291 !list_empty(&plug->cb_list));
12901292 }
12911293
1292−int blkdev_issue_flush(struct block_device *, gfp_t);
1294+int blkdev_issue_flush(struct block_device *bdev);
12931295 long nr_blockdev_pages(void);
12941296 #else /* CONFIG_BLOCK */
12951297 struct blk_plug {
@@ -1317,7 +1319,7 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) · 21 unmodified lines
13171319 return false;
13181320 }
13191321
1320−static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask)
1322+static inline int blkdev_issue_flush(struct block_device *bdev)
13211323 {
13221324 return 0;
13231325 }
@@ -1474,6 +1476,18 @@ static inline int bdev_io_opt(struct block_device *bdev) · 150 unmodified lines
14741476 return queue_io_opt(bdev_get_queue(bdev));
14751477 }
14761478
1479+static inline unsigned int
1480+queue_zone_write_granularity(const struct request_queue *q)
1481+{
1482+ return q->limits.zone_write_granularity;
1483+}
1484+
1485+static inline unsigned int
1486+bdev_zone_write_granularity(struct block_device *bdev)
1487+{
1488+ return queue_zone_write_granularity(bdev_get_queue(bdev));
1489+}
1490+
14771491 static inline int queue_alignment_offset(const struct request_queue *q)
14781492 {
14791493 if (q->limits.misaligned)
@@ -1954,21 +1968,9 @@ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, · 474 unmodified lines
19541968 void disk_end_io_acct(struct gendisk *disk, unsigned int op,
19551969 unsigned long start_time);
19561970
1957−unsigned long part_start_io_acct(struct gendisk *disk,
1958− struct block_device **part, struct bio *bio);
1959−void part_end_io_acct(struct block_device *part, struct bio *bio,
1960− unsigned long start_time);
1961−
1962−/**
1963− * bio_start_io_acct - start I/O accounting for bio based drivers
1964− * @bio: bio to start account for
1965− *
1966− * Returns the start time that should be passed back to bio_end_io_acct().
1967− */
1968−static inline unsigned long bio_start_io_acct(struct bio *bio)
1969−{
1970− return disk_start_io_acct(bio->bi_disk, bio_sectors(bio), bio_op(bio));
1971−}
1971+unsigned long bio_start_io_acct(struct bio *bio);
1972+void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1973+ struct block_device *orig_bdev);
19721974
19731975 /**
19741976 * bio_end_io_acct - end I/O accounting for bio based drivers
@@ -1977,7 +1979,7 @@ static inline unsigned long bio_start_io_acct(struct bio *bio) · 2 unmodified lines
19771979 */
19781980 static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
19791981 {
1980− return disk_end_io_acct(bio->bi_disk, bio_op(bio), start_time);
1982+ return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
19811983 }
19821984
19831985 int bdev_read_only(struct block_device *bdev);
@@ -2012,21 +2014,16 @@ void bdev_add(struct block_device *bdev, dev_t dev); · 28 unmodified lines
20122014 struct block_device *I_BDEV(struct inode *inode);
20132015 struct block_device *bdgrab(struct block_device *bdev);
20142016 void bdput(struct block_device *);
2017+int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
2018+ loff_t lend);
20152019
20162020 #ifdef CONFIG_BLOCK
20172021 void invalidate_bdev(struct block_device *bdev);
2018−int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
2019− loff_t lend);
20202022 int sync_blockdev(struct block_device *bdev);
20212023 #else
20222024 static inline void invalidate_bdev(struct block_device *bdev)
20232025 {
20242026 }
2025−static inline int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
2026− loff_t lstart, loff_t lend)
2027−{
2028− return 0;
2029−}
20302027 static inline int sync_blockdev(struct block_device *bdev)
20312028 {
20322029 return 0;
include/linux/elevator.h+2−0
@@ -172,6 +172,8 @@ extern struct request *elv_rb_find(struct rb_root *, sector_t); · 171 unmodified lines
172172
173173 /* Supports zoned block devices sequential write constraint */
174174 #define ELEVATOR_F_ZBD_SEQ_WRITE (1U << 0)
175+/* Supports scheduling on multiple hardware queues */
176+#define ELEVATOR_F_MQ_AWARE (1U << 1)
175177
176178 #endif /* CONFIG_BLOCK */
177179 #endif
include/linux/genhd.h+9−18
@@ -32,6 +32,7 @@ extern struct class block_class; · 31 unmodified lines
3232 #include <linux/string.h>
3333 #include <linux/fs.h>
3434 #include <linux/workqueue.h>
35+#include <linux/xarray.h>
3536
3637 #define PARTITION_META_INFO_VOLNAMELTH 64
3738 /*
@@ -116,13 +117,6 @@ enum { · 78 unmodified lines
116117 DISK_EVENT_FLAG_UEVENT = 1 << 1,
117118 };
118119
119−struct disk_part_tbl {
120− struct rcu_head rcu_head;
121− int len;
122− struct block_device __rcu *last_lookup;
123− struct block_device __rcu *part[];
124−};
125−
126120 struct disk_events;
127121 struct badblocks;
128122
@@ -148,12 +142,7 @@ struct gendisk { · 19 unmodified lines
148142 unsigned short events; /* supported events */
149143 unsigned short event_flags; /* flags related to event processing */
150144
151− /* Array of pointers to partitions indexed by partno.
152− * Protected with matching bdev lock but stat and other
153− * non-critical accesses use RCU. Always access through
154− * helpers.
155− */
156− struct disk_part_tbl __rcu *part_tbl;
145+ struct xarray part_tbl;
157146 struct block_device *part0;
158147
159148 const struct block_device_operations *fops;
@@ -163,6 +152,7 @@ struct gendisk { · 3 unmodified lines
163152 int flags;
164153 unsigned long state;
165154 #define GD_NEED_PART_SCAN 0
155+#define GD_READ_ONLY 1
166156 struct kobject *slave_dir;
167157
168158 struct timer_rand_state *random;
@@ -212,10 +202,11 @@ static inline dev_t disk_devt(struct gendisk *disk) · 43 unmodified lines
212202 return MKDEV(disk->major, disk->first_minor);
213203 }
214204
205+void disk_uevent(struct gendisk *disk, enum kobject_action action);
206+
215207 /*
216208 * Smarter partition iterator without context limits.
217209 */
218−#define DISK_PITER_REVERSE (1 << 0) /* iterate in the reverse direction */
219210 #define DISK_PITER_INCL_EMPTY (1 << 1) /* include 0-sized parts */
220211 #define DISK_PITER_INCL_PART0 (1 << 2) /* include partition 0 */
221212 #define DISK_PITER_INCL_EMPTY_PART0 (1 << 3) /* include empty partition 0 */
@@ -223,7 +214,7 @@ static inline dev_t disk_devt(struct gendisk *disk) · 1 unmodified line
223214 struct disk_part_iter {
224215 struct gendisk *disk;
225216 struct block_device *part;
226− int idx;
217+ unsigned long idx;
227218 unsigned int flags;
228219 };
229220
@@ -231,7 +222,6 @@ extern void disk_part_iter_init(struct disk_part_iter *piter, · 1 unmodified line
231222 struct gendisk *disk, unsigned int flags);
232223 struct block_device *disk_part_iter_next(struct disk_part_iter *piter);
233224 extern void disk_part_iter_exit(struct disk_part_iter *piter);
234−extern bool disk_has_partitions(struct gendisk *disk);
235225
236226 /* block/genhd.c */
237227 extern void device_add_disk(struct device *parent, struct gendisk *disk,
@@ -249,11 +239,12 @@ static inline void add_disk_no_queue_reg(struct gendisk *disk) · 11 unmodified lines
249239 extern void del_gendisk(struct gendisk *gp);
250240 extern struct block_device *bdget_disk(struct gendisk *disk, int partno);
251241
252−extern void set_disk_ro(struct gendisk *disk, int flag);
242+void set_disk_ro(struct gendisk *disk, bool read_only);
253243
254244 static inline int get_disk_ro(struct gendisk *disk)
255245 {
256− return disk->part0->bd_read_only;
246+ return disk->part0->bd_read_only ||
247+ test_bit(GD_READ_ONLY, &disk->state);
257248 }
258249
259250 extern void disk_block_events(struct gendisk *disk);
include/linux/swap.h+0−1
@@ -468,7 +468,6 @@ extern int free_swap_and_cache(swp_entry_t); · 467 unmodified lines
468468 int swap_type_of(dev_t device, sector_t offset);
469469 int find_first_swap(dev_t *device);
470470 extern unsigned int count_swap_pages(int, int);
471−extern sector_t map_swap_page(struct page *, struct block_device **);
472471 extern sector_t swapdev_block(int, pgoff_t);
473472 extern int page_swapcount(struct page *);
474473 extern int __swap_count(swp_entry_t entry);
kernel/trace/blktrace.c+9−7
@@ -903,7 +903,7 @@ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, · 902 unmodified lines
903903
904904 static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio)
905905 {
906− blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BOUNCE, 0);
906+ blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BOUNCE, 0);
907907 }
908908
909909 static void blk_add_trace_bio_complete(void *ignore,
@@ -915,22 +915,24 @@ static void blk_add_trace_bio_complete(void *ignore, · 5 unmodified lines
915915
916916 static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio)
917917 {
918− blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BACKMERGE, 0);
918+ blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE,
919+ 0);
919920 }
920921
921922 static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio)
922923 {
923− blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_FRONTMERGE, 0);
924+ blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE,
925+ 0);
924926 }
925927
926928 static void blk_add_trace_bio_queue(void *ignore, struct bio *bio)
927929 {
928− blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_QUEUE, 0);
930+ blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0);
929931 }
930932
931933 static void blk_add_trace_getrq(void *ignore, struct bio *bio)
932934 {
933− blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_GETRQ, 0);
935+ blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0);
934936 }
935937
936938 static void blk_add_trace_plug(void *ignore, struct request_queue *q)
@@ -967,7 +969,7 @@ static void blk_add_trace_unplug(void *ignore, struct request_queue *q, · 30 unmodified lines
967969
968970 static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu)
969971 {
970− struct request_queue *q = bio->bi_disk->queue;
972+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
971973 struct blk_trace *bt;
972974
973975 rcu_read_lock();
@@ -997,7 +999,7 @@ static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) · 23 unmodified lines
997999 static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev,
9981000 sector_t from)
9991001 {
1000− struct request_queue *q = bio->bi_disk->queue;
1002+ struct request_queue *q = bio->bi_bdev->bd_disk->queue;
10011003 struct blk_trace *bt;
10021004 struct blk_io_trace_remap r;
10031005
lib/iov_iter.c+19−2
@@ -72,8 +72,6 @@ · 71 unmodified lines
7272 __start.bi_bvec_done = skip; \
7373 __start.bi_idx = 0; \
7474 for_each_bvec(__v, i->bvec, __bi, __start) { \
75− if (!__v.bv_len) \
76− continue; \
7775 (void)(STEP); \
7876 } \
7977 }
@@ -1069,6 +1067,21 @@ static void pipe_advance(struct iov_iter *i, size_t size) · 989 unmodified lines
10691067 pipe_truncate(i);
10701068 }
10711069
1070+static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
1071+{
1072+ struct bvec_iter bi;
1073+
1074+ bi.bi_size = i->count;
1075+ bi.bi_bvec_done = i->iov_offset;
1076+ bi.bi_idx = 0;
1077+ bvec_iter_advance(i->bvec, &bi, size);
1078+
1079+ i->bvec += bi.bi_idx;
1080+ i->nr_segs -= bi.bi_idx;
1081+ i->count = bi.bi_size;
1082+ i->iov_offset = bi.bi_bvec_done;
1083+}
1084+
10721085 void iov_iter_advance(struct iov_iter *i, size_t size)
10731086 {
10741087 if (unlikely(iov_iter_is_pipe(i))) {
@@ -1079,6 +1092,10 @@ void iov_iter_advance(struct iov_iter *i, size_t size) · 4 unmodified lines
10791092 i->count -= size;
10801093 return;
10811094 }
1095+ if (iov_iter_is_bvec(i)) {
1096+ iov_iter_bvec_advance(i, size);
1097+ return;
1098+ }
10821099 iterate_and_advance(i, size, v, 0, 0, 0)
10831100 }
10841101 EXPORT_SYMBOL(iov_iter_advance);
mm/page_io.c+14−33
@@ -26,25 +26,6 @@ · 25 unmodified lines
2626 #include <linux/uio.h>
2727 #include <linux/sched/task.h>
2828
29−static struct bio *get_swap_bio(gfp_t gfp_flags,
30− struct page *page, bio_end_io_t end_io)
31−{
32− struct bio *bio;
33−
34− bio = bio_alloc(gfp_flags, 1);
35− if (bio) {
36− struct block_device *bdev;
37−
38− bio->bi_iter.bi_sector = map_swap_page(page, &bdev);
39− bio_set_dev(bio, bdev);
40− bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9;
41− bio->bi_end_io = end_io;
42−
43− bio_add_page(bio, page, thp_size(page), 0);
44− }
45− return bio;
46−}
47−
4829 void end_swap_bio_write(struct bio *bio)
4930 {
5031 struct page *page = bio_first_page_all(bio);
@@ -361,13 +342,13 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc, · 310 unmodified lines
361342 return 0;
362343 }
363344
364− bio = get_swap_bio(GFP_NOIO, page, end_write_func);
365− if (bio == NULL) {
366− set_page_dirty(page);
367− unlock_page(page);
368− return -ENOMEM;
369− }
345+ bio = bio_alloc(GFP_NOIO, 1);
346+ bio_set_dev(bio, sis->bdev);
347+ bio->bi_iter.bi_sector = swap_page_sector(page);
370348 bio->bi_opf = REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc);
349+ bio->bi_end_io = end_write_func;
350+ bio_add_page(bio, page, thp_size(page), 0);
351+
371352 bio_associate_blkg_from_page(bio, page);
372353 count_swpout_vm_event(page);
373354 set_page_writeback(page);
@@ -427,18 +408,18 @@ int swap_readpage(struct page *page, bool synchronous) · 53 unmodified lines
427408 }
428409
429410 ret = 0;
430− bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read);
431− if (bio == NULL) {
432− unlock_page(page);
433− ret = -ENOMEM;
434− goto out;
435− }
436− disk = bio->bi_disk;
411+ bio = bio_alloc(GFP_KERNEL, 1);
412+ bio_set_dev(bio, sis->bdev);
413+ bio->bi_opf = REQ_OP_READ;
414+ bio->bi_iter.bi_sector = swap_page_sector(page);
415+ bio->bi_end_io = end_swap_bio_read;
416+ bio_add_page(bio, page, thp_size(page), 0);
417+
418+ disk = bio->bi_bdev->bd_disk;
437419 /*
438420 * Keep this task valid during swap readpage because the oom killer may
439421 * attempt to access it in the page fault retry time check.
440422 */
441− bio_set_op_attrs(bio, REQ_OP_READ, 0);
442423 if (synchronous) {
443424 bio->bi_opf |= REQ_HIPRI;
444425 get_task_struct(current);
mm/swapfile.c+3−33
@@ -47,7 +47,6 @@ · 46 unmodified lines
4747 static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
4848 unsigned char);
4949 static void free_swap_count_continuations(struct swap_info_struct *);
50−static sector_t map_swap_entry(swp_entry_t, struct block_device**);
5150
5251 DEFINE_SPINLOCK(swap_lock);
5352 static unsigned int nr_swapfiles;
@@ -1850,12 +1849,13 @@ int find_first_swap(dev_t *device) · 1796 unmodified lines
18501849 */
18511850 sector_t swapdev_block(int type, pgoff_t offset)
18521851 {
1853− struct block_device *bdev;
18541852 struct swap_info_struct *si = swap_type_to_swap_info(type);
1853+ struct swap_extent *se;
18551854
18561855 if (!si || !(si->flags & SWP_WRITEOK))
18571856 return 0;
1858− return map_swap_entry(swp_entry(type, offset), &bdev);
1857+ se = offset_to_swap_extent(si, offset);
1858+ return se->start_block + (offset - se->start_page);
18591859 }
18601860
18611861 /*
@@ -2281,36 +2281,6 @@ static void drain_mmlist(void) · 419 unmodified lines
22812281 spin_unlock(&mmlist_lock);
22822282 }
22832283
2284−/*
2285− * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
2286− * corresponds to page offset for the specified swap entry.
2287− * Note that the type of this function is sector_t, but it returns page offset
2288− * into the bdev, not sector offset.
2289− */
2290−static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
2291−{
2292− struct swap_info_struct *sis;
2293− struct swap_extent *se;
2294− pgoff_t offset;
2295−
2296− sis = swp_swap_info(entry);
2297− *bdev = sis->bdev;
2298−
2299− offset = swp_offset(entry);
2300− se = offset_to_swap_extent(sis, offset);
2301− return se->start_block + (offset - se->start_page);
2302−}
2303−
2304−/*
2305− * Returns the page offset into bdev for the specified page's swap entry.
2306− */
2307−sector_t map_swap_page(struct page *page, struct block_device **bdev)
2308−{
2309− swp_entry_t entry;
2310− entry.val = page_private(page);
2311− return map_swap_entry(entry, bdev);
2312−}
2313−
23142284 /*
23152285 * Free all of a swapdev's extent information
23162286 */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t