History

Merge tag 'for-5.20/block-2022-07-29' of git://git.kernel.dk/linux-block

Pull block updates from Jens Axboe: - Improve the type checking of request flags (Bart) - Ensure queue mapping for a single queues always picks the right queue (Bart) - Sanitize the io priority handling (Jan) - rq-qos race fix (Jinke) - Reserved tags handling improvements (John) - Separate memory alignment from file/disk offset aligment for O_DIRECT (Keith) - Add new ublk driver, userspace block driver using io_uring for communication with the userspace backend (Ming) - Use try_cmpxchg() to cleanup the code in various spots (Uros) - Finally remove bdevname() (Christoph) - Clean up the zoned device handling (Christoph) - Clean up independent access range support (Christoph) - Clean up and improve block sysfs handling (Christoph) - Clean up and improve teardown of block devices. This turns the usual two step process into something that is simpler to implement and handle in block drivers (Christoph) - Clean up chunk size handling (Christoph) - Misc cleanups and fixes (Bart, Bo, Dan, GuoYong, Jason, Keith, Liu, Ming, Sebastian, Yang, Ying) * tag 'for-5.20/block-2022-07-29' of git://git.kernel.dk/linux-block: (178 commits) ublk_drv: fix double shift bug ublk_drv: make sure that correct flags(features) returned to userspace ublk_drv: fix error handling of ublk_add_dev ublk_drv: fix lockdep warning block: remove __blk_get_queue block: call blk_mq_exit_queue from disk_release for never added disks blk-mq: fix error handling in __blk_mq_alloc_disk ublk: defer disk allocation ublk: rewrite ublk_ctrl_get_queue_affinity to not rely on hctx->cpumask ublk: fold __ublk_create_dev into ublk_ctrl_add_dev ublk: cleanup ublk_ctrl_uring_cmd ublk: simplify ublk_ch_open and ublk_ch_release ublk: remove the empty open and release block device operations ublk: remove UBLK_IO_F_PREFLUSH ublk: add a MAINTAINERS entry block: don't allow the same type rq_qos add more than once mmc: fix disk/queue leak in case of adding disk failure ublk_drv: fix an IS_ERR() vs NULL check ublk: remove UBLK_IO_F_INTEGRITY ublk_drv: remove unneeded semicolon ...

Linus Torvalds committed 4 years ago

Commit c013d0a2 parents 42df1cb 8d9fdb6261 files · +3635 −2158Browse files

Unified · Split

261 files
  1. Documentation/ABI/stable/sysfs-block +9 −0
  2. Documentation/scsi/scsi_eh.rst +1 −2
  3. Documentation/scsi/scsi_mid_low_api.rst +1 −1
  4. MAINTAINERS +7 −0
  5. arch/m68k/emu/nfblock.c +2 −2
  6. arch/um/drivers/ubd_kern.c +4 −4
  7. arch/xtensa/platforms/iss/simdisk.c +2 −2
  8. block/Kconfig +0 −1
  9. block/bdev.c +4 −6
  10. block/bfq-cgroup.c +17 −17
  11. block/bfq-iosched.c +8 −8
  12. block/bfq-iosched.h +4 −4
  13. block/bfq-wf2q.c +2 −3
  14. block/bio.c +57 −69
  15. block/blk-cgroup-rwstat.h +4 −4
  16. block/blk-cgroup.c +40 −36
  17. block/blk-cgroup.h +4 −8
  18. block/blk-core.c +17 −63
  19. block/blk-flush.c +3 −3
  20. block/blk-ia-ranges.c +20 −45
  21. block/blk-ioc.c +2 −0
  22. block/blk-iocost.c +14 −8
  23. block/blk-iolatency.c +16 −14
  24. block/blk-ioprio.c +7 −50
  25. block/blk-ioprio.h +9 −0
  26. block/blk-lib.c +2 −4
  27. block/blk-merge.c +46 −33
  28. block/blk-mq-debugfs-zoned.c +3 −3
  29. block/blk-mq-debugfs.c +10 −9
  30. block/blk-mq-sysfs.c +22 −23
  31. block/blk-mq-tag.c +12 −19
  32. block/blk-mq-tag.h +4 −6
  33. block/blk-mq.c +78 −27
  34. block/blk-mq.h +19 −20
  35. block/blk-rq-qos.c +2 −8
  36. block/blk-rq-qos.h +10 −1
  37. block/blk-settings.c +6 −5
  38. block/blk-sysfs.c +23 −37
  39. block/blk-throttle.c +4 −3
  40. block/blk-wbt.c +20 −10
  41. block/blk-zoned.c +46 −46
  42. block/blk.h +23 −11
  43. block/bounce.c +10 −3
  44. block/bsg-lib.c +3 −3
  45. block/elevator.h +1 −1
  46. block/fops.c +16 −12
  47. block/genhd.c +32 −48
  48. block/ioctl.c +1 −1
  49. block/ioprio.c +50 −8
  50. block/kyber-iosched.c +4 −4
  51. block/mq-deadline.c +2 −2
  52. block/partitions/core.c +1 −2
  53. drivers/block/Kconfig +9 −0
  54. drivers/block/Makefile +2 −0
  55. drivers/block/amiflop.c +1 −1
  56. drivers/block/aoe/aoeblk.c +1 −1
  57. drivers/block/aoe/aoedev.c +1 −1
  58. drivers/block/ataflop.c +2 −3
  59. drivers/block/brd.c +4 −4
  60. drivers/block/drbd/drbd_actlog.c +5 −4
  61. drivers/block/drbd/drbd_bitmap.c +1 −1
  62. drivers/block/drbd/drbd_int.h +2 −3
  63. drivers/block/drbd/drbd_main.c +2 −2
  64. drivers/block/drbd/drbd_receiver.c +12 −12
  65. drivers/block/drbd/drbd_req.c +2 −4
  66. drivers/block/drbd/drbd_worker.c +1 −1
  67. drivers/block/floppy.c +4 −4
  68. drivers/block/loop.c +1 −2
  69. drivers/block/mtip32xx/mtip32xx.c +48 −259
  70. drivers/block/mtip32xx/mtip32xx.h +0 −5
  71. drivers/block/n64cart.c +1 −1
  72. drivers/block/nbd.c +4 −5
  73. drivers/block/null_blk/main.c +7 −8
  74. drivers/block/null_blk/null_blk.h +5 −7
  75. drivers/block/null_blk/trace.h +1 −1
  76. drivers/block/null_blk/zoned.c +6 −6
  77. drivers/block/paride/pcd.c +2 −2
  78. drivers/block/paride/pd.c +4 −2
  79. drivers/block/paride/pf.c +2 −2
  80. drivers/block/pktcdvd.c +5 −9
  81. drivers/block/ps3disk.c +2 −2
  82. drivers/block/ps3vram.c +2 −2
  83. drivers/block/rbd.c +1 −1
  84. drivers/block/rnbd/rnbd-clt.c +3 −3
  85. drivers/block/rnbd/rnbd-proto.h +4 −3
  86. drivers/block/rnbd/rnbd-srv-dev.c +0 −1
  87. drivers/block/rnbd/rnbd-srv-dev.h +0 −1
  88. drivers/block/rnbd/rnbd-srv-sysfs.c +2 −3
  89. drivers/block/rnbd/rnbd-srv.c +4 −5
  90. drivers/block/rnbd/rnbd-srv.h +1 −2
  91. drivers/block/sunvdc.c +2 −2
  92. drivers/block/swim.c +1 −1
  93. drivers/block/swim3.c +1 −1
  94. drivers/block/sx8.c +3 −3
  95. drivers/block/ublk_drv.c +1545 −0
  96. drivers/block/virtio_blk.c +1 −2
  97. drivers/block/xen-blkback/blkback.c +3 −3
  98. drivers/block/xen-blkfront.c +2 −2
  99. drivers/block/z2ram.c +1 −2
  100. drivers/block/zram/zram_drv.c +4 −4
  101. drivers/cdrom/gdrom.c +1 −2
  102. drivers/infiniband/ulp/srp/ib_srp.c +1 −2
  103. drivers/md/bcache/super.c +13 −14
  104. drivers/md/dm-bufio.c +12 −14
  105. drivers/md/dm-ebs-target.c +8 −7
  106. drivers/md/dm-flakey.c +6 −2
  107. drivers/md/dm-integrity.c +38 −38
  108. drivers/md/dm-io.c +19 −19
  109. drivers/md/dm-kcopyd.c +13 −13
  110. drivers/md/dm-log.c +3 −5
  111. drivers/md/dm-raid.c +1 −1
  112. drivers/md/dm-raid1.c +5 −7
  113. drivers/md/dm-snap-persistent.c +12 −13
  114. drivers/md/dm-table.c +2 −4
  115. drivers/md/dm-writecache.c +4 −8
  116. drivers/md/dm-zone.c +43 −45
  117. drivers/md/dm-zoned-metadata.c +3 −2
  118. drivers/md/dm-zoned-target.c +12 −13
  119. drivers/md/dm-zoned.h +1 −1
  120. drivers/md/dm.c +15 −18
  121. drivers/md/md-bitmap.c +3 −3
  122. drivers/md/md.c +8 −8
  123. drivers/md/md.h +1 −2
  124. drivers/md/raid1.c +7 −7
  125. drivers/md/raid10.c +11 −11
  126. drivers/md/raid5-cache.c +6 −6
  127. drivers/md/raid5-ppl.c +6 −6
  128. drivers/md/raid5.c +2 −1
  129. drivers/memstick/core/ms_block.c +1 −2
  130. drivers/memstick/core/mspro_block.c +1 −2
  131. drivers/mmc/core/block.c +3 −3
  132. drivers/mmc/core/queue.c +1 −3
  133. drivers/mtd/mtd_blkdevs.c +2 −2
  134. drivers/mtd/ubi/block.c +2 −2
  135. drivers/nvdimm/btt.c +4 −4
  136. drivers/nvdimm/pmem.c +3 −3
  137. drivers/nvme/host/apple.c +2 −3
  138. drivers/nvme/host/core.c +2 −3
  139. drivers/nvme/host/fc.c +8 −10
  140. drivers/nvme/host/ioctl.c +2 −2
  141. drivers/nvme/host/multipath.c +2 −2
  142. drivers/nvme/host/nvme.h +2 −2
  143. drivers/nvme/host/pci.c +2 −2
  144. drivers/nvme/host/rdma.c +7 −8
  145. drivers/nvme/host/tcp.c +7 −8
  146. drivers/nvme/host/zns.c +3 −3
  147. drivers/nvme/target/io-cmd-bdev.c +9 −8
  148. drivers/nvme/target/loop.c +6 −6
  149. drivers/nvme/target/zns.c +12 −12
  150. drivers/s390/block/dasd.c +2 −2
  151. drivers/s390/block/dasd_genhd.c +2 −2
  152. drivers/s390/block/dasd_int.h +1 −1
  153. drivers/s390/block/dcssblk.c +4 −4
  154. drivers/s390/block/scm_blk.c +2 −2
  155. drivers/scsi/aacraid/comminit.c +1 −1
  156. drivers/scsi/aacraid/linit.c +1 −1
  157. drivers/scsi/device_handler/scsi_dh_alua.c +2 −2
  158. drivers/scsi/device_handler/scsi_dh_emc.c +1 −1
  159. drivers/scsi/device_handler/scsi_dh_hp_sw.c +2 −2
  160. drivers/scsi/device_handler/scsi_dh_rdac.c +1 −1
  161. drivers/scsi/fnic/fnic_scsi.c +4 −10
  162. drivers/scsi/hosts.c +6 −8
  163. drivers/scsi/mpi3mr/mpi3mr_os.c +4 −12
  164. drivers/scsi/scsi_error.c +11 −11
  165. drivers/scsi/scsi_lib.c +9 −17
  166. drivers/scsi/scsi_priv.h +2 −2
  167. drivers/scsi/scsi_sysfs.c +1 −1
  168. drivers/scsi/sd.c +5 −5
  169. drivers/scsi/sd_zbc.c +6 −6
  170. drivers/scsi/sr.c +2 −2
  171. drivers/target/target_core_iblock.c +2 −2
  172. drivers/ufs/core/ufshcd.c +2 −2
  173. drivers/ufs/core/ufshpb.c +3 −4
  174. fs/btrfs/check-integrity.c +2 −2
  175. fs/btrfs/compression.c +3 −3
  176. fs/btrfs/compression.h +1 −1
  177. fs/btrfs/extent_io.c +9 −9
  178. fs/btrfs/inode.c +2 −2
  179. fs/btrfs/raid56.c +2 −2
  180. fs/buffer.c +31 −30
  181. fs/direct-io.c +23 −17
  182. fs/ext4/ext4.h +4 −4
  183. fs/ext4/fast_commit.c +2 −2
  184. fs/ext4/mmp.c +6 −5
  185. fs/ext4/super.c +10 −10
  186. fs/f2fs/data.c +6 −5
  187. fs/f2fs/f2fs.h +3 −3
  188. fs/f2fs/node.c +1 −1
  189. fs/f2fs/segment.c +1 −1
  190. fs/gfs2/bmap.c +2 −3
  191. fs/gfs2/dir.c +2 −3
  192. fs/gfs2/log.c +2 −2
  193. fs/gfs2/log.h +1 −1
  194. fs/gfs2/lops.c +2 −2
  195. fs/gfs2/lops.h +1 −1
  196. fs/gfs2/meta_io.c +8 −10
  197. fs/gfs2/quota.c +1 −1
  198. fs/hfsplus/hfsplus_fs.h +1 −1
  199. fs/hfsplus/part_tbl.c +2 −3
  200. fs/hfsplus/super.c +2 −2
  201. fs/hfsplus/wrapper.c +6 −6
  202. fs/iomap/direct-io.c +6 −6
  203. fs/isofs/compress.c +1 −1
  204. fs/jbd2/commit.c +4 −4
  205. fs/jbd2/journal.c +14 −11
  206. fs/jbd2/recovery.c +2 −2
  207. fs/mpage.c +3 −3
  208. fs/nfs/blocklayout/blocklayout.c +6 −7
  209. fs/nilfs2/btnode.c +4 −4
  210. fs/nilfs2/btnode.h +2 −2
  211. fs/nilfs2/btree.c +3 −3
  212. fs/nilfs2/gcinode.c +3 −4
  213. fs/nilfs2/mdt.c +10 −9
  214. fs/ntfs/aops.c +3 −3
  215. fs/ntfs/compress.c +1 −1
  216. fs/ntfs/file.c +1 −1
  217. fs/ntfs/logfile.c +1 −1
  218. fs/ntfs/mft.c +2 −2
  219. fs/ntfs3/file.c +1 −1
  220. fs/ntfs3/fsntfs.c +1 −1
  221. fs/ntfs3/inode.c +1 −1
  222. fs/ntfs3/ntfs_fs.h +1 −1
  223. fs/ocfs2/aops.c +1 −1
  224. fs/ocfs2/buffer_head_io.c +4 −4
  225. fs/ocfs2/cluster/heartbeat.c +35 −40
  226. fs/ocfs2/super.c +1 −1
  227. fs/reiserfs/inode.c +2 −2
  228. fs/reiserfs/journal.c +6 −6
  229. fs/reiserfs/stree.c +2 −2
  230. fs/reiserfs/super.c +1 −1
  231. fs/udf/dir.c +1 −1
  232. fs/udf/directory.c +1 −1
  233. fs/udf/inode.c +1 −1
  234. fs/ufs/balloc.c +1 −1
  235. fs/xfs/xfs_bio_io.c +1 −1
  236. fs/xfs/xfs_buf.c +2 −2
  237. fs/xfs/xfs_linux.h +1 −1
  238. fs/xfs/xfs_log_recover.c +1 −1
  239. fs/zonefs/super.c +10 −12
  240. fs/zonefs/trace.h +2 −2
  241. include/linux/bio.h +5 −5
  242. include/linux/blk-mq.h +22 −12
  243. include/linux/blk_types.h +63 −56
  244. include/linux/blkdev.h +89 −135
  245. include/linux/blktrace_api.h +2 −11
  246. include/linux/buffer_head.h +5 −4
  247. include/linux/dm-io.h +2 −2
  248. include/linux/ioprio.h +10 −14
  249. include/linux/jbd2.h +1 −1
  250. include/linux/uio.h +2 −0
  251. include/linux/writeback.h +2 −2
  252. include/scsi/scsi_cmnd.h +2 −2
  253. include/scsi/scsi_device.h +1 −1
  254. include/scsi/scsi_host.h +1 −1
  255. include/trace/events/f2fs.h +11 −11
  256. include/trace/events/jbd2.h +6 −6
  257. include/trace/events/nilfs2.h +2 −2
  258. include/uapi/linux/ublk_cmd.h +161 −0
  259. kernel/power/swap.c +13 −16
  260. kernel/trace/blktrace.c +30 −42
  261. lib/iov_iter.c +92 −0
Documentation/ABI/stable/sysfs-block+9−0
@@ -260,6 +260,15 @@ Description: · 259 unmodified lines
260260 for discards, and don't read this file.
261261
262262
263+What: /sys/block/<disk>/queue/dma_alignment
264+Date: May 2022
265+Contact: linux-block@vger.kernel.org
266+Description:
267+ Reports the alignment that user space addresses must have to be
268+ used for raw block device access with O_DIRECT and other driver
269+ specific passthrough mechanisms.
270+
271+
263272 What: /sys/block/<disk>/queue/fua
264273 Date: May 2018
265274 Contact: linux-block@vger.kernel.org
Documentation/scsi/scsi_eh.rst+1−2
@@ -87,8 +87,7 @@ with the command. · 86 unmodified lines
8787 1.2.2 Completing a scmd w/ timeout
8888 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
8989
90−The timeout handler is scsi_times_out(). When a timeout occurs, this
91−function
90+The timeout handler is scsi_timeout(). When a timeout occurs, this function
9291
9392 1. invokes optional hostt->eh_timed_out() callback. Return value can
9493 be one of
Documentation/scsi/scsi_mid_low_api.rst+1−1
@@ -731,7 +731,7 @@ Details:: · 730 unmodified lines
731731 * Notes: If 'no_async_abort' is defined this callback
732732 * will be invoked from scsi_eh thread. No other commands
733733 * will then be queued on current host during eh.
734− * Otherwise it will be called whenever scsi_times_out()
734+ * Otherwise it will be called whenever scsi_timeout()
735735 * is called due to a command timeout.
736736 *
737737 * Optionally defined in: LLD
MAINTAINERS+7−0
@@ -20539,6 +20539,13 @@ F: Documentation/filesystems/ubifs-authentication.rst · 20538 unmodified lines
2053920539 F: Documentation/filesystems/ubifs.rst
2054020540 F: fs/ubifs/
2054120541
20542+UBLK USERSPACE BLOCK DRIVER
20543+M: Ming Lei <ming.lei@redhat.com>
20544+L: linux-block@vger.kernel.org
20545+S: Maintained
20546+F: drivers/block/ublk_drv.c
20547+F: include/uapi/linux/ublk_cmd.h
20548+
2054220549 UCLINUX (M68KNOMMU AND COLDFIRE)
2054320550 M: Greg Ungerer <gerg@linux-m68k.org>
2054420551 L: linux-m68k@lists.linux-m68k.org
arch/m68k/emu/nfblock.c+2−2
@@ -138,7 +138,7 @@ static int __init nfhd_init_one(int id, u32 blocks, u32 bsize) · 137 unmodified lines
138138 return 0;
139139
140140 out_cleanup_disk:
141− blk_cleanup_disk(dev->disk);
141+ put_disk(dev->disk);
142142 free_dev:
143143 kfree(dev);
144144 out:
@@ -180,7 +180,7 @@ static void __exit nfhd_exit(void) · 35 unmodified lines
180180 list_for_each_entry_safe(dev, next, &nfhd_list, list) {
181181 list_del(&dev->list);
182182 del_gendisk(dev->disk);
183− blk_cleanup_disk(dev->disk);
183+ put_disk(dev->disk);
184184 kfree(dev);
185185 }
186186 unregister_blkdev(major_num, "nfhd");
arch/um/drivers/ubd_kern.c+4−4
@@ -925,7 +925,7 @@ static int ubd_add(int n, char **error_out) · 924 unmodified lines
925925 return 0;
926926
927927 out_cleanup_disk:
928− blk_cleanup_disk(disk);
928+ put_disk(disk);
929929 out_cleanup_tags:
930930 blk_mq_free_tag_set(&ubd_dev->tag_set);
931931 out:
@@ -1032,7 +1032,7 @@ static int ubd_remove(int n, char **error_out) · 100 unmodified lines
10321032 ubd_gendisk[n] = NULL;
10331033 if(disk != NULL){
10341034 del_gendisk(disk);
1035− blk_cleanup_disk(disk);
1035+ put_disk(disk);
10361036 }
10371037
10381038 err = 0;
@@ -1262,7 +1262,7 @@ static void ubd_map_req(struct ubd *dev, struct io_thread_req *io_req, · 223 unmodified lines
12621262 struct req_iterator iter;
12631263 int i = 0;
12641264 unsigned long byte_offset = io_req->offset;
1265− int op = req_op(req);
1265+ enum req_op op = req_op(req);
12661266
12671267 if (op == REQ_OP_WRITE_ZEROES || op == REQ_OP_DISCARD) {
12681268 io_req->io_desc[0].buffer = NULL;
@@ -1325,7 +1325,7 @@ static int ubd_submit_request(struct ubd *dev, struct request *req) · 56 unmodified lines
13251325 int segs = 0;
13261326 struct io_thread_req *io_req;
13271327 int ret;
1328− int op = req_op(req);
1328+ enum req_op op = req_op(req);
13291329
13301330 if (op == REQ_OP_FLUSH)
13311331 segs = 0;
arch/xtensa/platforms/iss/simdisk.c+2−2
@@ -290,7 +290,7 @@ static int __init simdisk_setup(struct simdisk *dev, int which, · 289 unmodified lines
290290 return 0;
291291
292292 out_cleanup_disk:
293− blk_cleanup_disk(dev->gd);
293+ put_disk(dev->gd);
294294 out:
295295 return err;
296296 }
@@ -344,7 +344,7 @@ static void simdisk_teardown(struct simdisk *dev, int which, · 47 unmodified lines
344344 simdisk_detach(dev);
345345 if (dev->gd) {
346346 del_gendisk(dev->gd);
347− blk_cleanup_disk(dev->gd);
347+ put_disk(dev->gd);
348348 }
349349 remove_proc_entry(tmp, procdir);
350350 }
block/Kconfig+0−1
@@ -147,7 +147,6 @@ config BLK_CGROUP_FC_APPID · 146 unmodified lines
147147 config BLK_CGROUP_IOCOST
148148 bool "Enable support for cost model based cgroup IO controller"
149149 depends on BLK_CGROUP
150− select BLK_RQ_IO_DATA_LEN
151150 select BLK_RQ_ALLOC_TIME
152151 help
153152 Enabling this option enables the .weight interface for cost
block/bdev.c+4−6
@@ -54,12 +54,10 @@ static void bdev_write_inode(struct block_device *bdev) · 53 unmodified lines
5454 while (inode->i_state & I_DIRTY) {
5555 spin_unlock(&inode->i_lock);
5656 ret = write_inode_now(inode, true);
57− if (ret) {
58− char name[BDEVNAME_SIZE];
59− pr_warn_ratelimited("VFS: Dirty inode writeback failed "
60− "for block device %s (err=%d).\n",
61− bdevname(bdev, name), ret);
62− }
57+ if (ret)
58+ pr_warn_ratelimited(
59+ "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n",
60+ bdev, ret);
6361 spin_lock(&inode->i_lock);
6462 }
6563 spin_unlock(&inode->i_lock);
block/bfq-cgroup.c+17−17
@@ -220,46 +220,46 @@ void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) · 219 unmodified lines
220220 }
221221
222222 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
223− unsigned int op)
223+ blk_opf_t opf)
224224 {
225− blkg_rwstat_add(&bfqg->stats.queued, op, 1);
225+ blkg_rwstat_add(&bfqg->stats.queued, opf, 1);
226226 bfqg_stats_end_empty_time(&bfqg->stats);
227227 if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))
228228 bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
229229 }
230230
231−void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op)
231+void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf)
232232 {
233− blkg_rwstat_add(&bfqg->stats.queued, op, -1);
233+ blkg_rwstat_add(&bfqg->stats.queued, opf, -1);
234234 }
235235
236−void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)
236+void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf)
237237 {
238− blkg_rwstat_add(&bfqg->stats.merged, op, 1);
238+ blkg_rwstat_add(&bfqg->stats.merged, opf, 1);
239239 }
240240
241241 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
242− u64 io_start_time_ns, unsigned int op)
242+ u64 io_start_time_ns, blk_opf_t opf)
243243 {
244244 struct bfqg_stats *stats = &bfqg->stats;
245245 u64 now = ktime_get_ns();
246246
247247 if (now > io_start_time_ns)
248− blkg_rwstat_add(&stats->service_time, op,
248+ blkg_rwstat_add(&stats->service_time, opf,
249249 now - io_start_time_ns);
250250 if (io_start_time_ns > start_time_ns)
251− blkg_rwstat_add(&stats->wait_time, op,
251+ blkg_rwstat_add(&stats->wait_time, opf,
252252 io_start_time_ns - start_time_ns);
253253 }
254254
255255 #else /* CONFIG_BFQ_CGROUP_DEBUG */
256256
257257 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
258− unsigned int op) { }
259−void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op) { }
260−void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op) { }
258+ blk_opf_t opf) { }
259+void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { }
260+void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { }
261261 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
262− u64 io_start_time_ns, unsigned int op) { }
262+ u64 io_start_time_ns, blk_opf_t opf) { }
263263 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }
264264 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { }
265265 void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }
@@ -706,10 +706,10 @@ void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, · 440 unmodified lines
706706 }
707707
708708 /**
709− * __bfq_bic_change_cgroup - move @bic to @cgroup.
709+ * __bfq_bic_change_cgroup - move @bic to @bfqg.
710710 * @bfqd: the queue descriptor.
711711 * @bic: the bic to move.
712− * @blkcg: the blk-cgroup to move to.
712+ * @bfqg: the group to move to.
713713 *
714714 * Move bic to blkcg, assuming that bfqd->lock is held; which makes
715715 * sure that the reference to cgroup is valid across the call (see
@@ -863,6 +863,7 @@ static void bfq_flush_idle_tree(struct bfq_service_tree *st) · 147 unmodified lines
863863 * @bfqd: the device data structure with the root group.
864864 * @entity: the entity to move, if entity is a leaf; or the parent entity
865865 * of an active leaf entity to move, if entity is not a leaf.
866+ * @ioprio_class: I/O priority class to reparent.
866867 */
867868 static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
868869 struct bfq_entity *entity,
@@ -892,6 +893,7 @@ static void bfq_reparent_leaf_entity(struct bfq_data *bfqd, · 23 unmodified lines
892893 * @bfqd: the device data structure with the root group.
893894 * @bfqg: the group to move from.
894895 * @st: the service tree to start the search from.
896+ * @ioprio_class: I/O priority class to reparent.
895897 */
896898 static void bfq_reparent_active_queues(struct bfq_data *bfqd,
897899 struct bfq_group *bfqg,
@@ -1471,8 +1473,6 @@ struct bfq_group *bfqq_group(struct bfq_queue *bfqq) · 573 unmodified lines
14711473 return bfqq->bfqd->root_group;
14721474 }
14731475
1474−void bfqg_and_blkg_get(struct bfq_group *bfqg) {}
1475−
14761476 void bfqg_and_blkg_put(struct bfq_group *bfqg) {}
14771477
14781478 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
block/bfq-iosched.c+8−8
@@ -668,19 +668,19 @@ static bool bfqq_request_over_limit(struct bfq_queue *bfqq, int limit) · 667 unmodified lines
668668 * significantly affect service guarantees coming from the BFQ scheduling
669669 * algorithm.
670670 */
671−static void bfq_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
671+static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data)
672672 {
673673 struct bfq_data *bfqd = data->q->elevator->elevator_data;
674674 struct bfq_io_cq *bic = bfq_bic_lookup(data->q);
675− struct bfq_queue *bfqq = bic ? bic_to_bfqq(bic, op_is_sync(op)) : NULL;
675+ struct bfq_queue *bfqq = bic ? bic_to_bfqq(bic, op_is_sync(opf)) : NULL;
676676 int depth;
677677 unsigned limit = data->q->nr_requests;
678678
679679 /* Sync reads have full depth available */
680− if (op_is_sync(op) && !op_is_write(op)) {
680+ if (op_is_sync(opf) && !op_is_write(opf)) {
681681 depth = 0;
682682 } else {
683− depth = bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(op)];
683+ depth = bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)];
684684 limit = (limit * depth) >> bfqd->full_depth_shift;
685685 }
686686
@@ -693,7 +693,7 @@ static void bfq_limit_depth(unsigned int op, struct blk_mq_alloc_data *data) · 6 unmodified lines
693693 depth = 1;
694694
695695 bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u",
696− __func__, bfqd->wr_busy_queues, op_is_sync(op), depth);
696+ __func__, bfqd->wr_busy_queues, op_is_sync(opf), depth);
697697 if (depth)
698698 data->shallow_depth = depth;
699699 }
@@ -6104,7 +6104,7 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) · 5404 unmodified lines
61046104 static void bfq_update_insert_stats(struct request_queue *q,
61056105 struct bfq_queue *bfqq,
61066106 bool idle_timer_disabled,
6107− unsigned int cmd_flags)
6107+ blk_opf_t cmd_flags)
61086108 {
61096109 if (!bfqq)
61106110 return;
@@ -6129,7 +6129,7 @@ static void bfq_update_insert_stats(struct request_queue *q, · 18 unmodified lines
61296129 static inline void bfq_update_insert_stats(struct request_queue *q,
61306130 struct bfq_queue *bfqq,
61316131 bool idle_timer_disabled,
6132− unsigned int cmd_flags) {}
6132+ blk_opf_t cmd_flags) {}
61336133 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
61346134
61356135 static struct bfq_queue *bfq_init_rq(struct request *rq);
@@ -6141,7 +6141,7 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, · 5 unmodified lines
61416141 struct bfq_data *bfqd = q->elevator->elevator_data;
61426142 struct bfq_queue *bfqq;
61436143 bool idle_timer_disabled = false;
6144− unsigned int cmd_flags;
6144+ blk_opf_t cmd_flags;
61456145 LIST_HEAD(free);
61466146
61476147 #ifdef CONFIG_BFQ_GROUP_IOSCHED
block/bfq-iosched.h+4−4
@@ -994,11 +994,11 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); · 993 unmodified lines
994994
995995 void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq);
996996 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
997− unsigned int op);
998−void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op);
999−void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op);
997+ blk_opf_t opf);
998+void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf);
999+void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf);
10001000 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
1001− u64 io_start_time_ns, unsigned int op);
1001+ u64 io_start_time_ns, blk_opf_t opf);
10021002 void bfqg_stats_update_dequeue(struct bfq_group *bfqg);
10031003 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg);
10041004 void bfqg_stats_update_idle_time(struct bfq_group *bfqg);
block/bfq-wf2q.c+2−3
@@ -1360,6 +1360,8 @@ left: · 1359 unmodified lines
13601360 /**
13611361 * __bfq_lookup_next_entity - return the first eligible entity in @st.
13621362 * @st: the service tree.
1363+ * @in_service: whether or not there is an in-service entity for the sched_data
1364+ * this active tree belongs to.
13631365 *
13641366 * If there is no in-service entity for the sched_data st belongs to,
13651367 * then return the entity that will be set in service if:
@@ -1472,9 +1474,6 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, · 106 unmodified lines
14721474 break;
14731475 }
14741476
1475− if (!entity)
1476− return NULL;
1477−
14781477 return entity;
14791478 }
14801479
block/bio.c+57−69
@@ -239,7 +239,7 @@ static void bio_free(struct bio *bio) · 238 unmodified lines
239239 * when IO has completed, or when the bio is released.
240240 */
241241 void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
242− unsigned short max_vecs, unsigned int opf)
242+ unsigned short max_vecs, blk_opf_t opf)
243243 {
244244 bio->bi_next = NULL;
245245 bio->bi_bdev = bdev;
@@ -292,7 +292,7 @@ EXPORT_SYMBOL(bio_init); · 46 unmodified lines
292292 * preserved are the ones that are initialized by bio_alloc_bioset(). See
293293 * comment in struct bio.
294294 */
295−void bio_reset(struct bio *bio, struct block_device *bdev, unsigned int opf)
295+void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf)
296296 {
297297 bio_uninit(bio);
298298 memset(bio, 0, BIO_RESET_BYTES);
@@ -341,7 +341,7 @@ void bio_chain(struct bio *bio, struct bio *parent) · 42 unmodified lines
341341 EXPORT_SYMBOL(bio_chain);
342342
343343 struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
344− unsigned int nr_pages, unsigned int opf, gfp_t gfp)
344+ unsigned int nr_pages, blk_opf_t opf, gfp_t gfp)
345345 {
346346 struct bio *new = bio_alloc(bdev, nr_pages, opf, gfp);
347347
@@ -409,7 +409,7 @@ static void punt_bios_to_rescuer(struct bio_set *bs) · 61 unmodified lines
409409 }
410410
411411 static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
412− unsigned short nr_vecs, unsigned int opf, gfp_t gfp,
412+ unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp,
413413 struct bio_set *bs)
414414 {
415415 struct bio_alloc_cache *cache;
@@ -468,7 +468,7 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, · 52 unmodified lines
468468 * Returns: Pointer to new bio on success, NULL on failure.
469469 */
470470 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
471− unsigned int opf, gfp_t gfp_mask,
471+ blk_opf_t opf, gfp_t gfp_mask,
472472 struct bio_set *bs)
473473 {
474474 gfp_t saved_gfp = gfp_mask;
@@ -1033,7 +1033,7 @@ int bio_add_zone_append_page(struct bio *bio, struct page *page, · 558 unmodified lines
10331033 if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND))
10341034 return 0;
10351035
1036− if (WARN_ON_ONCE(!blk_queue_is_zoned(q)))
1036+ if (WARN_ON_ONCE(!bdev_is_zoned(bio->bi_bdev)))
10371037 return 0;
10381038
10391039 return bio_add_hw_page(q, bio, page, len, offset,
@@ -1159,6 +1159,37 @@ static void bio_put_pages(struct page **pages, size_t size, size_t off) · 119 unmodified lines
11591159 put_page(pages[i]);
11601160 }
11611161
1162+static int bio_iov_add_page(struct bio *bio, struct page *page,
1163+ unsigned int len, unsigned int offset)
1164+{
1165+ bool same_page = false;
1166+
1167+ if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) {
1168+ if (WARN_ON_ONCE(bio_full(bio, len)))
1169+ return -EINVAL;
1170+ __bio_add_page(bio, page, len, offset);
1171+ return 0;
1172+ }
1173+
1174+ if (same_page)
1175+ put_page(page);
1176+ return 0;
1177+}
1178+
1179+static int bio_iov_add_zone_append_page(struct bio *bio, struct page *page,
1180+ unsigned int len, unsigned int offset)
1181+{
1182+ struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1183+ bool same_page = false;
1184+
1185+ if (bio_add_hw_page(q, bio, page, len, offset,
1186+ queue_max_zone_append_sectors(q), &same_page) != len)
1187+ return -EINVAL;
1188+ if (same_page)
1189+ put_page(page);
1190+ return 0;
1191+}
1192+
11621193 #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))
11631194
11641195 /**
@@ -1177,7 +1208,6 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · 12 unmodified lines
11771208 unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
11781209 struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
11791210 struct page **pages = (struct page **)bv;
1180− bool same_page = false;
11811211 ssize_t size, left;
11821212 unsigned len, i;
11831213 size_t offset;
@@ -1186,82 +1216,43 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · 2 unmodified lines
11861216 * Move page array up in the allocated memory for the bio vecs as far as
11871217 * possible so that we can start filling biovecs from the beginning
11881218 * without overwriting the temporary page array.
1189− */
1219+ */
11901220 BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
11911221 pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
11921222
1193− size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
1194− if (unlikely(size <= 0))
1195− return size ? size : -EFAULT;
1196−
1197− for (left = size, i = 0; left > 0; left -= len, i++) {
1198− struct page *page = pages[i];
1199−
1200− len = min_t(size_t, PAGE_SIZE - offset, left);
1201−
1202− if (__bio_try_merge_page(bio, page, len, offset, &same_page)) {
1203− if (same_page)
1204− put_page(page);
1205− } else {
1206− if (WARN_ON_ONCE(bio_full(bio, len))) {
1207− bio_put_pages(pages + i, left, offset);
1208− return -EINVAL;
1209− }
1210− __bio_add_page(bio, page, len, offset);
1211− }
1212− offset = 0;
1213− }
1214−
1215− iov_iter_advance(iter, size);
1216− return 0;
1217−}
1218−
1219−static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter)
1220−{
1221− unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
1222− unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
1223− struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1224− unsigned int max_append_sectors = queue_max_zone_append_sectors(q);
1225− struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
1226− struct page **pages = (struct page **)bv;
1227− ssize_t size, left;
1228− unsigned len, i;
1229− size_t offset;
1230− int ret = 0;
1231−
1232− if (WARN_ON_ONCE(!max_append_sectors))
1233− return 0;
1234−
12351223 /*
1236− * Move page array up in the allocated memory for the bio vecs as far as
1237− * possible so that we can start filling biovecs from the beginning
1238− * without overwriting the temporary page array.
1224+ * Each segment in the iov is required to be a block size multiple.
1225+ * However, we may not be able to get the entire segment if it spans
1226+ * more pages than bi_max_vecs allows, so we have to ALIGN_DOWN the
1227+ * result to ensure the bio's total size is correct. The remainder of
1228+ * the iov data will be picked up in the next bio iteration.
12391229 */
1240− BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
1241− pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
1242−
12431230 size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
1231+ if (size > 0)
1232+ size = ALIGN_DOWN(size, bdev_logical_block_size(bio->bi_bdev));
12441233 if (unlikely(size <= 0))
12451234 return size ? size : -EFAULT;
12461235
12471236 for (left = size, i = 0; left > 0; left -= len, i++) {
12481237 struct page *page = pages[i];
1249− bool same_page = false;
1238+ int ret;
12501239
12511240 len = min_t(size_t, PAGE_SIZE - offset, left);
1252− if (bio_add_hw_page(q, bio, page, len, offset,
1253− max_append_sectors, &same_page) != len) {
1241+ if (bio_op(bio) == REQ_OP_ZONE_APPEND)
1242+ ret = bio_iov_add_zone_append_page(bio, page, len,
1243+ offset);
1244+ else
1245+ ret = bio_iov_add_page(bio, page, len, offset);
1246+
1247+ if (ret) {
12541248 bio_put_pages(pages + i, left, offset);
1255− ret = -EINVAL;
1256− break;
1249+ return ret;
12571250 }
1258− if (same_page)
1259− put_page(page);
12601251 offset = 0;
12611252 }
12621253
1263− iov_iter_advance(iter, size - left);
1264− return ret;
1254+ iov_iter_advance(iter, size);
1255+ return 0;
12651256 }
12661257
12671258 /**
@@ -1298,10 +1289,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · 30 unmodified lines
12981289 }
12991290
13001291 do {
1301− if (bio_op(bio) == REQ_OP_ZONE_APPEND)
1302− ret = __bio_iov_append_get_pages(bio, iter);
1303− else
1304− ret = __bio_iov_iter_get_pages(bio, iter);
1292+ ret = __bio_iov_iter_get_pages(bio, iter);
13051293 } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
13061294
13071295 /* don't account direct I/O as memory stall */
block/blk-cgroup-rwstat.h+4−4
@@ -59,20 +59,20 @@ void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, · 58 unmodified lines
5959 * caller is responsible for synchronizing calls to this function.
6060 */
6161 static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
62− unsigned int op, uint64_t val)
62+ blk_opf_t opf, uint64_t val)
6363 {
6464 struct percpu_counter *cnt;
6565
66− if (op_is_discard(op))
66+ if (op_is_discard(opf))
6767 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
68− else if (op_is_write(op))
68+ else if (op_is_write(opf))
6969 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
7070 else
7171 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
7272
7373 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
7474
75− if (op_is_sync(op))
75+ if (op_is_sync(opf))
7676 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
7777 else
7878 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
block/blk-cgroup.c+40−36
@@ -846,6 +846,21 @@ static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src) · 845 unmodified lines
846846 }
847847 }
848848
849+static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
850+ struct blkg_iostat *last)
851+{
852+ struct blkg_iostat delta;
853+ unsigned long flags;
854+
855+ /* propagate percpu delta to global */
856+ flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
857+ blkg_iostat_set(&delta, cur);
858+ blkg_iostat_sub(&delta, last);
859+ blkg_iostat_add(&blkg->iostat.cur, &delta);
860+ blkg_iostat_add(last, &delta);
861+ u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
862+}
863+
849864 static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
850865 {
851866 struct blkcg *blkcg = css_to_blkcg(css);
@@ -860,8 +875,7 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) · 8 unmodified lines
860875 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
861876 struct blkcg_gq *parent = blkg->parent;
862877 struct blkg_iostat_set *bisc = per_cpu_ptr(blkg->iostat_cpu, cpu);
863− struct blkg_iostat cur, delta;
864− unsigned long flags;
878+ struct blkg_iostat cur;
865879 unsigned int seq;
866880
867881 /* fetch the current per-cpu values */
@@ -870,23 +884,12 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) · 2 unmodified lines
870884 blkg_iostat_set(&cur, &bisc->cur);
871885 } while (u64_stats_fetch_retry(&bisc->sync, seq));
872886
873− /* propagate percpu delta to global */
874− flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
875− blkg_iostat_set(&delta, &cur);
876− blkg_iostat_sub(&delta, &bisc->last);
877− blkg_iostat_add(&blkg->iostat.cur, &delta);
878− blkg_iostat_add(&bisc->last, &delta);
879− u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
887+ blkcg_iostat_update(blkg, &cur, &bisc->last);
880888
881889 /* propagate global delta to parent (unless that's root) */
882− if (parent && parent->parent) {
883− flags = u64_stats_update_begin_irqsave(&parent->iostat.sync);
884− blkg_iostat_set(&delta, &blkg->iostat.cur);
885− blkg_iostat_sub(&delta, &blkg->iostat.last);
886− blkg_iostat_add(&parent->iostat.cur, &delta);
887− blkg_iostat_add(&blkg->iostat.last, &delta);
888− u64_stats_update_end_irqrestore(&parent->iostat.sync, flags);
889− }
890+ if (parent && parent->parent)
891+ blkcg_iostat_update(parent, &blkg->iostat.cur,
892+ &blkg->iostat.last);
890893 }
891894
892895 rcu_read_unlock();
@@ -1299,6 +1302,7 @@ int blkcg_init_queue(struct request_queue *q) · 406 unmodified lines
12991302 ret = blk_iolatency_init(q);
13001303 if (ret) {
13011304 blk_throtl_exit(q);
1305+ blk_ioprio_exit(q);
13021306 goto err_destroy_all;
13031307 }
13041308
@@ -1529,6 +1533,18 @@ void blkcg_deactivate_policy(struct request_queue *q, · 224 unmodified lines
15291533 }
15301534 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
15311535
1536+static void blkcg_free_all_cpd(struct blkcg_policy *pol)
1537+{
1538+ struct blkcg *blkcg;
1539+
1540+ list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1541+ if (blkcg->cpd[pol->plid]) {
1542+ pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1543+ blkcg->cpd[pol->plid] = NULL;
1544+ }
1545+ }
1546+}
1547+
15321548 /**
15331549 * blkcg_policy_register - register a blkcg policy
15341550 * @pol: blkcg policy to register
@@ -1593,14 +1609,9 @@ int blkcg_policy_register(struct blkcg_policy *pol) · 58 unmodified lines
15931609 return 0;
15941610
15951611 err_free_cpds:
1596− if (pol->cpd_free_fn) {
1597− list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1598− if (blkcg->cpd[pol->plid]) {
1599− pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1600− blkcg->cpd[pol->plid] = NULL;
1601− }
1602− }
1603− }
1612+ if (pol->cpd_free_fn)
1613+ blkcg_free_all_cpd(pol);
1614+
16041615 blkcg_policy[pol->plid] = NULL;
16051616 err_unlock:
16061617 mutex_unlock(&blkcg_pol_mutex);
@@ -1617,8 +1628,6 @@ EXPORT_SYMBOL_GPL(blkcg_policy_register); · 10 unmodified lines
16171628 */
16181629 void blkcg_policy_unregister(struct blkcg_policy *pol)
16191630 {
1620− struct blkcg *blkcg;
1621−
16221631 mutex_lock(&blkcg_pol_register_mutex);
16231632
16241633 if (WARN_ON(blkcg_policy[pol->plid] != pol))
@@ -1633,14 +1642,9 @@ void blkcg_policy_unregister(struct blkcg_policy *pol) · 8 unmodified lines
16331642 /* remove cpds and unregister */
16341643 mutex_lock(&blkcg_pol_mutex);
16351644
1636− if (pol->cpd_free_fn) {
1637− list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1638− if (blkcg->cpd[pol->plid]) {
1639− pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1640− blkcg->cpd[pol->plid] = NULL;
1641− }
1642− }
1643− }
1645+ if (pol->cpd_free_fn)
1646+ blkcg_free_all_cpd(pol);
1647+
16441648 blkcg_policy[pol->plid] = NULL;
16451649
16461650 mutex_unlock(&blkcg_pol_mutex);
@@ -1696,7 +1700,7 @@ static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now) · 49 unmodified lines
16961700 * everybody is happy with their IO latencies.
16971701 */
16981702 if (time_before64(old + NSEC_PER_SEC, now) &&
1699− atomic64_cmpxchg(&blkg->delay_start, old, now) == old) {
1703+ atomic64_try_cmpxchg(&blkg->delay_start, &old, now)) {
17001704 u64 cur = atomic64_read(&blkg->delay_nsec);
17011705 u64 sub = min_t(u64, blkg->last_delay, now - old);
17021706 int cur_use = atomic_read(&blkg->use_delay);
block/blk-cgroup.h+4−8
@@ -430,12 +430,8 @@ static inline int blkcg_unuse_delay(struct blkcg_gq *blkg) · 429 unmodified lines
430430 * then check to see if we were the last delay so we can drop the
431431 * congestion count on the cgroup.
432432 */
433− while (old) {
434− int cur = atomic_cmpxchg(&blkg->use_delay, old, old - 1);
435− if (cur == old)
436− break;
437− old = cur;
438− }
433+ while (old && !atomic_try_cmpxchg(&blkg->use_delay, &old, old - 1))
434+ ;
439435
440436 if (old == 0)
441437 return 0;
@@ -458,7 +454,7 @@ static inline void blkcg_set_delay(struct blkcg_gq *blkg, u64 delay) · 16 unmodified lines
458454 int old = atomic_read(&blkg->use_delay);
459455
460456 /* We only want 1 person setting the congestion count for this blkg. */
461− if (!old && atomic_cmpxchg(&blkg->use_delay, old, -1) == old)
457+ if (!old && atomic_try_cmpxchg(&blkg->use_delay, &old, -1))
462458 atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);
463459
464460 atomic64_set(&blkg->delay_nsec, delay);
@@ -475,7 +471,7 @@ static inline void blkcg_clear_delay(struct blkcg_gq *blkg) · 10 unmodified lines
475471 int old = atomic_read(&blkg->use_delay);
476472
477473 /* We only want 1 person clearing the congestion count for this blkg. */
478− if (old && atomic_cmpxchg(&blkg->use_delay, old, 0) == old)
474+ if (old && atomic_try_cmpxchg(&blkg->use_delay, &old, 0))
479475 atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);
480476 }
481477
block/blk-core.c+17−63
@@ -136,7 +136,7 @@ static const char *const blk_op_name[] = { · 135 unmodified lines
136136 * string format. Useful in the debugging and tracing bio or request. For
137137 * invalid REQ_OP_XXX it returns string "UNKNOWN".
138138 */
139−inline const char *blk_op_str(unsigned int op)
139+inline const char *blk_op_str(enum req_op op)
140140 {
141141 const char *op_str = "UNKNOWN";
142142
@@ -284,49 +284,6 @@ void blk_queue_start_drain(struct request_queue *q) · 141 unmodified lines
284284 wake_up_all(&q->mq_freeze_wq);
285285 }
286286
287−/**
288− * blk_cleanup_queue - shutdown a request queue
289− * @q: request queue to shutdown
290− *
291− * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
292− * put it. All future requests will be failed immediately with -ENODEV.
293− *
294− * Context: can sleep
295− */
296−void blk_cleanup_queue(struct request_queue *q)
297−{
298− /* cannot be called from atomic context */
299− might_sleep();
300−
301− WARN_ON_ONCE(blk_queue_registered(q));
302−
303− /* mark @q DYING, no new request or merges will be allowed afterwards */
304− blk_queue_flag_set(QUEUE_FLAG_DYING, q);
305− blk_queue_start_drain(q);
306−
307− blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
308− blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
309−
310− /*
311− * Drain all requests queued before DYING marking. Set DEAD flag to
312− * prevent that blk_mq_run_hw_queues() accesses the hardware queues
313− * after draining finished.
314− */
315− blk_freeze_queue(q);
316−
317− blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
318−
319− blk_sync_queue(q);
320− if (queue_is_mq(q)) {
321− blk_mq_cancel_work_sync(q);
322− blk_mq_exit_queue(q);
323− }
324−
325− /* @q is and will stay empty, shutdown and put */
326− blk_put_queue(q);
327−}
328−EXPORT_SYMBOL(blk_cleanup_queue);
329−
330287 /**
331288 * blk_queue_enter() - try to increase q->q_usage_counter
332289 * @q: request queue pointer
@@ -435,7 +392,7 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu) · 102 unmodified lines
435392
436393 q->last_merge = NULL;
437394
438− q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL);
395+ q->id = ida_alloc(&blk_queue_ida, GFP_KERNEL);
439396 if (q->id < 0)
440397 goto fail_srcu;
441398
@@ -485,7 +442,7 @@ fail_stats: · 43 unmodified lines
485442 fail_split:
486443 bioset_exit(&q->bio_split);
487444 fail_id:
488− ida_simple_remove(&blk_queue_ida, q->id);
445+ ida_free(&blk_queue_ida, q->id);
489446 fail_srcu:
490447 if (alloc_srcu)
491448 cleanup_srcu_struct(q->srcu);
@@ -504,12 +461,10 @@ fail_q: · 12 unmodified lines
504461 */
505462 bool blk_get_queue(struct request_queue *q)
506463 {
507− if (likely(!blk_queue_dying(q))) {
508− __blk_get_queue(q);
509− return true;
510− }
511−
512− return false;
464+ if (unlikely(blk_queue_dying(q)))
465+ return false;
466+ kobject_get(&q->kobj);
467+ return true;
513468 }
514469 EXPORT_SYMBOL(blk_get_queue);
515470
@@ -608,16 +563,15 @@ static int blk_partition_remap(struct bio *bio) · 92 unmodified lines
608563 static inline blk_status_t blk_check_zone_append(struct request_queue *q,
609564 struct bio *bio)
610565 {
611− sector_t pos = bio->bi_iter.bi_sector;
612566 int nr_sectors = bio_sectors(bio);
613567
614568 /* Only applicable to zoned block devices */
615− if (!blk_queue_is_zoned(q))
569+ if (!bdev_is_zoned(bio->bi_bdev))
616570 return BLK_STS_NOTSUPP;
617571
618572 /* The bio sector must point to the start of a sequential zone */
619− if (pos & (blk_queue_zone_sectors(q) - 1) ||
620− !blk_queue_zone_is_seq(q, pos))
573+ if (bio->bi_iter.bi_sector & (bdev_zone_sectors(bio->bi_bdev) - 1) ||
574+ !bio_zone_is_seq(bio))
621575 return BLK_STS_IOERR;
622576
623577 /*
@@ -762,7 +716,7 @@ void submit_bio_noacct(struct bio *bio) · 138 unmodified lines
762716
763717 might_sleep();
764718
765− plug = blk_mq_plug(q, bio);
719+ plug = blk_mq_plug(bio);
766720 if (plug && plug->nowait)
767721 bio->bi_opf |= REQ_NOWAIT;
768722
@@ -818,11 +772,11 @@ void submit_bio_noacct(struct bio *bio) · 49 unmodified lines
818772 case REQ_OP_ZONE_OPEN:
819773 case REQ_OP_ZONE_CLOSE:
820774 case REQ_OP_ZONE_FINISH:
821− if (!blk_queue_is_zoned(q))
775+ if (!bdev_is_zoned(bio->bi_bdev))
822776 goto not_supported;
823777 break;
824778 case REQ_OP_ZONE_RESET_ALL:
825− if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q))
779+ if (!bdev_is_zoned(bio->bi_bdev) || !blk_queue_zone_resetall(q))
826780 goto not_supported;
827781 break;
828782 case REQ_OP_WRITE_ZEROES:
@@ -987,7 +941,7 @@ void update_io_ticks(struct block_device *part, unsigned long now, bool end) · 158 unmodified lines
987941 again:
988942 stamp = READ_ONCE(part->bd_stamp);
989943 if (unlikely(time_after(now, stamp))) {
990− if (likely(cmpxchg(&part->bd_stamp, stamp, now) == stamp))
944+ if (likely(try_cmpxchg(&part->bd_stamp, &stamp, now)))
991945 __part_stat_add(part, io_ticks, end ? now - stamp : 1);
992946 }
993947 if (part->bd_partno) {
@@ -997,7 +951,7 @@ again: · 3 unmodified lines
997951 }
998952
999953 unsigned long bdev_start_io_acct(struct block_device *bdev,
1000− unsigned int sectors, unsigned int op,
954+ unsigned int sectors, enum req_op op,
1001955 unsigned long start_time)
1002956 {
1003957 const int sgrp = op_stat_group(op);
@@ -1038,7 +992,7 @@ unsigned long bio_start_io_acct(struct bio *bio) · 34 unmodified lines
1038992 }
1039993 EXPORT_SYMBOL_GPL(bio_start_io_acct);
1040994
1041−void bdev_end_io_acct(struct block_device *bdev, unsigned int op,
995+void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
1042996 unsigned long start_time)
1043997 {
1044998 const int sgrp = op_stat_group(op);
@@ -1247,7 +1201,7 @@ EXPORT_SYMBOL_GPL(blk_io_schedule); · 202 unmodified lines
12471201
12481202 int __init blk_dev_init(void)
12491203 {
1250− BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
1204+ BUILD_BUG_ON((__force u32)REQ_OP_LAST >= (1 << REQ_OP_BITS));
12511205 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
12521206 sizeof_field(struct request, cmd_flags));
12531207 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
block/blk-flush.c+3−3
@@ -94,7 +94,7 @@ enum { · 93 unmodified lines
9494 };
9595
9696 static void blk_kick_flush(struct request_queue *q,
97− struct blk_flush_queue *fq, unsigned int flags);
97+ struct blk_flush_queue *fq, blk_opf_t flags);
9898
9999 static inline struct blk_flush_queue *
100100 blk_get_flush_queue(struct request_queue *q, struct blk_mq_ctx *ctx)
@@ -173,7 +173,7 @@ static void blk_flush_complete_seq(struct request *rq, · 72 unmodified lines
173173 {
174174 struct request_queue *q = rq->q;
175175 struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
176− unsigned int cmd_flags;
176+ blk_opf_t cmd_flags;
177177
178178 BUG_ON(rq->flush.seq & seq);
179179 rq->flush.seq |= seq;
@@ -290,7 +290,7 @@ bool is_flush_rq(struct request *rq) · 110 unmodified lines
290290 *
291291 */
292292 static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq,
293− unsigned int flags)
293+ blk_opf_t flags)
294294 {
295295 struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
296296 struct request *first_rq =
block/blk-ia-ranges.c+20−45
@@ -102,31 +102,18 @@ static struct kobj_type blk_ia_ranges_ktype = { · 101 unmodified lines
102102 * disk_register_independent_access_ranges - register with sysfs a set of
103103 * independent access ranges
104104 * @disk: Target disk
105− * @new_iars: New set of independent access ranges
106105 *
107106 * Register with sysfs a set of independent access ranges for @disk.
108− * If @new_iars is not NULL, this set of ranges is registered and the old set
109− * specified by q->ia_ranges is unregistered. Otherwise, q->ia_ranges is
110− * registered if it is not already.
111107 */
112−int disk_register_independent_access_ranges(struct gendisk *disk,
113− struct blk_independent_access_ranges *new_iars)
108+int disk_register_independent_access_ranges(struct gendisk *disk)
114109 {
110+ struct blk_independent_access_ranges *iars = disk->ia_ranges;
115111 struct request_queue *q = disk->queue;
116− struct blk_independent_access_ranges *iars;
117112 int i, ret;
118113
119114 lockdep_assert_held(&q->sysfs_dir_lock);
120115 lockdep_assert_held(&q->sysfs_lock);
121116
122− /* If a new range set is specified, unregister the old one */
123− if (new_iars) {
124− if (q->ia_ranges)
125− disk_unregister_independent_access_ranges(disk);
126− q->ia_ranges = new_iars;
127− }
128−
129− iars = q->ia_ranges;
130117 if (!iars)
131118 return 0;
132119
@@ -138,7 +125,7 @@ int disk_register_independent_access_ranges(struct gendisk *disk, · 5 unmodified lines
138125 ret = kobject_init_and_add(&iars->kobj, &blk_ia_ranges_ktype,
139126 &q->kobj, "%s", "independent_access_ranges");
140127 if (ret) {
141− q->ia_ranges = NULL;
128+ disk->ia_ranges = NULL;
142129 kobject_put(&iars->kobj);
143130 return ret;
144131 }
@@ -164,7 +151,7 @@ int disk_register_independent_access_ranges(struct gendisk *disk, · 19 unmodified lines
164151 void disk_unregister_independent_access_ranges(struct gendisk *disk)
165152 {
166153 struct request_queue *q = disk->queue;
167− struct blk_independent_access_ranges *iars = q->ia_ranges;
154+ struct blk_independent_access_ranges *iars = disk->ia_ranges;
168155 int i;
169156
170157 lockdep_assert_held(&q->sysfs_dir_lock);
@@ -182,7 +169,7 @@ void disk_unregister_independent_access_ranges(struct gendisk *disk) · 11 unmodified lines
182169 kfree(iars);
183170 }
184171
185− q->ia_ranges = NULL;
172+ disk->ia_ranges = NULL;
186173 }
187174
188175 static struct blk_independent_access_range *
@@ -210,6 +197,9 @@ static bool disk_check_ia_ranges(struct gendisk *disk, · 21 unmodified lines
210197 sector_t sector = 0;
211198 int i;
212199
200+ if (WARN_ON_ONCE(!iars->nr_ia_ranges))
201+ return false;
202+
213203 /*
214204 * While sorting the ranges in increasing LBA order, check that the
215205 * ranges do not overlap, that there are no sector holes and that all
@@ -242,7 +232,7 @@ static bool disk_check_ia_ranges(struct gendisk *disk, · 26 unmodified lines
242232 static bool disk_ia_ranges_changed(struct gendisk *disk,
243233 struct blk_independent_access_ranges *new)
244234 {
245− struct blk_independent_access_ranges *old = disk->queue->ia_ranges;
235+ struct blk_independent_access_ranges *old = disk->ia_ranges;
246236 int i;
247237
248238 if (!old)
@@ -298,25 +288,15 @@ void disk_set_independent_access_ranges(struct gendisk *disk, · 49 unmodified lines
298288 {
299289 struct request_queue *q = disk->queue;
300290
301− if (WARN_ON_ONCE(iars && !iars->nr_ia_ranges)) {
291+ mutex_lock(&q->sysfs_dir_lock);
292+ mutex_lock(&q->sysfs_lock);
293+ if (iars && !disk_check_ia_ranges(disk, iars)) {
302294 kfree(iars);
303295 iars = NULL;
304296 }
305−
306− mutex_lock(&q->sysfs_dir_lock);
307− mutex_lock(&q->sysfs_lock);
308−
309− if (iars) {
310− if (!disk_check_ia_ranges(disk, iars)) {
311− kfree(iars);
312− iars = NULL;
313− goto reg;
314− }
315−
316− if (!disk_ia_ranges_changed(disk, iars)) {
317− kfree(iars);
318− goto unlock;
319− }
297+ if (iars && !disk_ia_ranges_changed(disk, iars)) {
298+ kfree(iars);
299+ goto unlock;
320300 }
321301
322302 /*
@@ -324,17 +304,12 @@ void disk_set_independent_access_ranges(struct gendisk *disk, · 1 unmodified line
324304 * revalidation. If that is the case, we need to unregister the old
325305 * set of independent access ranges and register the new set. If the
326306 * queue is not registered, registration of the device request queue
327− * will register the independent access ranges, so only swap in the
328− * new set and free the old one.
307+ * will register the independent access ranges.
329308 */
330−reg:
331− if (blk_queue_registered(q)) {
332− disk_register_independent_access_ranges(disk, iars);
333− } else {
334− swap(q->ia_ranges, iars);
335− kfree(iars);
336− }
337−
309+ disk_unregister_independent_access_ranges(disk);
310+ disk->ia_ranges = iars;
311+ if (blk_queue_registered(q))
312+ disk_register_independent_access_ranges(disk);
338313 unlock:
339314 mutex_unlock(&q->sysfs_lock);
340315 mutex_unlock(&q->sysfs_dir_lock);
block/blk-ioc.c+2−0
@@ -247,6 +247,8 @@ static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) · 246 unmodified lines
247247 INIT_HLIST_HEAD(&ioc->icq_list);
248248 INIT_WORK(&ioc->release_work, ioc_release_fn);
249249 #endif
250+ ioc->ioprio = IOPRIO_DEFAULT;
251+
250252 return ioc;
251253 }
252254
block/blk-iocost.c+14−8
@@ -2769,7 +2769,7 @@ static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq) · 2768 unmodified lines
27692769 if (!ioc->enabled || !rq->alloc_time_ns || !rq->start_time_ns)
27702770 return;
27712771
2772− switch (req_op(rq) & REQ_OP_MASK) {
2772+ switch (req_op(rq)) {
27732773 case REQ_OP_READ:
27742774 pidx = QOS_RLAT;
27752775 rw = READ;
@@ -2886,15 +2886,21 @@ static int blk_iocost_init(struct request_queue *q) · 110 unmodified lines
28862886 * called before policy activation completion, can't assume that the
28872887 * target bio has an iocg associated and need to test for NULL iocg.
28882888 */
2889− rq_qos_add(q, rqos);
2889+ ret = rq_qos_add(q, rqos);
2890+ if (ret)
2891+ goto err_free_ioc;
2892+
28902893 ret = blkcg_activate_policy(q, &blkcg_policy_iocost);
2891− if (ret) {
2892− rq_qos_del(q, rqos);
2893− free_percpu(ioc->pcpu_stat);
2894− kfree(ioc);
2895− return ret;
2896− }
2894+ if (ret)
2895+ goto err_del_qos;
28972896 return 0;
2897+
2898+err_del_qos:
2899+ rq_qos_del(q, rqos);
2900+err_free_ioc:
2901+ free_percpu(ioc->pcpu_stat);
2902+ kfree(ioc);
2903+ return ret;
28982904 }
28992905
29002906 static struct blkcg_policy_data *ioc_cpd_alloc(gfp_t gfp)
block/blk-iolatency.c+16−14
@@ -401,7 +401,6 @@ static void check_scale_change(struct iolatency_grp *iolat) · 400 unmodified lines
401401 unsigned int cur_cookie;
402402 unsigned int our_cookie = atomic_read(&iolat->scale_cookie);
403403 u64 scale_lat;
404− unsigned int old;
405404 int direction = 0;
406405
407406 if (lat_to_blkg(iolat)->parent == NULL)
@@ -422,11 +421,10 @@ static void check_scale_change(struct iolatency_grp *iolat) · 14 unmodified lines
422421 else
423422 return;
424423
425− old = atomic_cmpxchg(&iolat->scale_cookie, our_cookie, cur_cookie);
426−
427− /* Somebody beat us to the punch, just bail. */
428− if (old != our_cookie)
424+ if (!atomic_try_cmpxchg(&iolat->scale_cookie, &our_cookie, cur_cookie)) {
425+ /* Somebody beat us to the punch, just bail. */
429426 return;
427+ }
430428
431429 if (direction < 0 && iolat->min_lat_nsec) {
432430 u64 samples_thresh;
@@ -633,8 +631,8 @@ static void blkcg_iolatency_done_bio(struct rq_qos *rqos, struct bio *bio) · 200 unmodified lines
633631 window_start = atomic64_read(&iolat->window_start);
634632 if (now > window_start &&
635633 (now - window_start) >= iolat->cur_win_nsec) {
636− if (atomic64_cmpxchg(&iolat->window_start,
637− window_start, now) == window_start)
634+ if (atomic64_try_cmpxchg(&iolat->window_start,
635+ &window_start, now))
638636 iolatency_check_latencies(iolat, now);
639637 }
640638 }
@@ -773,19 +771,23 @@ int blk_iolatency_init(struct request_queue *q) · 132 unmodified lines
773771 rqos->ops = &blkcg_iolatency_ops;
774772 rqos->q = q;
775773
776− rq_qos_add(q, rqos);
777−
774+ ret = rq_qos_add(q, rqos);
775+ if (ret)
776+ goto err_free;
778777 ret = blkcg_activate_policy(q, &blkcg_policy_iolatency);
779− if (ret) {
780− rq_qos_del(q, rqos);
781− kfree(blkiolat);
782− return ret;
783− }
778+ if (ret)
779+ goto err_qos_del;
784780
785781 timer_setup(&blkiolat->timer, blkiolatency_timer_fn, 0);
786782 INIT_WORK(&blkiolat->enable_work, blkiolatency_enable_work_fn);
787783
788784 return 0;
785+
786+err_qos_del:
787+ rq_qos_del(q, rqos);
788+err_free:
789+ kfree(blkiolat);
790+ return ret;
789791 }
790792
791793 static void iolatency_set_min_lat_nsec(struct blkcg_gq *blkg, u64 val)
block/blk-ioprio.c+7−50
@@ -62,7 +62,6 @@ struct ioprio_blkg { · 61 unmodified lines
6262 struct ioprio_blkcg {
6363 struct blkcg_policy_data cpd;
6464 enum prio_policy prio_policy;
65− bool prio_set;
6665 };
6766
6867 static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd)
@@ -113,7 +112,6 @@ static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf, · 44 unmodified lines
113112 if (ret < 0)
114113 return ret;
115114 blkcg->prio_policy = ret;
116− blkcg->prio_set = true;
117115 return nbytes;
118116 }
119117
@@ -183,26 +181,20 @@ static struct blkcg_policy ioprio_policy = { · 63 unmodified lines
183181 .pd_free_fn = ioprio_free_pd,
184182 };
185183
186−struct blk_ioprio {
187− struct rq_qos rqos;
188−};
189−
190−static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq,
191− struct bio *bio)
184+void blkcg_set_ioprio(struct bio *bio)
192185 {
193186 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio);
194187 u16 prio;
195188
196− if (!blkcg->prio_set)
189+ if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE)
197190 return;
198191
199192 /*
200193 * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers
201194 * correspond to a lower priority. Hence, the max_t() below selects
202195 * the lower priority of bi_ioprio and the cgroup I/O priority class.
203− * If the cgroup policy has been set to POLICY_NO_CHANGE == 0, the
204− * bio I/O priority is not modified. If the bio I/O priority equals
205− * IOPRIO_CLASS_NONE, the cgroup I/O priority is assigned to the bio.
196+ * If the bio I/O priority equals IOPRIO_CLASS_NONE, the cgroup I/O
197+ * priority is assigned to the bio.
206198 */
207199 prio = max_t(u16, bio->bi_ioprio,
208200 IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0));
@@ -210,49 +202,14 @@ static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq, · 1 unmodified line
210202 bio->bi_ioprio = prio;
211203 }
212204
213−static void blkcg_ioprio_exit(struct rq_qos *rqos)
205+void blk_ioprio_exit(struct request_queue *q)
214206 {
215− struct blk_ioprio *blkioprio_blkg =
216− container_of(rqos, typeof(*blkioprio_blkg), rqos);
217−
218− blkcg_deactivate_policy(rqos->q, &ioprio_policy);
219− kfree(blkioprio_blkg);
207+ blkcg_deactivate_policy(q, &ioprio_policy);
220208 }
221209
222−static struct rq_qos_ops blkcg_ioprio_ops = {
223− .track = blkcg_ioprio_track,
224− .exit = blkcg_ioprio_exit,
225−};
226−
227210 int blk_ioprio_init(struct request_queue *q)
228211 {
229− struct blk_ioprio *blkioprio_blkg;
230− struct rq_qos *rqos;
231− int ret;
232−
233− blkioprio_blkg = kzalloc(sizeof(*blkioprio_blkg), GFP_KERNEL);
234− if (!blkioprio_blkg)
235− return -ENOMEM;
236−
237− ret = blkcg_activate_policy(q, &ioprio_policy);
238− if (ret) {
239− kfree(blkioprio_blkg);
240− return ret;
241− }
242−
243− rqos = &blkioprio_blkg->rqos;
244− rqos->id = RQ_QOS_IOPRIO;
245− rqos->ops = &blkcg_ioprio_ops;
246− rqos->q = q;
247−
248− /*
249− * Registering the rq-qos policy after activating the blk-cgroup
250− * policy guarantees that ioprio_blkcg_from_bio(bio) != NULL in the
251− * rq-qos callbacks.
252− */
253− rq_qos_add(q, rqos);
254−
255− return 0;
212+ return blkcg_activate_policy(q, &ioprio_policy);
256213 }
257214
258215 static int __init ioprio_init(void)
block/blk-ioprio.h+9−0
@@ -6,14 +6,23 @@ · 5 unmodified lines
66 #include <linux/kconfig.h>
77
88 struct request_queue;
9+struct bio;
910
1011 #ifdef CONFIG_BLK_CGROUP_IOPRIO
1112 int blk_ioprio_init(struct request_queue *q);
13+void blk_ioprio_exit(struct request_queue *q);
14+void blkcg_set_ioprio(struct bio *bio);
1215 #else
1316 static inline int blk_ioprio_init(struct request_queue *q)
1417 {
1518 return 0;
1619 }
20+static inline void blk_ioprio_exit(struct request_queue *q)
21+{
22+}
23+static inline void blkcg_set_ioprio(struct bio *bio)
24+{
25+}
1726 #endif
1827
1928 #endif /* _BLK_IOPRIO_H_ */
block/blk-lib.c+2−4
@@ -48,10 +48,8 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, · 47 unmodified lines
4848
4949 /* In case the discard granularity isn't set by buggy device driver */
5050 if (WARN_ON_ONCE(!bdev_discard_granularity(bdev))) {
51− char dev_name[BDEVNAME_SIZE];
52−
53− bdevname(bdev, dev_name);
54− pr_err_ratelimited("%s: Error: discard_granularity is 0.\n", dev_name);
51+ pr_err_ratelimited("%pg: Error: discard_granularity is 0.\n",
52+ bdev);
5553 return -EOPNOTSUPP;
5654 }
5755
block/blk-merge.c+46−33
@@ -164,18 +164,21 @@ static struct bio *blk_bio_write_zeroes_split(struct request_queue *q, · 163 unmodified lines
164164 static inline unsigned get_max_io_size(struct request_queue *q,
165165 struct bio *bio)
166166 {
167− unsigned sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector, 0);
168− unsigned max_sectors = sectors;
169167 unsigned pbs = queue_physical_block_size(q) >> SECTOR_SHIFT;
170168 unsigned lbs = queue_logical_block_size(q) >> SECTOR_SHIFT;
171− unsigned start_offset = bio->bi_iter.bi_sector & (pbs - 1);
169+ unsigned max_sectors = queue_max_sectors(q), start, end;
172170
173− max_sectors += start_offset;
174− max_sectors &= ~(pbs - 1);
175− if (max_sectors > start_offset)
176− return max_sectors - start_offset;
171+ if (q->limits.chunk_sectors) {
172+ max_sectors = min(max_sectors,
173+ blk_chunk_sectors_left(bio->bi_iter.bi_sector,
174+ q->limits.chunk_sectors));
175+ }
177176
178− return sectors & ~(lbs - 1);
177+ start = bio->bi_iter.bi_sector & (pbs - 1);
178+ end = (start + max_sectors) & ~(pbs - 1);
179+ if (end > start)
180+ return end - start;
181+ return max_sectors & ~(lbs - 1);
179182 }
180183
181184 static inline unsigned get_max_segment_size(const struct request_queue *q,
@@ -201,11 +204,11 @@ static inline unsigned get_max_segment_size(const struct request_queue *q, · 19 unmodified lines
201204 * @nsegs: [in,out] Number of segments in the bio being built. Incremented
202205 * by the number of segments from @bv that may be appended to that
203206 * bio without exceeding @max_segs
204− * @sectors: [in,out] Number of sectors in the bio being built. Incremented
205− * by the number of sectors from @bv that may be appended to that
206− * bio without exceeding @max_sectors
207+ * @bytes: [in,out] Number of bytes in the bio being built. Incremented
208+ * by the number of bytes from @bv that may be appended to that
209+ * bio without exceeding @max_bytes
207210 * @max_segs: [in] upper bound for *@nsegs
208− * @max_sectors: [in] upper bound for *@sectors
211+ * @max_bytes: [in] upper bound for *@bytes
209212 *
210213 * When splitting a bio, it can happen that a bvec is encountered that is too
211214 * big to fit in a single segment and hence that it has to be split in the
@@ -216,10 +219,10 @@ static inline unsigned get_max_segment_size(const struct request_queue *q, · 4 unmodified lines
216219 */
217220 static bool bvec_split_segs(const struct request_queue *q,
218221 const struct bio_vec *bv, unsigned *nsegs,
219− unsigned *sectors, unsigned max_segs,
220− unsigned max_sectors)
222+ unsigned *bytes, unsigned max_segs,
223+ unsigned max_bytes)
221224 {
222− unsigned max_len = (min(max_sectors, UINT_MAX >> 9) - *sectors) << 9;
225+ unsigned max_len = min(max_bytes, UINT_MAX) - *bytes;
223226 unsigned len = min(bv->bv_len, max_len);
224227 unsigned total_len = 0;
225228 unsigned seg_size = 0;
@@ -237,7 +240,7 @@ static bool bvec_split_segs(const struct request_queue *q, · 11 unmodified lines
237240 break;
238241 }
239242
240− *sectors += total_len >> 9;
243+ *bytes += total_len;
241244
242245 /* tell the caller to split the bvec if it is too big to fit */
243246 return len > 0 || bv->bv_len > max_len;
@@ -269,8 +272,8 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, · 25 unmodified lines
269272 {
270273 struct bio_vec bv, bvprv, *bvprvp = NULL;
271274 struct bvec_iter iter;
272− unsigned nsegs = 0, sectors = 0;
273− const unsigned max_sectors = get_max_io_size(q, bio);
275+ unsigned nsegs = 0, bytes = 0;
276+ const unsigned max_bytes = get_max_io_size(q, bio) << 9;
274277 const unsigned max_segs = queue_max_segments(q);
275278
276279 bio_for_each_bvec(bv, bio, iter) {
@@ -282,12 +285,12 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, · 5 unmodified lines
282285 goto split;
283286
284287 if (nsegs < max_segs &&
285− sectors + (bv.bv_len >> 9) <= max_sectors &&
288+ bytes + bv.bv_len <= max_bytes &&
286289 bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
287290 nsegs++;
288− sectors += bv.bv_len >> 9;
289− } else if (bvec_split_segs(q, &bv, &nsegs, &sectors, max_segs,
290− max_sectors)) {
291+ bytes += bv.bv_len;
292+ } else if (bvec_split_segs(q, &bv, &nsegs, &bytes, max_segs,
293+ max_bytes)) {
291294 goto split;
292295 }
293296
@@ -300,13 +303,20 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, · 6 unmodified lines
300303 split:
301304 *segs = nsegs;
302305
306+ /*
307+ * Individual bvecs might not be logical block aligned. Round down the
308+ * split size so that each bio is properly block size aligned, even if
309+ * we do not use the full hardware limits.
310+ */
311+ bytes = ALIGN_DOWN(bytes, queue_logical_block_size(q));
312+
303313 /*
304314 * Bio splitting may cause subtle trouble such as hang when doing sync
305315 * iopoll in direct IO routine. Given performance gain of iopoll for
306316 * big IO can be trival, disable iopoll when split needed.
307317 */
308318 bio_clear_polled(bio);
309− return bio_split(bio, sectors, GFP_NOIO, bs);
319+ return bio_split(bio, bytes >> SECTOR_SHIFT, GFP_NOIO, bs);
310320 }
311321
312322 /**
@@ -376,7 +386,7 @@ EXPORT_SYMBOL(blk_queue_split); · 63 unmodified lines
376386 unsigned int blk_recalc_rq_segments(struct request *rq)
377387 {
378388 unsigned int nr_phys_segs = 0;
379− unsigned int nr_sectors = 0;
389+ unsigned int bytes = 0;
380390 struct req_iterator iter;
381391 struct bio_vec bv;
382392
@@ -396,10 +406,12 @@ unsigned int blk_recalc_rq_segments(struct request *rq) · 13 unmodified lines
396406 return 1;
397407 case REQ_OP_WRITE_ZEROES:
398408 return 0;
409+ default:
410+ break;
399411 }
400412
401413 rq_for_each_bvec(bv, rq, iter)
402− bvec_split_segs(rq->q, &bv, &nr_phys_segs, &nr_sectors,
414+ bvec_split_segs(rq->q, &bv, &nr_phys_segs, &bytes,
403415 UINT_MAX, UINT_MAX);
404416 return nr_phys_segs;
405417 }
@@ -560,17 +572,18 @@ static inline unsigned int blk_rq_get_max_sectors(struct request *rq, · 154 unmodified lines
560572 sector_t offset)
561573 {
562574 struct request_queue *q = rq->q;
575+ unsigned int max_sectors;
563576
564577 if (blk_rq_is_passthrough(rq))
565578 return q->limits.max_hw_sectors;
566579
580+ max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
567581 if (!q->limits.chunk_sectors ||
568582 req_op(rq) == REQ_OP_DISCARD ||
569583 req_op(rq) == REQ_OP_SECURE_ERASE)
570− return blk_queue_get_max_sectors(q, req_op(rq));
571−
572− return min(blk_max_size_offset(q, offset, 0),
573− blk_queue_get_max_sectors(q, req_op(rq)));
584+ return max_sectors;
585+ return min(max_sectors,
586+ blk_chunk_sectors_left(offset, q->limits.chunk_sectors));
574587 }
575588
576589 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
@@ -700,7 +713,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, · 123 unmodified lines
700713 */
701714 void blk_rq_set_mixed_merge(struct request *rq)
702715 {
703− unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
716+ blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
704717 struct bio *bio;
705718
706719 if (rq->rq_flags & RQF_MIXED_MERGE)
@@ -916,7 +929,7 @@ enum bio_merge_status { · 209 unmodified lines
916929 static enum bio_merge_status bio_attempt_back_merge(struct request *req,
917930 struct bio *bio, unsigned int nr_segs)
918931 {
919− const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
932+ const blk_opf_t ff = bio->bi_opf & REQ_FAILFAST_MASK;
920933
921934 if (!ll_back_merge_fn(req, bio, nr_segs))
922935 return BIO_MERGE_FAILED;
@@ -940,7 +953,7 @@ static enum bio_merge_status bio_attempt_back_merge(struct request *req, · 17 unmodified lines
940953 static enum bio_merge_status bio_attempt_front_merge(struct request *req,
941954 struct bio *bio, unsigned int nr_segs)
942955 {
943− const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
956+ const blk_opf_t ff = bio->bi_opf & REQ_FAILFAST_MASK;
944957
945958 if (!ll_front_merge_fn(req, bio, nr_segs))
946959 return BIO_MERGE_FAILED;
@@ -1041,7 +1054,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, · 94 unmodified lines
10411054 struct blk_plug *plug;
10421055 struct request *rq;
10431056
1044− plug = blk_mq_plug(q, bio);
1057+ plug = blk_mq_plug(bio);
10451058 if (!plug || rq_list_empty(plug->mq_list))
10461059 return false;
10471060
block/blk-mq-debugfs-zoned.c+3−3
@@ -11,11 +11,11 @@ int queue_zone_wlock_show(void *data, struct seq_file *m) · 10 unmodified lines
1111 struct request_queue *q = data;
1212 unsigned int i;
1313
14− if (!q->seq_zones_wlock)
14+ if (!q->disk->seq_zones_wlock)
1515 return 0;
1616
17− for (i = 0; i < q->nr_zones; i++)
18− if (test_bit(i, q->seq_zones_wlock))
17+ for (i = 0; i < q->disk->nr_zones; i++)
18+ if (test_bit(i, q->disk->seq_zones_wlock))
1919 seq_printf(m, "%u\n", i);
2020
2121 return 0;
block/blk-mq-debugfs.c+10−9
@@ -116,7 +116,6 @@ static const char *const blk_queue_flag_name[] = { · 115 unmodified lines
116116 QUEUE_FLAG_NAME(NOXMERGES),
117117 QUEUE_FLAG_NAME(ADD_RANDOM),
118118 QUEUE_FLAG_NAME(SAME_FORCE),
119− QUEUE_FLAG_NAME(DEAD),
120119 QUEUE_FLAG_NAME(INIT_DONE),
121120 QUEUE_FLAG_NAME(STABLE_WRITES),
122121 QUEUE_FLAG_NAME(POLL),
@@ -151,11 +150,10 @@ static ssize_t queue_state_write(void *data, const char __user *buf, · 28 unmodified lines
151150 char opbuf[16] = { }, *op;
152151
153152 /*
154− * The "state" attribute is removed after blk_cleanup_queue() has called
155− * blk_mq_free_queue(). Return if QUEUE_FLAG_DEAD has been set to avoid
156− * triggering a use-after-free.
153+ * The "state" attribute is removed when the queue is removed. Don't
154+ * allow setting the state on a dying queue to avoid a use-after-free.
157155 */
158− if (blk_queue_dead(q))
156+ if (blk_queue_dying(q))
159157 return -ENOENT;
160158
161159 if (count >= sizeof(opbuf)) {
@@ -306,7 +304,7 @@ static const char *blk_mq_rq_state_name(enum mq_rq_state rq_state) · 144 unmodified lines
306304 int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq)
307305 {
308306 const struct blk_mq_ops *const mq_ops = rq->q->mq_ops;
309− const unsigned int op = req_op(rq);
307+ const enum req_op op = req_op(rq);
310308 const char *op_str = blk_op_str(op);
311309
312310 seq_printf(m, "%p {.op=", rq);
@@ -315,8 +313,8 @@ int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq) · 2 unmodified lines
315313 else
316314 seq_printf(m, "%s", op_str);
317315 seq_puts(m, ", .cmd_flags=");
318− blk_flags_show(m, rq->cmd_flags & ~REQ_OP_MASK, cmd_flag_name,
319− ARRAY_SIZE(cmd_flag_name));
316+ blk_flags_show(m, (__force unsigned int)(rq->cmd_flags & ~REQ_OP_MASK),
317+ cmd_flag_name, ARRAY_SIZE(cmd_flag_name));
320318 seq_puts(m, ", .rq_flags=");
321319 blk_flags_show(m, (__force unsigned int)rq->rq_flags, rqf_name,
322320 ARRAY_SIZE(rqf_name));
@@ -377,7 +375,7 @@ struct show_busy_params { · 54 unmodified lines
377375 * e.g. due to a concurrent blk_mq_finish_request() call. Returns true to
378376 * keep iterating requests.
379377 */
380−static bool hctx_show_busy_rq(struct request *rq, void *data, bool reserved)
378+static bool hctx_show_busy_rq(struct request *rq, void *data)
381379 {
382380 const struct show_busy_params *params = data;
383381
@@ -730,6 +728,9 @@ void blk_mq_debugfs_register_hctx(struct request_queue *q, · 346 unmodified lines
730728 char name[20];
731729 int i;
732730
731+ if (!q->debugfs_dir)
732+ return;
733+
733734 snprintf(name, sizeof(name), "hctx%u", hctx->queue_num);
734735 hctx->debugfs_dir = debugfs_create_dir(name, q->debugfs_dir);
735736
block/blk-mq-sysfs.c+22−23
@@ -203,23 +203,6 @@ static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) · 202 unmodified lines
203203 return ret;
204204 }
205205
206−void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
207−{
208− struct blk_mq_hw_ctx *hctx;
209− unsigned long i;
210−
211− lockdep_assert_held(&q->sysfs_dir_lock);
212−
213− queue_for_each_hw_ctx(q, hctx, i)
214− blk_mq_unregister_hctx(hctx);
215−
216− kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
217− kobject_del(q->mq_kobj);
218− kobject_put(&dev->kobj);
219−
220− q->mq_sysfs_init_done = false;
221−}
222−
223206 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
224207 {
225208 kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
@@ -252,16 +235,16 @@ void blk_mq_sysfs_init(struct request_queue *q) · 26 unmodified lines
252235 }
253236 }
254237
255−int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
238+int blk_mq_sysfs_register(struct gendisk *disk)
256239 {
240+ struct request_queue *q = disk->queue;
257241 struct blk_mq_hw_ctx *hctx;
258242 unsigned long i, j;
259243 int ret;
260244
261− WARN_ON_ONCE(!q->kobj.parent);
262245 lockdep_assert_held(&q->sysfs_dir_lock);
263246
264− ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
247+ ret = kobject_add(q->mq_kobj, &disk_to_dev(disk)->kobj, "mq");
265248 if (ret < 0)
266249 goto out;
267250
@@ -286,11 +269,27 @@ unreg: · 18 unmodified lines
286269
287270 kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
288271 kobject_del(q->mq_kobj);
289− kobject_put(&dev->kobj);
290272 return ret;
291273 }
292274
293−void blk_mq_sysfs_unregister(struct request_queue *q)
275+void blk_mq_sysfs_unregister(struct gendisk *disk)
276+{
277+ struct request_queue *q = disk->queue;
278+ struct blk_mq_hw_ctx *hctx;
279+ unsigned long i;
280+
281+ lockdep_assert_held(&q->sysfs_dir_lock);
282+
283+ queue_for_each_hw_ctx(q, hctx, i)
284+ blk_mq_unregister_hctx(hctx);
285+
286+ kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
287+ kobject_del(q->mq_kobj);
288+
289+ q->mq_sysfs_init_done = false;
290+}
291+
292+void blk_mq_sysfs_unregister_hctxs(struct request_queue *q)
294293 {
295294 struct blk_mq_hw_ctx *hctx;
296295 unsigned long i;
@@ -306,7 +305,7 @@ unlock: · 9 unmodified lines
306305 mutex_unlock(&q->sysfs_dir_lock);
307306 }
308307
309−int blk_mq_sysfs_register(struct request_queue *q)
308+int blk_mq_sysfs_register_hctxs(struct request_queue *q)
310309 {
311310 struct blk_mq_hw_ctx *hctx;
312311 unsigned long i;
block/blk-mq-tag.c+12−19
@@ -37,29 +37,25 @@ static void blk_mq_update_wake_batch(struct blk_mq_tags *tags, · 36 unmodified lines
3737 * to get tag when first time, the other shared-tag users could reserve
3838 * budget for it.
3939 */
40−bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
40+void __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
4141 {
4242 unsigned int users;
4343
4444 if (blk_mq_is_shared_tags(hctx->flags)) {
4545 struct request_queue *q = hctx->queue;
4646
47− if (test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags) ||
48− test_and_set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags)) {
49− return true;
50− }
47+ if (test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags))
48+ return;
49+ set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags);
5150 } else {
52− if (test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) ||
53− test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) {
54− return true;
55− }
51+ if (test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
52+ return;
53+ set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state);
5654 }
5755
5856 users = atomic_inc_return(&hctx->tags->active_queues);
5957
6058 blk_mq_update_wake_batch(hctx->tags, users);
61−
62− return true;
6359 }
6460
6561 /*
@@ -266,7 +262,6 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) · 200 unmodified lines
266262 struct blk_mq_hw_ctx *hctx = iter_data->hctx;
267263 struct request_queue *q = iter_data->q;
268264 struct blk_mq_tag_set *set = q->tag_set;
269− bool reserved = iter_data->reserved;
270265 struct blk_mq_tags *tags;
271266 struct request *rq;
272267 bool ret = true;
@@ -276,7 +271,7 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) · 3 unmodified lines
276271 else
277272 tags = hctx->tags;
278273
279− if (!reserved)
274+ if (!iter_data->reserved)
280275 bitnr += tags->nr_reserved_tags;
281276 /*
282277 * We can hit rq == NULL here, because the tagging functions
@@ -287,7 +282,7 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) · 4 unmodified lines
287282 return true;
288283
289284 if (rq->q == q && (!hctx || rq->mq_hctx == hctx))
290− ret = iter_data->fn(rq, iter_data->data, reserved);
285+ ret = iter_data->fn(rq, iter_data->data);
291286 blk_mq_put_rq_ref(rq);
292287 return ret;
293288 }
@@ -337,12 +332,11 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) · 43 unmodified lines
337332 {
338333 struct bt_tags_iter_data *iter_data = data;
339334 struct blk_mq_tags *tags = iter_data->tags;
340− bool reserved = iter_data->flags & BT_TAG_ITER_RESERVED;
341335 struct request *rq;
342336 bool ret = true;
343337 bool iter_static_rqs = !!(iter_data->flags & BT_TAG_ITER_STATIC_RQS);
344338
345− if (!reserved)
339+ if (!(iter_data->flags & BT_TAG_ITER_RESERVED))
346340 bitnr += tags->nr_reserved_tags;
347341
348342 /*
@@ -358,7 +352,7 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) · 9 unmodified lines
358352
359353 if (!(iter_data->flags & BT_TAG_ITER_STARTED) ||
360354 blk_mq_request_started(rq))
361− ret = iter_data->fn(rq, iter_data->data, reserved);
355+ ret = iter_data->fn(rq, iter_data->data);
362356 if (!iter_static_rqs)
363357 blk_mq_put_rq_ref(rq);
364358 return ret;
@@ -448,8 +442,7 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, · 83 unmodified lines
448442 }
449443 EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
450444
451−static bool blk_mq_tagset_count_completed_rqs(struct request *rq,
452− void *data, bool reserved)
445+static bool blk_mq_tagset_count_completed_rqs(struct request *rq, void *data)
453446 {
454447 unsigned *count = data;
455448
block/blk-mq-tag.h+4−6
@@ -47,15 +47,13 @@ enum { · 46 unmodified lines
4747 BLK_MQ_TAG_MAX = BLK_MQ_NO_TAG - 1,
4848 };
4949
50−extern bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *);
50+extern void __blk_mq_tag_busy(struct blk_mq_hw_ctx *);
5151 extern void __blk_mq_tag_idle(struct blk_mq_hw_ctx *);
5252
53−static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
53+static inline void blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
5454 {
55− if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
56− return false;
57−
58− return __blk_mq_tag_busy(hctx);
55+ if (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
56+ __blk_mq_tag_busy(hctx);
5957 }
6058
6159 static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
block/blk-mq.c+78−27
@@ -42,6 +42,7 @@ · 41 unmodified lines
4242 #include "blk-stat.h"
4343 #include "blk-mq-sched.h"
4444 #include "blk-rq-qos.h"
45+#include "blk-ioprio.h"
4546
4647 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
4748
@@ -128,8 +129,7 @@ struct mq_inflight { · 80 unmodified lines
128129 unsigned int inflight[2];
129130 };
130131
131−static bool blk_mq_check_inflight(struct request *rq, void *priv,
132− bool reserved)
132+static bool blk_mq_check_inflight(struct request *rq, void *priv)
133133 {
134134 struct mq_inflight *mi = priv;
135135
@@ -474,6 +474,9 @@ retry: · 338 unmodified lines
474474 if (!(data->rq_flags & RQF_ELV))
475475 blk_mq_tag_busy(data->hctx);
476476
477+ if (data->flags & BLK_MQ_REQ_RESERVED)
478+ data->rq_flags |= RQF_RESV;
479+
477480 /*
478481 * Try batched alloc if we want more than 1 tag.
479482 */
@@ -507,13 +510,13 @@ retry: · 27 unmodified lines
507510 alloc_time_ns);
508511 }
509512
510−struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
513+struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
511514 blk_mq_req_flags_t flags)
512515 {
513516 struct blk_mq_alloc_data data = {
514517 .q = q,
515518 .flags = flags,
516− .cmd_flags = op,
519+ .cmd_flags = opf,
517520 .nr_tags = 1,
518521 };
519522 struct request *rq;
@@ -537,12 +540,12 @@ out_queue_exit: · 17 unmodified lines
537540 EXPORT_SYMBOL(blk_mq_alloc_request);
538541
539542 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
540− unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx)
543+ blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx)
541544 {
542545 struct blk_mq_alloc_data data = {
543546 .q = q,
544547 .flags = flags,
545− .cmd_flags = op,
548+ .cmd_flags = opf,
546549 .nr_tags = 1,
547550 };
548551 u64 alloc_time_ns = 0;
@@ -588,6 +591,9 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, · 39 unmodified lines
588591 else
589592 data.rq_flags |= RQF_ELV;
590593
594+ if (flags & BLK_MQ_REQ_RESERVED)
595+ data.rq_flags |= RQF_RESV;
596+
591597 ret = -EWOULDBLOCK;
592598 tag = blk_mq_get_tag(&data);
593599 if (tag == BLK_MQ_NO_TAG)
@@ -654,7 +660,7 @@ void blk_dump_rq_flags(struct request *rq, char *msg) · 60 unmodified lines
654660 {
655661 printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
656662 rq->q->disk ? rq->q->disk->disk_name : "?",
657− (unsigned long long) rq->cmd_flags);
663+ (__force unsigned long long) rq->cmd_flags);
658664
659665 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
660666 (unsigned long long)blk_rq_pos(rq),
@@ -707,8 +713,9 @@ static void blk_print_req_error(struct request *req, blk_status_t status) · 46 unmodified lines
707713 "phys_seg %u prio class %u\n",
708714 blk_status_to_str(status),
709715 req->q->disk ? req->q->disk->disk_name : "?",
710− blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)),
711− req->cmd_flags & ~REQ_OP_MASK,
716+ blk_rq_pos(req), (__force u32)req_op(req),
717+ blk_op_str(req_op(req)),
718+ (__force u32)(req->cmd_flags & ~REQ_OP_MASK),
712719 req->nr_phys_segments,
713720 IOPRIO_PRIO_CLASS(req->ioprio));
714721 }
@@ -1393,8 +1400,7 @@ void blk_mq_delay_kick_requeue_list(struct request_queue *q, · 678 unmodified lines
13931400 }
13941401 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
13951402
1396−static bool blk_mq_rq_inflight(struct request *rq, void *priv,
1397− bool reserved)
1403+static bool blk_mq_rq_inflight(struct request *rq, void *priv)
13981404 {
13991405 /*
14001406 * If we find a request that isn't idle we know the queue is busy
@@ -1420,13 +1426,13 @@ bool blk_mq_queue_inflight(struct request_queue *q) · 19 unmodified lines
14201426 }
14211427 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
14221428
1423−static void blk_mq_rq_timed_out(struct request *req, bool reserved)
1429+static void blk_mq_rq_timed_out(struct request *req)
14241430 {
14251431 req->rq_flags |= RQF_TIMED_OUT;
14261432 if (req->q->mq_ops->timeout) {
14271433 enum blk_eh_timer_return ret;
14281434
1429− ret = req->q->mq_ops->timeout(req, reserved);
1435+ ret = req->q->mq_ops->timeout(req);
14301436 if (ret == BLK_EH_DONE)
14311437 return;
14321438 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
@@ -1463,7 +1469,7 @@ void blk_mq_put_rq_ref(struct request *rq) · 30 unmodified lines
14631469 __blk_mq_free_request(rq);
14641470 }
14651471
1466−static bool blk_mq_check_expired(struct request *rq, void *priv, bool reserved)
1472+static bool blk_mq_check_expired(struct request *rq, void *priv)
14671473 {
14681474 unsigned long *next = priv;
14691475
@@ -1475,7 +1481,7 @@ static bool blk_mq_check_expired(struct request *rq, void *priv, bool reserved) · 5 unmodified lines
14751481 * from blk_mq_check_expired().
14761482 */
14771483 if (blk_mq_req_expired(rq, next))
1478− blk_mq_rq_timed_out(rq, reserved);
1484+ blk_mq_rq_timed_out(rq);
14791485 return true;
14801486 }
14811487
@@ -2085,14 +2091,10 @@ static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, · 603 unmodified lines
20852091 return;
20862092
20872093 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
2088− int cpu = get_cpu();
2089− if (cpumask_test_cpu(cpu, hctx->cpumask)) {
2094+ if (cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) {
20902095 __blk_mq_run_hw_queue(hctx);
2091− put_cpu();
20922096 return;
20932097 }
2094−
2095− put_cpu();
20962098 }
20972099
20982100 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
@@ -2156,7 +2158,7 @@ static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q) · 57 unmodified lines
21562158 * just causes lock contention inside the scheduler and pointless cache
21572159 * bouncing.
21582160 */
2159− struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, 0, ctx);
2161+ struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];
21602162
21612163 if (!blk_mq_hctx_stopped(hctx))
21622164 return hctx;
@@ -2783,6 +2785,14 @@ static inline struct request *blk_mq_get_cached_request(struct request_queue *q, · 620 unmodified lines
27832785 return rq;
27842786 }
27852787
2788+static void bio_set_ioprio(struct bio *bio)
2789+{
2790+ /* Nobody set ioprio so far? Initialize it based on task's nice value */
2791+ if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE)
2792+ bio->bi_ioprio = get_current_ioprio();
2793+ blkcg_set_ioprio(bio);
2794+}
2795+
27862796 /**
27872797 * blk_mq_submit_bio - Create and send a request to block device.
27882798 * @bio: Bio pointer.
@@ -2799,7 +2809,7 @@ static inline struct request *blk_mq_get_cached_request(struct request_queue *q, · 10 unmodified lines
27992809 void blk_mq_submit_bio(struct bio *bio)
28002810 {
28012811 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
2802− struct blk_plug *plug = blk_mq_plug(q, bio);
2812+ struct blk_plug *plug = blk_mq_plug(bio);
28032813 const int is_sync = op_is_sync(bio->bi_opf);
28042814 struct request *rq;
28052815 unsigned int nr_segs = 1;
@@ -2812,6 +2822,8 @@ void blk_mq_submit_bio(struct bio *bio) · 6 unmodified lines
28122822 if (!bio_integrity_prep(bio))
28132823 return;
28142824
2825+ bio_set_ioprio(bio);
2826+
28152827 rq = blk_mq_get_cached_request(q, plug, &bio, nr_segs);
28162828 if (!rq) {
28172829 if (!bio)
@@ -3276,7 +3288,7 @@ struct rq_iter_data { · 458 unmodified lines
32763288 bool has_rq;
32773289 };
32783290
3279−static bool blk_mq_has_request(struct request *rq, void *data, bool reserved)
3291+static bool blk_mq_has_request(struct request *rq, void *data)
32803292 {
32813293 struct rq_iter_data *iter_data = data;
32823294
@@ -3895,7 +3907,7 @@ static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set, · 612 unmodified lines
38953907 q->queuedata = queuedata;
38963908 ret = blk_mq_init_allocated_queue(set, q);
38973909 if (ret) {
3898− blk_cleanup_queue(q);
3910+ blk_put_queue(q);
38993911 return ERR_PTR(ret);
39003912 }
39013913 return q;
@@ -3907,6 +3919,35 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) · 5 unmodified lines
39073919 }
39083920 EXPORT_SYMBOL(blk_mq_init_queue);
39093921
3922+/**
3923+ * blk_mq_destroy_queue - shutdown a request queue
3924+ * @q: request queue to shutdown
3925+ *
3926+ * This shuts down a request queue allocated by blk_mq_init_queue() and drops
3927+ * the initial reference. All future requests will failed with -ENODEV.
3928+ *
3929+ * Context: can sleep
3930+ */
3931+void blk_mq_destroy_queue(struct request_queue *q)
3932+{
3933+ WARN_ON_ONCE(!queue_is_mq(q));
3934+ WARN_ON_ONCE(blk_queue_registered(q));
3935+
3936+ might_sleep();
3937+
3938+ blk_queue_flag_set(QUEUE_FLAG_DYING, q);
3939+ blk_queue_start_drain(q);
3940+ blk_freeze_queue(q);
3941+
3942+ blk_sync_queue(q);
3943+ blk_mq_cancel_work_sync(q);
3944+ blk_mq_exit_queue(q);
3945+
3946+ /* @q is and will stay empty, shutdown and put */
3947+ blk_put_queue(q);
3948+}
3949+EXPORT_SYMBOL(blk_mq_destroy_queue);
3950+
39103951 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
39113952 struct lock_class_key *lkclass)
39123953 {
@@ -3919,13 +3960,23 @@ struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata, · 6 unmodified lines
39193960
39203961 disk = __alloc_disk_node(q, set->numa_node, lkclass);
39213962 if (!disk) {
3922− blk_cleanup_queue(q);
3963+ blk_mq_destroy_queue(q);
39233964 return ERR_PTR(-ENOMEM);
39243965 }
3966+ set_bit(GD_OWNS_QUEUE, &disk->state);
39253967 return disk;
39263968 }
39273969 EXPORT_SYMBOL(__blk_mq_alloc_disk);
39283970
3971+struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
3972+ struct lock_class_key *lkclass)
3973+{
3974+ if (!blk_get_queue(q))
3975+ return NULL;
3976+ return __alloc_disk_node(q, NUMA_NO_NODE, lkclass);
3977+}
3978+EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue);
3979+
39293980 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
39303981 struct blk_mq_tag_set *set, struct request_queue *q,
39313982 int hctx_idx, int node)
@@ -4513,7 +4564,7 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, · 581 unmodified lines
45134564
45144565 list_for_each_entry(q, &set->tag_list, tag_set_list) {
45154566 blk_mq_debugfs_unregister_hctxs(q);
4516− blk_mq_sysfs_unregister(q);
4567+ blk_mq_sysfs_unregister_hctxs(q);
45174568 }
45184569
45194570 prev_nr_hw_queues = set->nr_hw_queues;
@@ -4544,7 +4595,7 @@ fallback: · 24 unmodified lines
45444595
45454596 reregister:
45464597 list_for_each_entry(q, &set->tag_list, tag_set_list) {
4547− blk_mq_sysfs_register(q);
4598+ blk_mq_sysfs_register_hctxs(q);
45484599 blk_mq_debugfs_register_hctxs(q);
45494600 }
45504601
block/blk-mq.h+19−20
@@ -86,16 +86,16 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue * · 85 unmodified lines
8686 return xa_load(&q->hctx_table, q->tag_set->map[type].mq_map[cpu]);
8787 }
8888
89−static inline enum hctx_type blk_mq_get_hctx_type(unsigned int flags)
89+static inline enum hctx_type blk_mq_get_hctx_type(blk_opf_t opf)
9090 {
9191 enum hctx_type type = HCTX_TYPE_DEFAULT;
9292
9393 /*
9494 * The caller ensure that if REQ_POLLED, poll must be enabled.
9595 */
96− if (flags & REQ_POLLED)
96+ if (opf & REQ_POLLED)
9797 type = HCTX_TYPE_POLL;
98− else if ((flags & REQ_OP_MASK) == REQ_OP_READ)
98+ else if ((opf & REQ_OP_MASK) == REQ_OP_READ)
9999 type = HCTX_TYPE_READ;
100100 return type;
101101 }
@@ -103,14 +103,14 @@ static inline enum hctx_type blk_mq_get_hctx_type(unsigned int flags) · 1 unmodified line
103103 /*
104104 * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue
105105 * @q: request queue
106− * @flags: request command flags
106+ * @opf: operation type (REQ_OP_*) and flags (e.g. REQ_POLLED).
107107 * @ctx: software queue cpu ctx
108108 */
109109 static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
110− unsigned int flags,
110+ blk_opf_t opf,
111111 struct blk_mq_ctx *ctx)
112112 {
113− return ctx->hctxs[blk_mq_get_hctx_type(flags)];
113+ return ctx->hctxs[blk_mq_get_hctx_type(opf)];
114114 }
115115
116116 /*
@@ -118,9 +118,10 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, · 1 unmodified line
118118 */
119119 extern void blk_mq_sysfs_init(struct request_queue *q);
120120 extern void blk_mq_sysfs_deinit(struct request_queue *q);
121−extern int __blk_mq_register_dev(struct device *dev, struct request_queue *q);
122−extern int blk_mq_sysfs_register(struct request_queue *q);
123−extern void blk_mq_sysfs_unregister(struct request_queue *q);
121+int blk_mq_sysfs_register(struct gendisk *disk);
122+void blk_mq_sysfs_unregister(struct gendisk *disk);
123+int blk_mq_sysfs_register_hctxs(struct request_queue *q);
124+void blk_mq_sysfs_unregister_hctxs(struct request_queue *q);
124125 extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
125126 void blk_mq_free_plug_rqs(struct blk_plug *plug);
126127 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
@@ -151,7 +152,7 @@ struct blk_mq_alloc_data { · 24 unmodified lines
151152 struct request_queue *q;
152153 blk_mq_req_flags_t flags;
153154 unsigned int shallow_depth;
154− unsigned int cmd_flags;
155+ blk_opf_t cmd_flags;
155156 req_flags_t rq_flags;
156157
157158 /* allocate multiple requests/tags in one go */
@@ -293,7 +294,6 @@ static inline void blk_mq_clear_mq_map(struct blk_mq_queue_map *qmap) · 135 unmodified lines
293294
294295 /*
295296 * blk_mq_plug() - Get caller context plug
296− * @q: request queue
297297 * @bio : the bio being submitted by the caller context
298298 *
299299 * Plugging, by design, may delay the insertion of BIOs into the elevator in
@@ -304,23 +304,22 @@ static inline void blk_mq_clear_mq_map(struct blk_mq_queue_map *qmap) · 4 unmodified lines
304304 * order. While this is not a problem with regular block devices, this ordering
305305 * change can cause write BIO failures with zoned block devices as these
306306 * require sequential write patterns to zones. Prevent this from happening by
307− * ignoring the plug state of a BIO issuing context if the target request queue
308− * is for a zoned block device and the BIO to plug is a write operation.
307+ * ignoring the plug state of a BIO issuing context if it is for a zoned block
308+ * device and the BIO to plug is a write operation.
309309 *
310310 * Return current->plug if the bio can be plugged and NULL otherwise
311311 */
312−static inline struct blk_plug *blk_mq_plug(struct request_queue *q,
313− struct bio *bio)
312+static inline struct blk_plug *blk_mq_plug( struct bio *bio)
314313 {
314+ /* Zoned block device write operation case: do not plug the BIO */
315+ if (bdev_is_zoned(bio->bi_bdev) && op_is_write(bio_op(bio)))
316+ return NULL;
317+
315318 /*
316319 * For regular block devices or read operations, use the context plug
317320 * which may be NULL if blk_start_plug() was not executed.
318321 */
319− if (!blk_queue_is_zoned(q) || !op_is_write(bio_op(bio)))
320− return current->plug;
321−
322− /* Zoned block device write operation case: do not plug the BIO */
323− return NULL;
322+ return current->plug;
324323 }
325324
326325 /* Free all requests on the list */
block/blk-rq-qos.c+2−8
@@ -10,16 +10,10 @@ static bool atomic_inc_below(atomic_t *v, unsigned int below) · 9 unmodified lines
1010 {
1111 unsigned int cur = atomic_read(v);
1212
13− for (;;) {
14− unsigned int old;
15−
13+ do {
1614 if (cur >= below)
1715 return false;
18− old = atomic_cmpxchg(v, cur, cur + 1);
19− if (old == cur)
20− break;
21− cur = old;
22− }
16+ } while (!atomic_try_cmpxchg(v, &cur, cur + 1));
2317
2418 return true;
2519 }
block/blk-rq-qos.h+10−1
@@ -86,7 +86,7 @@ static inline void rq_wait_init(struct rq_wait *rq_wait) · 85 unmodified lines
8686 init_waitqueue_head(&rq_wait->wait);
8787 }
8888
89−static inline void rq_qos_add(struct request_queue *q, struct rq_qos *rqos)
89+static inline int rq_qos_add(struct request_queue *q, struct rq_qos *rqos)
9090 {
9191 /*
9292 * No IO can be in-flight when adding rqos, so freeze queue, which
@@ -98,6 +98,8 @@ static inline void rq_qos_add(struct request_queue *q, struct rq_qos *rqos) · 5 unmodified lines
9898 blk_mq_freeze_queue(q);
9999
100100 spin_lock_irq(&q->queue_lock);
101+ if (rq_qos_id(q, rqos->id))
102+ goto ebusy;
101103 rqos->next = q->rq_qos;
102104 q->rq_qos = rqos;
103105 spin_unlock_irq(&q->queue_lock);
@@ -109,6 +111,13 @@ static inline void rq_qos_add(struct request_queue *q, struct rq_qos *rqos) · 5 unmodified lines
109111 blk_mq_debugfs_register_rqos(rqos);
110112 mutex_unlock(&q->debugfs_mutex);
111113 }
114+
115+ return 0;
116+ebusy:
117+ spin_unlock_irq(&q->queue_lock);
118+ blk_mq_unfreeze_queue(q);
119+ return -EBUSY;
120+
112121 }
113122
114123 static inline void rq_qos_del(struct request_queue *q, struct rq_qos *rqos)
block/blk-settings.c+6−5
@@ -893,18 +893,19 @@ static bool disk_has_partitions(struct gendisk *disk) · 892 unmodified lines
893893 }
894894
895895 /**
896− * blk_queue_set_zoned - configure a disk queue zoned model.
896+ * disk_set_zoned - configure the zoned model for a disk
897897 * @disk: the gendisk of the queue to configure
898898 * @model: the zoned model to set
899899 *
900− * Set the zoned model of the request queue of @disk according to @model.
900+ * Set the zoned model of @disk to @model.
901+ *
901902 * When @model is BLK_ZONED_HM (host managed), this should be called only
902903 * if zoned block device support is enabled (CONFIG_BLK_DEV_ZONED option).
903904 * If @model specifies BLK_ZONED_HA (host aware), the effective model used
904905 * depends on CONFIG_BLK_DEV_ZONED settings and on the existence of partitions
905906 * on the disk.
906907 */
907−void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model)
908+void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model)
908909 {
909910 struct request_queue *q = disk->queue;
910911
@@ -945,10 +946,10 @@ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) · 34 unmodified lines
945946 blk_queue_zone_write_granularity(q,
946947 queue_logical_block_size(q));
947948 } else {
948− blk_queue_clear_zone_settings(q);
949+ disk_clear_zone_settings(disk);
949950 }
950951 }
951−EXPORT_SYMBOL_GPL(blk_queue_set_zoned);
952+EXPORT_SYMBOL_GPL(disk_set_zoned);
952953
953954 int bdev_alignment_offset(struct block_device *bdev)
954955 {
block/blk-sysfs.c+23−37
@@ -274,6 +274,11 @@ static ssize_t queue_virt_boundary_mask_show(struct request_queue *q, char *page · 273 unmodified lines
274274 return queue_var_show(q->limits.virt_boundary_mask, page);
275275 }
276276
277+static ssize_t queue_dma_alignment_show(struct request_queue *q, char *page)
278+{
279+ return queue_var_show(queue_dma_alignment(q), page);
280+}
281+
277282 #define QUEUE_SYSFS_BIT_FNS(name, flag, neg) \
278283 static ssize_t \
279284 queue_##name##_show(struct request_queue *q, char *page) \
@@ -320,17 +325,17 @@ static ssize_t queue_zoned_show(struct request_queue *q, char *page) · 40 unmodified lines
320325
321326 static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)
322327 {
323− return queue_var_show(blk_queue_nr_zones(q), page);
328+ return queue_var_show(disk_nr_zones(q->disk), page);
324329 }
325330
326331 static ssize_t queue_max_open_zones_show(struct request_queue *q, char *page)
327332 {
328− return queue_var_show(queue_max_open_zones(q), page);
333+ return queue_var_show(bdev_max_open_zones(q->disk->part0), page);
329334 }
330335
331336 static ssize_t queue_max_active_zones_show(struct request_queue *q, char *page)
332337 {
333− return queue_var_show(queue_max_active_zones(q), page);
338+ return queue_var_show(bdev_max_active_zones(q->disk->part0), page);
334339 }
335340
336341 static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
@@ -606,6 +611,7 @@ QUEUE_RO_ENTRY(queue_dax, "dax"); · 269 unmodified lines
606611 QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
607612 QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
608613 QUEUE_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");
614+QUEUE_RO_ENTRY(queue_dma_alignment, "dma_alignment");
609615
610616 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
611617 QUEUE_RW_ENTRY(blk_throtl_sample_time, "throttle_sample_time");
@@ -667,6 +673,7 @@ static struct attribute *queue_attrs[] = { · 55 unmodified lines
667673 &blk_throtl_sample_time_entry.attr,
668674 #endif
669675 &queue_virt_boundary_mask_entry.attr,
676+ &queue_dma_alignment_entry.attr,
670677 NULL,
671678 };
672679
@@ -748,11 +755,6 @@ static void blk_free_queue_rcu(struct rcu_head *rcu_head) · 75 unmodified lines
748755 * decremented with blk_put_queue(). Once the refcount reaches 0 this function
749756 * is called.
750757 *
751− * For drivers that have a request_queue on a gendisk and added with
752− * __device_add_disk() the refcount to request_queue will reach 0 with
753− * the last put_disk() called by the driver. For drivers which don't use
754− * __device_add_disk() this happens with blk_cleanup_queue().
755− *
756758 * Drivers exist which depend on the release of the request_queue to be
757759 * synchronous, it should not be deferred.
758760 *
@@ -774,8 +776,6 @@ static void blk_release_queue(struct kobject *kobj) · 15 unmodified lines
774776 blk_free_queue_stats(q->stats);
775777 kfree(q->poll_stat);
776778
777− blk_queue_free_zone_bitmaps(q);
778−
779779 if (queue_is_mq(q))
780780 blk_mq_release(q);
781781
@@ -784,7 +784,7 @@ static void blk_release_queue(struct kobject *kobj) · 2 unmodified lines
784784 if (blk_queue_has_srcu(q))
785785 cleanup_srcu_struct(q->srcu);
786786
787− ida_simple_remove(&blk_queue_ida, q->id);
787+ ida_free(&blk_queue_ida, q->id);
788788 call_rcu(&q->rcu_head, blk_free_queue_rcu);
789789 }
790790
@@ -793,7 +793,13 @@ static const struct sysfs_ops queue_sysfs_ops = { · 2 unmodified lines
793793 .store = queue_attr_store,
794794 };
795795
796+static const struct attribute_group *blk_queue_attr_groups[] = {
797+ &queue_attr_group,
798+ NULL
799+};
800+
796801 struct kobj_type blk_queue_ktype = {
802+ .default_groups = blk_queue_attr_groups,
797803 .sysfs_ops = &queue_sysfs_ops,
798804 .release = blk_release_queue,
799805 };
@@ -804,32 +810,17 @@ struct kobj_type blk_queue_ktype = { · 4 unmodified lines
804810 */
805811 int blk_register_queue(struct gendisk *disk)
806812 {
807− int ret;
808− struct device *dev = disk_to_dev(disk);
809813 struct request_queue *q = disk->queue;
810−
811− ret = blk_trace_init_sysfs(dev);
812− if (ret)
813− return ret;
814+ int ret;
814815
815816 mutex_lock(&q->sysfs_dir_lock);
816817
817− ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
818− if (ret < 0) {
819− blk_trace_remove_sysfs(dev);
820− goto unlock;
821− }
822−
823− ret = sysfs_create_group(&q->kobj, &queue_attr_group);
824− if (ret) {
825− blk_trace_remove_sysfs(dev);
826− kobject_del(&q->kobj);
827− kobject_put(&dev->kobj);
818+ ret = kobject_add(&q->kobj, &disk_to_dev(disk)->kobj, "queue");
819+ if (ret < 0)
828820 goto unlock;
829− }
830821
831822 if (queue_is_mq(q))
832− __blk_mq_register_dev(dev, q);
823+ blk_mq_sysfs_register(disk);
833824 mutex_lock(&q->sysfs_lock);
834825
835826 mutex_lock(&q->debugfs_mutex);
@@ -839,7 +830,7 @@ int blk_register_queue(struct gendisk *disk) · 3 unmodified lines
839830 blk_mq_debugfs_register(q);
840831 mutex_unlock(&q->debugfs_mutex);
841832
842− ret = disk_register_independent_access_ranges(disk, NULL);
833+ ret = disk_register_independent_access_ranges(disk);
843834 if (ret)
844835 goto put_dev;
845836
@@ -888,8 +879,6 @@ put_dev: · 42 unmodified lines
888879 mutex_unlock(&q->sysfs_lock);
889880 mutex_unlock(&q->sysfs_dir_lock);
890881 kobject_del(&q->kobj);
891− blk_trace_remove_sysfs(dev);
892− kobject_put(&dev->kobj);
893882
894883 return ret;
895884 }
@@ -927,9 +916,8 @@ void blk_unregister_queue(struct gendisk *disk) · 31 unmodified lines
927916 * structures that can be modified through sysfs.
928917 */
929918 if (queue_is_mq(q))
930− blk_mq_unregister_dev(disk_to_dev(disk), q);
919+ blk_mq_sysfs_unregister(disk);
931920 blk_crypto_sysfs_unregister(q);
932− blk_trace_remove_sysfs(disk_to_dev(disk));
933921
934922 mutex_lock(&q->sysfs_lock);
935923 elv_unregister_queue(q);
@@ -948,6 +936,4 @@ void blk_unregister_queue(struct gendisk *disk) · 12 unmodified lines
948936 q->sched_debugfs_dir = NULL;
949937 q->rqos_debugfs_dir = NULL;
950938 mutex_unlock(&q->debugfs_mutex);
951−
952− kobject_put(&disk_to_dev(disk)->kobj);
953939 }
block/blk-throttle.c+4−3
@@ -2203,8 +2203,9 @@ out_unlock: · 2202 unmodified lines
22032203
22042204 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
22052205 static void throtl_track_latency(struct throtl_data *td, sector_t size,
2206− int op, unsigned long time)
2206+ enum req_op op, unsigned long time)
22072207 {
2208+ const bool rw = op_is_write(op);
22082209 struct latency_bucket *latency;
22092210 int index;
22102211
@@ -2215,10 +2216,10 @@ static void throtl_track_latency(struct throtl_data *td, sector_t size, · 4 unmodified lines
22152216
22162217 index = request_bucket_index(size);
22172218
2218− latency = get_cpu_ptr(td->latency_buckets[op]);
2219+ latency = get_cpu_ptr(td->latency_buckets[rw]);
22192220 latency[index].total_latency += time;
22202221 latency[index].samples++;
2221− put_cpu_ptr(td->latency_buckets[op]);
2222+ put_cpu_ptr(td->latency_buckets[rw]);
22222223 }
22232224
22242225 void blk_throtl_stat_add(struct request *rq, u64 time_ns)
block/blk-wbt.c+20−10
@@ -451,7 +451,7 @@ static bool close_io(struct rq_wb *rwb) · 450 unmodified lines
451451
452452 #define REQ_HIPRIO (REQ_SYNC | REQ_META | REQ_PRIO)
453453
454−static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)
454+static inline unsigned int get_limit(struct rq_wb *rwb, blk_opf_t opf)
455455 {
456456 unsigned int limit;
457457
@@ -462,7 +462,7 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) · 4 unmodified lines
462462 if (!rwb_enabled(rwb))
463463 return UINT_MAX;
464464
465− if ((rw & REQ_OP_MASK) == REQ_OP_DISCARD)
465+ if ((opf & REQ_OP_MASK) == REQ_OP_DISCARD)
466466 return rwb->wb_background;
467467
468468 /*
@@ -473,9 +473,9 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) · 4 unmodified lines
473473 * the idle limit, or go to normal if we haven't had competing
474474 * IO for a bit.
475475 */
476− if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd())
476+ if ((opf & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd())
477477 limit = rwb->rq_depth.max_depth;
478− else if ((rw & REQ_BACKGROUND) || close_io(rwb)) {
478+ else if ((opf & REQ_BACKGROUND) || close_io(rwb)) {
479479 /*
480480 * If less than 100ms since we completed unrelated IO,
481481 * limit us to half the depth for background writeback.
@@ -490,13 +490,13 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) · 8 unmodified lines
490490 struct wbt_wait_data {
491491 struct rq_wb *rwb;
492492 enum wbt_flags wb_acct;
493− unsigned long rw;
493+ blk_opf_t opf;
494494 };
495495
496496 static bool wbt_inflight_cb(struct rq_wait *rqw, void *private_data)
497497 {
498498 struct wbt_wait_data *data = private_data;
499− return rq_wait_inc_below(rqw, get_limit(data->rwb, data->rw));
499+ return rq_wait_inc_below(rqw, get_limit(data->rwb, data->opf));
500500 }
501501
502502 static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data)
@@ -510,13 +510,13 @@ static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data) · 7 unmodified lines
510510 * the timer to kick off queuing again.
511511 */
512512 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,
513− unsigned long rw)
513+ blk_opf_t opf)
514514 {
515515 struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
516516 struct wbt_wait_data data = {
517517 .rwb = rwb,
518518 .wb_acct = wb_acct,
519− .rw = rw,
519+ .opf = opf,
520520 };
521521
522522 rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb);
@@ -670,7 +670,7 @@ u64 wbt_default_latency_nsec(struct request_queue *q) · 147 unmodified lines
670670
671671 static int wbt_data_dir(const struct request *rq)
672672 {
673− const int op = req_op(rq);
673+ const enum req_op op = req_op(rq);
674674
675675 if (op == REQ_OP_READ)
676676 return READ;
@@ -820,6 +820,7 @@ int wbt_init(struct request_queue *q) · 143 unmodified lines
820820 {
821821 struct rq_wb *rwb;
822822 int i;
823+ int ret;
823824
824825 rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);
825826 if (!rwb)
@@ -846,7 +847,10 @@ int wbt_init(struct request_queue *q) · 20 unmodified lines
846847 /*
847848 * Assign rwb and add the stats callback.
848849 */
849− rq_qos_add(q, &rwb->rqos);
850+ ret = rq_qos_add(q, &rwb->rqos);
851+ if (ret)
852+ goto err_free;
853+
850854 blk_stat_add_callback(q, rwb->cb);
851855
852856 rwb->min_lat_nsec = wbt_default_latency_nsec(q);
@@ -855,4 +859,10 @@ int wbt_init(struct request_queue *q) · 2 unmodified lines
855859 wbt_set_write_cache(q, test_bit(QUEUE_FLAG_WC, &q->queue_flags));
856860
857861 return 0;
862+
863+err_free:
864+ blk_stat_free_callback(rwb->cb);
865+ kfree(rwb);
866+ return ret;
867+
858868 }
block/blk-zoned.c+46−46
@@ -57,10 +57,10 @@ EXPORT_SYMBOL_GPL(blk_zone_cond_str); · 56 unmodified lines
5757 */
5858 bool blk_req_needs_zone_write_lock(struct request *rq)
5959 {
60− if (!rq->q->seq_zones_wlock)
60+ if (blk_rq_is_passthrough(rq))
6161 return false;
6262
63− if (blk_rq_is_passthrough(rq))
63+ if (!rq->q->disk->seq_zones_wlock)
6464 return false;
6565
6666 switch (req_op(rq)) {
@@ -77,7 +77,7 @@ bool blk_req_zone_write_trylock(struct request *rq) · 10 unmodified lines
7777 {
7878 unsigned int zno = blk_rq_zone_no(rq);
7979
80− if (test_and_set_bit(zno, rq->q->seq_zones_wlock))
80+ if (test_and_set_bit(zno, rq->q->disk->seq_zones_wlock))
8181 return false;
8282
8383 WARN_ON_ONCE(rq->rq_flags & RQF_ZONE_WRITE_LOCKED);
@@ -90,7 +90,7 @@ EXPORT_SYMBOL_GPL(blk_req_zone_write_trylock); · 6 unmodified lines
9090 void __blk_req_zone_write_lock(struct request *rq)
9191 {
9292 if (WARN_ON_ONCE(test_and_set_bit(blk_rq_zone_no(rq),
93− rq->q->seq_zones_wlock)))
93+ rq->q->disk->seq_zones_wlock)))
9494 return;
9595
9696 WARN_ON_ONCE(rq->rq_flags & RQF_ZONE_WRITE_LOCKED);
@@ -101,28 +101,29 @@ EXPORT_SYMBOL_GPL(__blk_req_zone_write_lock); · 4 unmodified lines
101101 void __blk_req_zone_write_unlock(struct request *rq)
102102 {
103103 rq->rq_flags &= ~RQF_ZONE_WRITE_LOCKED;
104− if (rq->q->seq_zones_wlock)
104+ if (rq->q->disk->seq_zones_wlock)
105105 WARN_ON_ONCE(!test_and_clear_bit(blk_rq_zone_no(rq),
106− rq->q->seq_zones_wlock));
106+ rq->q->disk->seq_zones_wlock));
107107 }
108108 EXPORT_SYMBOL_GPL(__blk_req_zone_write_unlock);
109109
110110 /**
111− * blkdev_nr_zones - Get number of zones
112− * @disk: Target gendisk
111+ * bdev_nr_zones - Get number of zones
112+ * @bdev: Target device
113113 *
114114 * Return the total number of zones of a zoned block device. For a block
115115 * device without zone capabilities, the number of zones is always 0.
116116 */
117−unsigned int blkdev_nr_zones(struct gendisk *disk)
117+unsigned int bdev_nr_zones(struct block_device *bdev)
118118 {
119− sector_t zone_sectors = blk_queue_zone_sectors(disk->queue);
119+ sector_t zone_sectors = bdev_zone_sectors(bdev);
120120
121− if (!blk_queue_is_zoned(disk->queue))
121+ if (!bdev_is_zoned(bdev))
122122 return 0;
123− return (get_capacity(disk) + zone_sectors - 1) >> ilog2(zone_sectors);
123+ return (bdev_nr_sectors(bdev) + zone_sectors - 1) >>
124+ ilog2(zone_sectors);
124125 }
125−EXPORT_SYMBOL_GPL(blkdev_nr_zones);
126+EXPORT_SYMBOL_GPL(bdev_nr_zones);
126127
127128 /**
128129 * blkdev_report_zones - Get zones information
@@ -149,8 +150,7 @@ int blkdev_report_zones(struct block_device *bdev, sector_t sector, · 20 unmodified lines
149150 struct gendisk *disk = bdev->bd_disk;
150151 sector_t capacity = get_capacity(disk);
151152
152− if (!blk_queue_is_zoned(bdev_get_queue(bdev)) ||
153− WARN_ON_ONCE(!disk->fops->report_zones))
153+ if (!bdev_is_zoned(bdev) || WARN_ON_ONCE(!disk->fops->report_zones))
154154 return -EOPNOTSUPP;
155155
156156 if (!nr_zones || sector >= capacity)
@@ -189,27 +189,26 @@ static int blk_zone_need_reset_cb(struct blk_zone *zone, unsigned int idx, · 32 unmodified lines
189189 static int blkdev_zone_reset_all_emulated(struct block_device *bdev,
190190 gfp_t gfp_mask)
191191 {
192− struct request_queue *q = bdev_get_queue(bdev);
193− sector_t capacity = get_capacity(bdev->bd_disk);
194− sector_t zone_sectors = blk_queue_zone_sectors(q);
192+ struct gendisk *disk = bdev->bd_disk;
193+ sector_t capacity = bdev_nr_sectors(bdev);
194+ sector_t zone_sectors = bdev_zone_sectors(bdev);
195195 unsigned long *need_reset;
196196 struct bio *bio = NULL;
197197 sector_t sector = 0;
198198 int ret;
199199
200− need_reset = blk_alloc_zone_bitmap(q->node, q->nr_zones);
200+ need_reset = blk_alloc_zone_bitmap(disk->queue->node, disk->nr_zones);
201201 if (!need_reset)
202202 return -ENOMEM;
203203
204− ret = bdev->bd_disk->fops->report_zones(bdev->bd_disk, 0,
205− q->nr_zones, blk_zone_need_reset_cb,
206− need_reset);
204+ ret = disk->fops->report_zones(disk, 0, disk->nr_zones,
205+ blk_zone_need_reset_cb, need_reset);
207206 if (ret < 0)
208207 goto out_free_need_reset;
209208
210209 ret = 0;
211210 while (sector < capacity) {
212− if (!test_bit(blk_queue_zone_no(q, sector), need_reset)) {
211+ if (!test_bit(disk_zone_no(disk, sector), need_reset)) {
213212 sector += zone_sectors;
214213 continue;
215214 }
@@ -257,18 +256,17 @@ static int blkdev_zone_reset_all(struct block_device *bdev, gfp_t gfp_mask) · 41 unmodified lines
257256 * The operation to execute on each zone can be a zone reset, open, close
258257 * or finish request.
259258 */
260−int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
261− sector_t sector, sector_t nr_sectors,
262− gfp_t gfp_mask)
259+int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
260+ sector_t sector, sector_t nr_sectors, gfp_t gfp_mask)
263261 {
264262 struct request_queue *q = bdev_get_queue(bdev);
265− sector_t zone_sectors = blk_queue_zone_sectors(q);
266− sector_t capacity = get_capacity(bdev->bd_disk);
263+ sector_t zone_sectors = bdev_zone_sectors(bdev);
264+ sector_t capacity = bdev_nr_sectors(bdev);
267265 sector_t end_sector = sector + nr_sectors;
268266 struct bio *bio = NULL;
269267 int ret = 0;
270268
271− if (!blk_queue_is_zoned(q))
269+ if (!bdev_is_zoned(bdev))
272270 return -EOPNOTSUPP;
273271
274272 if (bdev_read_only(bdev))
@@ -350,7 +348,7 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, · 75 unmodified lines
350348 if (!q)
351349 return -ENXIO;
352350
353− if (!blk_queue_is_zoned(q))
351+ if (!bdev_is_zoned(bdev))
354352 return -ENOTTY;
355353
356354 if (copy_from_user(&rep, argp, sizeof(struct blk_zone_report)))
@@ -398,7 +396,7 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, · 41 unmodified lines
398396 void __user *argp = (void __user *)arg;
399397 struct request_queue *q;
400398 struct blk_zone_range zrange;
401− enum req_opf op;
399+ enum req_op op;
402400 int ret;
403401
404402 if (!argp)
@@ -408,7 +406,7 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, · 3 unmodified lines
408406 if (!q)
409407 return -ENXIO;
410408
411− if (!blk_queue_is_zoned(q))
409+ if (!bdev_is_zoned(bdev))
412410 return -ENOTTY;
413411
414412 if (!(mode & FMODE_WRITE))
@@ -450,12 +448,12 @@ fail: · 35 unmodified lines
450448 return ret;
451449 }
452450
453−void blk_queue_free_zone_bitmaps(struct request_queue *q)
451+void disk_free_zone_bitmaps(struct gendisk *disk)
454452 {
455− kfree(q->conv_zones_bitmap);
456− q->conv_zones_bitmap = NULL;
457− kfree(q->seq_zones_wlock);
458− q->seq_zones_wlock = NULL;
453+ kfree(disk->conv_zones_bitmap);
454+ disk->conv_zones_bitmap = NULL;
455+ kfree(disk->seq_zones_wlock);
456+ disk->seq_zones_wlock = NULL;
459457 }
460458
461459 struct blk_revalidate_zone_args {
@@ -605,15 +603,15 @@ int blk_revalidate_disk_zones(struct gendisk *disk, · 143 unmodified lines
605603 blk_mq_freeze_queue(q);
606604 if (ret > 0) {
607605 blk_queue_chunk_sectors(q, args.zone_sectors);
608− q->nr_zones = args.nr_zones;
609− swap(q->seq_zones_wlock, args.seq_zones_wlock);
610− swap(q->conv_zones_bitmap, args.conv_zones_bitmap);
606+ disk->nr_zones = args.nr_zones;
607+ swap(disk->seq_zones_wlock, args.seq_zones_wlock);
608+ swap(disk->conv_zones_bitmap, args.conv_zones_bitmap);
611609 if (update_driver_data)
612610 update_driver_data(disk);
613611 ret = 0;
614612 } else {
615613 pr_warn("%s: failed to revalidate zones\n", disk->disk_name);
616− blk_queue_free_zone_bitmaps(q);
614+ disk_free_zone_bitmaps(disk);
617615 }
618616 blk_mq_unfreeze_queue(q);
619617
@@ -623,16 +621,18 @@ int blk_revalidate_disk_zones(struct gendisk *disk, · 3 unmodified lines
623621 }
624622 EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones);
625623
626−void blk_queue_clear_zone_settings(struct request_queue *q)
624+void disk_clear_zone_settings(struct gendisk *disk)
627625 {
626+ struct request_queue *q = disk->queue;
627+
628628 blk_mq_freeze_queue(q);
629629
630− blk_queue_free_zone_bitmaps(q);
630+ disk_free_zone_bitmaps(disk);
631631 blk_queue_flag_clear(QUEUE_FLAG_ZONE_RESETALL, q);
632632 q->required_elevator_features &= ~ELEVATOR_F_ZBD_SEQ_WRITE;
633− q->nr_zones = 0;
634− q->max_open_zones = 0;
635− q->max_active_zones = 0;
633+ disk->nr_zones = 0;
634+ disk->max_open_zones = 0;
635+ disk->max_active_zones = 0;
636636 q->limits.chunk_sectors = 0;
637637 q->limits.zone_write_granularity = 0;
638638 q->limits.max_zone_append_sectors = 0;
block/blk.h+23−11
@@ -31,11 +31,6 @@ extern struct kmem_cache *blk_requestq_srcu_cachep; · 30 unmodified lines
3131 extern struct kobj_type blk_queue_ktype;
3232 extern struct ida blk_queue_ida;
3333
34−static inline void __blk_get_queue(struct request_queue *q)
35−{
36− kobject_get(&q->kobj);
37−}
38−
3934 bool is_flush_rq(struct request *req);
4035
4136 struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
@@ -159,6 +154,19 @@ static inline bool blk_discard_mergable(struct request *req) · 117 unmodified lines
159154 return false;
160155 }
161156
157+static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
158+ enum req_op op)
159+{
160+ if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
161+ return min(q->limits.max_discard_sectors,
162+ UINT_MAX >> SECTOR_SHIFT);
163+
164+ if (unlikely(op == REQ_OP_WRITE_ZEROES))
165+ return q->limits.max_write_zeroes_sectors;
166+
167+ return q->limits.max_sectors;
168+}
169+
162170 #ifdef CONFIG_BLK_DEV_INTEGRITY
163171 void blk_flush_integrity(void);
164172 bool __bio_integrity_endio(struct bio *);
@@ -392,11 +400,11 @@ static inline int blk_iolatency_init(struct request_queue *q) { return 0; } · 227 unmodified lines
392400 #endif
393401
394402 #ifdef CONFIG_BLK_DEV_ZONED
395−void blk_queue_free_zone_bitmaps(struct request_queue *q);
396−void blk_queue_clear_zone_settings(struct request_queue *q);
403+void disk_free_zone_bitmaps(struct gendisk *disk);
404+void disk_clear_zone_settings(struct gendisk *disk);
397405 #else
398−static inline void blk_queue_free_zone_bitmaps(struct request_queue *q) {}
399−static inline void blk_queue_clear_zone_settings(struct request_queue *q) {}
406+static inline void disk_free_zone_bitmaps(struct gendisk *disk) {}
407+static inline void disk_clear_zone_settings(struct gendisk *disk) {}
400408 #endif
401409
402410 int blk_alloc_ext_minor(void);
@@ -411,6 +419,9 @@ int bdev_resize_partition(struct gendisk *disk, int partno, sector_t start, · 8 unmodified lines
411419 sector_t length);
412420 void blk_drop_partitions(struct gendisk *disk);
413421
422+struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
423+ struct lock_class_key *lkclass);
424+
414425 int bio_add_hw_page(struct request_queue *q, struct bio *bio,
415426 struct page *page, unsigned int len, unsigned int offset,
416427 unsigned int max_sectors, bool *same_page);
@@ -436,13 +447,14 @@ extern struct device_attribute dev_attr_events; · 19 unmodified lines
436447 extern struct device_attribute dev_attr_events_async;
437448 extern struct device_attribute dev_attr_events_poll_msecs;
438449
450+extern struct attribute_group blk_trace_attr_group;
451+
439452 long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
440453 long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
441454
442455 extern const struct address_space_operations def_blk_aops;
443456
444−int disk_register_independent_access_ranges(struct gendisk *disk,
445− struct blk_independent_access_ranges *new_iars);
457+int disk_register_independent_access_ranges(struct gendisk *disk);
446458 void disk_unregister_independent_access_ranges(struct gendisk *disk);
447459
448460 #ifdef CONFIG_FAIL_MAKE_REQUEST
block/bounce.c+10−3
@@ -205,19 +205,26 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · 204 unmodified lines
205205 int rw = bio_data_dir(*bio_orig);
206206 struct bio_vec *to, from;
207207 struct bvec_iter iter;
208− unsigned i = 0;
208+ unsigned i = 0, bytes = 0;
209209 bool bounce = false;
210− int sectors = 0;
210+ int sectors;
211211
212212 bio_for_each_segment(from, *bio_orig, iter) {
213213 if (i++ < BIO_MAX_VECS)
214− sectors += from.bv_len >> 9;
214+ bytes += from.bv_len;
215215 if (PageHighMem(from.bv_page))
216216 bounce = true;
217217 }
218218 if (!bounce)
219219 return;
220220
221+ /*
222+ * Individual bvecs might not be logical block aligned. Round down
223+ * the split size so that each bio is properly block size aligned,
224+ * even if we do not use the full hardware limits.
225+ */
226+ sectors = ALIGN_DOWN(bytes, queue_logical_block_size(q)) >>
227+ SECTOR_SHIFT;
221228 if (sectors < bio_sectors(*bio_orig)) {
222229 bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
223230 bio_chain(bio, *bio_orig);
block/bsg-lib.c+3−3
@@ -324,14 +324,14 @@ void bsg_remove_queue(struct request_queue *q) · 323 unmodified lines
324324 container_of(q->tag_set, struct bsg_set, tag_set);
325325
326326 bsg_unregister_queue(bset->bd);
327− blk_cleanup_queue(q);
327+ blk_mq_destroy_queue(q);
328328 blk_mq_free_tag_set(&bset->tag_set);
329329 kfree(bset);
330330 }
331331 }
332332 EXPORT_SYMBOL_GPL(bsg_remove_queue);
333333
334−static enum blk_eh_timer_return bsg_timeout(struct request *rq, bool reserved)
334+static enum blk_eh_timer_return bsg_timeout(struct request *rq)
335335 {
336336 struct bsg_set *bset =
337337 container_of(rq->q->tag_set, struct bsg_set, tag_set);
@@ -399,7 +399,7 @@ struct request_queue *bsg_setup_queue(struct device *dev, const char *name, · 61 unmodified lines
399399
400400 return q;
401401 out_cleanup_queue:
402− blk_cleanup_queue(q);
402+ blk_mq_destroy_queue(q);
403403 out_queue:
404404 blk_mq_free_tag_set(set);
405405 out_tag_set:
block/elevator.h+1−1
@@ -34,7 +34,7 @@ struct elevator_mq_ops { · 33 unmodified lines
3434 int (*request_merge)(struct request_queue *q, struct request **, struct bio *);
3535 void (*request_merged)(struct request_queue *, struct request *, enum elv_merge);
3636 void (*requests_merged)(struct request_queue *, struct request *, struct request *);
37− void (*limit_depth)(unsigned int, struct blk_mq_alloc_data *);
37+ void (*limit_depth)(blk_opf_t, struct blk_mq_alloc_data *);
3838 void (*prepare_request)(struct request *);
3939 void (*finish_request)(struct request *);
4040 void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool);
block/fops.c+16−12
@@ -32,14 +32,21 @@ static int blkdev_get_block(struct inode *inode, sector_t iblock, · 31 unmodified lines
3232 return 0;
3333 }
3434
35−static unsigned int dio_bio_write_op(struct kiocb *iocb)
35+static blk_opf_t dio_bio_write_op(struct kiocb *iocb)
3636 {
37− unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
37+ blk_opf_t opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
3838
3939 /* avoid the need for a I/O completion work item */
4040 if (iocb->ki_flags & IOCB_DSYNC)
41− op |= REQ_FUA;
42− return op;
41+ opf |= REQ_FUA;
42+ return opf;
43+}
44+
45+static bool blkdev_dio_unaligned(struct block_device *bdev, loff_t pos,
46+ struct iov_iter *iter)
47+{
48+ return pos & (bdev_logical_block_size(bdev) - 1) ||
49+ !bdev_iter_is_aligned(bdev, iter);
4350 }
4451
4552 #define DIO_INLINE_BIO_VECS 4
@@ -54,8 +61,7 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb, · 8 unmodified lines
5461 struct bio bio;
5562 ssize_t ret;
5663
57− if ((pos | iov_iter_alignment(iter)) &
58− (bdev_logical_block_size(bdev) - 1))
64+ if (blkdev_dio_unaligned(bdev, pos, iter))
5965 return -EINVAL;
6066
6167 if (nr_pages <= DIO_INLINE_BIO_VECS)
@@ -169,12 +175,11 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, · 107 unmodified lines
169175 struct blkdev_dio *dio;
170176 struct bio *bio;
171177 bool is_read = (iov_iter_rw(iter) == READ), is_sync;
172− unsigned int opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
178+ blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
173179 loff_t pos = iocb->ki_pos;
174180 int ret = 0;
175181
176− if ((pos | iov_iter_alignment(iter)) &
177− (bdev_logical_block_size(bdev) - 1))
182+ if (blkdev_dio_unaligned(bdev, pos, iter))
178183 return -EINVAL;
179184
180185 if (iocb->ki_flags & IOCB_ALLOC_CACHE)
@@ -292,14 +297,13 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, · 111 unmodified lines
292297 {
293298 struct block_device *bdev = iocb->ki_filp->private_data;
294299 bool is_read = iov_iter_rw(iter) == READ;
295− unsigned int opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
300+ blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
296301 struct blkdev_dio *dio;
297302 struct bio *bio;
298303 loff_t pos = iocb->ki_pos;
299304 int ret = 0;
300305
301− if ((pos | iov_iter_alignment(iter)) &
302− (bdev_logical_block_size(bdev) - 1))
306+ if (blkdev_dio_unaligned(bdev, pos, iter))
303307 return -EINVAL;
304308
305309 if (iocb->ki_flags & IOCB_ALLOC_CACHE)
block/genhd.c+32−48
@@ -101,29 +101,6 @@ bool set_capacity_and_notify(struct gendisk *disk, sector_t size) · 100 unmodified lines
101101 }
102102 EXPORT_SYMBOL_GPL(set_capacity_and_notify);
103103
104−/*
105− * Format the device name of the indicated block device into the supplied buffer
106− * and return a pointer to that same buffer for convenience.
107− *
108− * Note: do not use this in new code, use the %pg specifier to sprintf and
109− * printk insted.
110− */
111−const char *bdevname(struct block_device *bdev, char *buf)
112−{
113− struct gendisk *hd = bdev->bd_disk;
114− int partno = bdev->bd_partno;
115−
116− if (!partno)
117− snprintf(buf, BDEVNAME_SIZE, "%s", hd->disk_name);
118− else if (isdigit(hd->disk_name[strlen(hd->disk_name)-1]))
119− snprintf(buf, BDEVNAME_SIZE, "%sp%d", hd->disk_name, partno);
120− else
121− snprintf(buf, BDEVNAME_SIZE, "%s%d", hd->disk_name, partno);
122−
123− return buf;
124−}
125−EXPORT_SYMBOL(bdevname);
126−
127104 static void part_stat_read_all(struct block_device *part,
128105 struct disk_stats *stat)
129106 {
@@ -617,6 +594,8 @@ void del_gendisk(struct gendisk *disk) · 487 unmodified lines
617594 * Fail any new I/O.
618595 */
619596 set_bit(GD_DEAD, &disk->state);
597+ if (test_bit(GD_OWNS_QUEUE, &disk->state))
598+ blk_queue_flag_set(QUEUE_FLAG_DYING, q);
620599 set_capacity(disk, 0);
621600
622601 /*
@@ -663,11 +642,16 @@ void del_gendisk(struct gendisk *disk) · 40 unmodified lines
663642 blk_mq_unquiesce_queue(q);
664643
665644 /*
666− * Allow using passthrough request again after the queue is torn down.
645+ * If the disk does not own the queue, allow using passthrough requests
646+ * again. Else leave the queue frozen to fail all I/O.
667647 */
668− blk_queue_flag_clear(QUEUE_FLAG_INIT_DONE, q);
669− __blk_mq_unfreeze_queue(q, true);
670−
648+ if (!test_bit(GD_OWNS_QUEUE, &disk->state)) {
649+ blk_queue_flag_clear(QUEUE_FLAG_INIT_DONE, q);
650+ __blk_mq_unfreeze_queue(q, true);
651+ } else {
652+ if (queue_is_mq(q))
653+ blk_mq_exit_queue(q);
654+ }
671655 }
672656 EXPORT_SYMBOL(del_gendisk);
673657
@@ -1127,6 +1111,9 @@ static struct attribute_group disk_attr_group = { · 453 unmodified lines
11271111
11281112 static const struct attribute_group *disk_attr_groups[] = {
11291113 &disk_attr_group,
1114+#ifdef CONFIG_BLK_DEV_IO_TRACE
1115+ &blk_trace_attr_group,
1116+#endif
11301117 NULL
11311118 };
11321119
@@ -1151,10 +1138,23 @@ static void disk_release(struct device *dev) · 18 unmodified lines
11511138 might_sleep();
11521139 WARN_ON_ONCE(disk_live(disk));
11531140
1141+ /*
1142+ * To undo the all initialization from blk_mq_init_allocated_queue in
1143+ * case of a probe failure where add_disk is never called we have to
1144+ * call blk_mq_exit_queue here. We can't do this for the more common
1145+ * teardown case (yet) as the tagset can be gone by the time the disk
1146+ * is released once it was added.
1147+ */
1148+ if (queue_is_mq(disk->queue) &&
1149+ test_bit(GD_OWNS_QUEUE, &disk->state) &&
1150+ !test_bit(GD_ADDED, &disk->state))
1151+ blk_mq_exit_queue(disk->queue);
1152+
11541153 blkcg_exit_queue(disk->queue);
11551154
11561155 disk_release_events(disk);
11571156 kfree(disk->random);
1157+ disk_free_zone_bitmaps(disk);
11581158 xa_destroy(&disk->part_tbl);
11591159
11601160 disk->queue->disk = NULL;
@@ -1338,9 +1338,6 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, · 177 unmodified lines
13381338 {
13391339 struct gendisk *disk;
13401340
1341− if (!blk_get_queue(q))
1342− return NULL;
1343−
13441341 disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
13451342 if (!disk)
13461343 goto out_put_queue;
@@ -1391,7 +1388,6 @@ out_put_queue: · 44 unmodified lines
13911388 blk_put_queue(q);
13921389 return NULL;
13931390 }
1394−EXPORT_SYMBOL(__alloc_disk_node);
13951391
13961392 struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass)
13971393 {
@@ -1404,9 +1400,10 @@ struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass) · 6 unmodified lines
14041400
14051401 disk = __alloc_disk_node(q, node, lkclass);
14061402 if (!disk) {
1407− blk_cleanup_queue(q);
1403+ blk_put_queue(q);
14081404 return NULL;
14091405 }
1406+ set_bit(GD_OWNS_QUEUE, &disk->state);
14101407 return disk;
14111408 }
14121409 EXPORT_SYMBOL(__blk_alloc_disk);
@@ -1418,6 +1415,9 @@ EXPORT_SYMBOL(__blk_alloc_disk); · 5 unmodified lines
14181415 * This decrements the refcount for the struct gendisk. When this reaches 0
14191416 * we'll have disk_release() called.
14201417 *
1418+ * Note: for blk-mq disk put_disk must be called before freeing the tag_set
1419+ * when handling probe errors (that is before add_disk() is called).
1420+ *
14211421 * Context: Any context, but the last reference must not be dropped from
14221422 * atomic context.
14231423 */
@@ -1428,22 +1428,6 @@ void put_disk(struct gendisk *disk) · 4 unmodified lines
14281428 }
14291429 EXPORT_SYMBOL(put_disk);
14301430
1431−/**
1432− * blk_cleanup_disk - shutdown a gendisk allocated by blk_alloc_disk
1433− * @disk: gendisk to shutdown
1434− *
1435− * Mark the queue hanging off @disk DYING, drain all pending requests, then mark
1436− * the queue DEAD, destroy and put it and the gendisk structure.
1437− *
1438− * Context: can sleep
1439− */
1440−void blk_cleanup_disk(struct gendisk *disk)
1441−{
1442− blk_cleanup_queue(disk->queue);
1443− put_disk(disk);
1444−}
1445−EXPORT_SYMBOL(blk_cleanup_disk);
1446−
14471431 static void set_disk_ro_uevent(struct gendisk *gd, int ro)
14481432 {
14491433 char event[] = "DISK_RO=1";
block/ioctl.c+1−1
@@ -495,7 +495,7 @@ static int blkdev_common_ioctl(struct block_device *bdev, fmode_t mode, · 494 unmodified lines
495495 case BLKGETZONESZ:
496496 return put_uint(argp, bdev_zone_sectors(bdev));
497497 case BLKGETNRZONES:
498− return put_uint(argp, blkdev_nr_zones(bdev->bd_disk));
498+ return put_uint(argp, bdev_nr_zones(bdev));
499499 case BLKROGET:
500500 return put_int(argp, bdev_read_only(bdev) != 0);
501501 case BLKSSZGET: /* get block device logical block size */
block/ioprio.c+50−8
@@ -138,6 +138,32 @@ out: · 137 unmodified lines
138138 return ret;
139139 }
140140
141+/*
142+ * If the task has set an I/O priority, use that. Otherwise, return
143+ * the default I/O priority.
144+ *
145+ * Expected to be called for current task or with task_lock() held to keep
146+ * io_context stable.
147+ */
148+int __get_task_ioprio(struct task_struct *p)
149+{
150+ struct io_context *ioc = p->io_context;
151+ int prio;
152+
153+ if (p != current)
154+ lockdep_assert_held(&p->alloc_lock);
155+ if (ioc)
156+ prio = ioc->ioprio;
157+ else
158+ prio = IOPRIO_DEFAULT;
159+
160+ if (IOPRIO_PRIO_CLASS(prio) == IOPRIO_CLASS_NONE)
161+ prio = IOPRIO_PRIO_VALUE(task_nice_ioclass(p),
162+ task_nice_ioprio(p));
163+ return prio;
164+}
165+EXPORT_SYMBOL_GPL(__get_task_ioprio);
166+
141167 static int get_task_ioprio(struct task_struct *p)
142168 {
143169 int ret;
@@ -145,22 +171,38 @@ static int get_task_ioprio(struct task_struct *p) · 1 unmodified line
145171 ret = security_task_getioprio(p);
146172 if (ret)
147173 goto out;
148− ret = IOPRIO_DEFAULT;
174+ task_lock(p);
175+ ret = __get_task_ioprio(p);
176+ task_unlock(p);
177+out:
178+ return ret;
179+}
180+
181+/*
182+ * Return raw IO priority value as set by userspace. We use this for
183+ * ioprio_get(pid, IOPRIO_WHO_PROCESS) so that we keep historical behavior and
184+ * also so that userspace can distinguish unset IO priority (which just gets
185+ * overriden based on task's nice value) from IO priority set to some value.
186+ */
187+static int get_task_raw_ioprio(struct task_struct *p)
188+{
189+ int ret;
190+
191+ ret = security_task_getioprio(p);
192+ if (ret)
193+ goto out;
149194 task_lock(p);
150195 if (p->io_context)
151196 ret = p->io_context->ioprio;
197+ else
198+ ret = IOPRIO_DEFAULT;
152199 task_unlock(p);
153200 out:
154201 return ret;
155202 }
156203
157−int ioprio_best(unsigned short aprio, unsigned short bprio)
204+static int ioprio_best(unsigned short aprio, unsigned short bprio)
158205 {
159− if (!ioprio_valid(aprio))
160− aprio = IOPRIO_DEFAULT;
161− if (!ioprio_valid(bprio))
162− bprio = IOPRIO_DEFAULT;
163−
164206 return min(aprio, bprio);
165207 }
166208
@@ -181,7 +223,7 @@ SYSCALL_DEFINE2(ioprio_get, int, which, int, who) · 14 unmodified lines
181223 else
182224 p = find_task_by_vpid(who);
183225 if (p)
184− ret = get_task_ioprio(p);
226+ ret = get_task_raw_ioprio(p);
185227 break;
186228 case IOPRIO_WHO_PGRP:
187229 if (!who)
block/kyber-iosched.c+4−4
@@ -195,9 +195,9 @@ struct kyber_hctx_data { · 194 unmodified lines
195195 static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags,
196196 void *key);
197197
198−static unsigned int kyber_sched_domain(unsigned int op)
198+static unsigned int kyber_sched_domain(blk_opf_t opf)
199199 {
200− switch (op & REQ_OP_MASK) {
200+ switch (opf & REQ_OP_MASK) {
201201 case REQ_OP_READ:
202202 return KYBER_READ;
203203 case REQ_OP_WRITE:
@@ -553,13 +553,13 @@ static void rq_clear_domain_token(struct kyber_queue_data *kqd, · 349 unmodified lines
553553 }
554554 }
555555
556−static void kyber_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
556+static void kyber_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data)
557557 {
558558 /*
559559 * We use the scheduler tags as per-hardware queue queueing tokens.
560560 * Async requests can be limited at this stage.
561561 */
562− if (!op_is_sync(op)) {
562+ if (!op_is_sync(opf)) {
563563 struct kyber_queue_data *kqd = data->q->elevator->elevator_data;
564564
565565 data->shallow_depth = kqd->async_depth;
block/mq-deadline.c+2−2
@@ -543,12 +543,12 @@ unlock: · 542 unmodified lines
543543 * Called by __blk_mq_alloc_request(). The shallow_depth value set by this
544544 * function is used by __blk_mq_get_tag().
545545 */
546−static void dd_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
546+static void dd_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data)
547547 {
548548 struct deadline_data *dd = data->q->elevator->elevator_data;
549549
550550 /* Do not throttle synchronous reads. */
551− if (op_is_sync(op) && !op_is_write(op))
551+ if (op_is_sync(opf) && !op_is_write(opf))
552552 return;
553553
554554 /*
block/partitions/core.c+1−2
@@ -9,7 +9,6 @@ · 8 unmodified lines
99 #include <linux/slab.h>
1010 #include <linux/ctype.h>
1111 #include <linux/vmalloc.h>
12−#include <linux/blktrace_api.h>
1312 #include <linux/raid/detect.h>
1413 #include "check.h"
1514
@@ -331,7 +330,7 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, · 315 unmodified lines
331330 case BLK_ZONED_HA:
332331 pr_info("%s: disabling host aware zoned block device support due to partitions\n",
333332 disk->disk_name);
334− blk_queue_set_zoned(disk, BLK_ZONED_NONE);
333+ disk_set_zoned(disk, BLK_ZONED_NONE);
335334 break;
336335 case BLK_ZONED_NONE:
337336 break;
drivers/block/Kconfig+9−0
@@ -408,6 +408,15 @@ config BLK_DEV_RBD · 407 unmodified lines
408408
409409 If unsure, say N.
410410
411+config BLK_DEV_UBLK
412+ tristate "Userspace block driver (Experimental)"
413+ select IO_URING
414+ help
415+ io_uring based userspace block driver. Together with ublk server, ublk
416+ has been working well, but interface with userspace or command data
417+ definition isn't finalized yet, and might change according to future
418+ requirement, so mark is as experimental now.
419+
411420 source "drivers/block/rnbd/Kconfig"
412421
413422 endif # BLK_DEV
drivers/block/Makefile+2−0
@@ -39,4 +39,6 @@ obj-$(CONFIG_BLK_DEV_RNBD) += rnbd/ · 38 unmodified lines
3939
4040 obj-$(CONFIG_BLK_DEV_NULL_BLK) += null_blk/
4141
42+obj-$(CONFIG_BLK_DEV_UBLK) += ublk_drv.o
43+
4244 swim_mod-y := swim.o swim_asm.o
drivers/block/amiflop.c+1−1
@@ -1802,7 +1802,7 @@ static int fd_alloc_disk(int drive, int system) · 1801 unmodified lines
18021802 unit[drive].gendisk[system] = disk;
18031803 err = add_disk(disk);
18041804 if (err)
1805− blk_cleanup_disk(disk);
1805+ put_disk(disk);
18061806 return err;
18071807 }
18081808
drivers/block/aoe/aoeblk.c+1−1
@@ -427,7 +427,7 @@ aoeblk_gdalloc(void *vp) · 426 unmodified lines
427427 return;
428428
429429 out_disk_cleanup:
430− blk_cleanup_disk(gd);
430+ put_disk(gd);
431431 err_tagset:
432432 blk_mq_free_tag_set(set);
433433 err_mempool:
drivers/block/aoe/aoedev.c+1−1
@@ -277,7 +277,7 @@ freedev(struct aoedev *d) · 276 unmodified lines
277277 if (d->gd) {
278278 aoedisk_rm_debugfs(d);
279279 del_gendisk(d->gd);
280− blk_cleanup_disk(d->gd);
280+ put_disk(d->gd);
281281 blk_mq_free_tag_set(&d->tag_set);
282282 }
283283 t = d->targets;
drivers/block/ataflop.c+2−3
@@ -2031,7 +2031,7 @@ static void ataflop_probe(dev_t dev) · 2030 unmodified lines
20312031 return;
20322032
20332033 cleanup_disk:
2034− blk_cleanup_disk(unit[drive].disk[type]);
2034+ put_disk(unit[drive].disk[type]);
20352035 unit[drive].disk[type] = NULL;
20362036 }
20372037
@@ -2045,7 +2045,6 @@ static void atari_floppy_cleanup(void) · 7 unmodified lines
20452045 if (!unit[i].disk[type])
20462046 continue;
20472047 del_gendisk(unit[i].disk[type]);
2048− blk_cleanup_queue(unit[i].disk[type]->queue);
20492048 put_disk(unit[i].disk[type]);
20502049 }
20512050 blk_mq_free_tag_set(&unit[i].tag_set);
@@ -2064,7 +2063,7 @@ static void atari_cleanup_floppy_disk(struct atari_floppy_struct *fs) · 12 unmodified lines
20642063 continue;
20652064 if (fs->registered[type])
20662065 del_gendisk(fs->disk[type]);
2067− blk_cleanup_disk(fs->disk[type]);
2066+ put_disk(fs->disk[type]);
20682067 }
20692068 blk_mq_free_tag_set(&fs->tag_set);
20702069 }
drivers/block/brd.c+4−4
@@ -256,7 +256,7 @@ static void copy_from_brd(void *dst, struct brd_device *brd, · 255 unmodified lines
256256 * Process a single bvec of a bio.
257257 */
258258 static int brd_do_bvec(struct brd_device *brd, struct page *page,
259− unsigned int len, unsigned int off, unsigned int op,
259+ unsigned int len, unsigned int off, enum req_op op,
260260 sector_t sector)
261261 {
262262 void *mem;
@@ -310,7 +310,7 @@ static void brd_submit_bio(struct bio *bio) · 47 unmodified lines
310310 }
311311
312312 static int brd_rw_page(struct block_device *bdev, sector_t sector,
313− struct page *page, unsigned int op)
313+ struct page *page, enum req_op op)
314314 {
315315 struct brd_device *brd = bdev->bd_disk->private_data;
316316 int err;
@@ -419,7 +419,7 @@ static int brd_alloc(int i) · 102 unmodified lines
419419 return 0;
420420
421421 out_cleanup_disk:
422− blk_cleanup_disk(disk);
422+ put_disk(disk);
423423 out_free_dev:
424424 list_del(&brd->brd_list);
425425 kfree(brd);
@@ -439,7 +439,7 @@ static void brd_cleanup(void) · 13 unmodified lines
439439
440440 list_for_each_entry_safe(brd, next, &brd_devices, brd_list) {
441441 del_gendisk(brd->brd_disk);
442− blk_cleanup_disk(brd->brd_disk);
442+ put_disk(brd->brd_disk);
443443 brd_free_pages(brd);
444444 list_del(&brd->brd_list);
445445 kfree(brd);
drivers/block/drbd/drbd_actlog.c+5−4
@@ -124,12 +124,13 @@ void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_b · 123 unmodified lines
124124
125125 static int _drbd_md_sync_page_io(struct drbd_device *device,
126126 struct drbd_backing_dev *bdev,
127− sector_t sector, int op)
127+ sector_t sector, enum req_op op)
128128 {
129129 struct bio *bio;
130130 /* we do all our meta data IO in aligned 4k blocks. */
131131 const int size = 4096;
132− int err, op_flags = 0;
132+ int err;
133+ blk_opf_t op_flags = 0;
133134
134135 device->md_io.done = 0;
135136 device->md_io.error = -ENODEV;
@@ -174,7 +175,7 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, · 38 unmodified lines
174175 }
175176
176177 int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bdev,
177− sector_t sector, int op)
178+ sector_t sector, enum req_op op)
178179 {
179180 int err;
180181 D_ASSERT(device, atomic_read(&device->md_io.in_use) == 1);
@@ -385,7 +386,7 @@ static int __al_write_transaction(struct drbd_device *device, struct al_transact · 204 unmodified lines
385386 write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
386387 rcu_read_unlock();
387388 if (write_al_updates) {
388− if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
389+ if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) {
389390 err = -EIO;
390391 drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
391392 } else {
drivers/block/drbd/drbd_bitmap.c+1−1
@@ -977,7 +977,7 @@ static void drbd_bm_endio(struct bio *bio) · 976 unmodified lines
977977 static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_hold(local)
978978 {
979979 struct drbd_device *device = ctx->device;
980− unsigned int op = (ctx->flags & BM_AIO_READ) ? REQ_OP_READ : REQ_OP_WRITE;
980+ enum req_op op = ctx->flags & BM_AIO_READ ? REQ_OP_READ : REQ_OP_WRITE;
981981 struct bio *bio = bio_alloc_bioset(device->ldev->md_bdev, 1, op,
982982 GFP_NOIO, &drbd_md_io_bio_set);
983983 struct drbd_bitmap *b = device->bitmap;
drivers/block/drbd/drbd_int.h+2−3
@@ -1495,7 +1495,7 @@ extern int drbd_resync_finished(struct drbd_device *device); · 1494 unmodified lines
14951495 extern void *drbd_md_get_buffer(struct drbd_device *device, const char *intent);
14961496 extern void drbd_md_put_buffer(struct drbd_device *device);
14971497 extern int drbd_md_sync_page_io(struct drbd_device *device,
1498− struct drbd_backing_dev *bdev, sector_t sector, int op);
1498+ struct drbd_backing_dev *bdev, sector_t sector, enum req_op op);
14991499 extern void drbd_ov_out_of_sync_found(struct drbd_device *, sector_t, int);
15001500 extern void wait_until_done_or_force_detached(struct drbd_device *device,
15011501 struct drbd_backing_dev *bdev, unsigned int *done);
@@ -1547,8 +1547,7 @@ extern bool drbd_rs_c_min_rate_throttle(struct drbd_device *device); · 45 unmodified lines
15471547 extern bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector,
15481548 bool throttle_if_app_is_waiting);
15491549 extern int drbd_submit_peer_request(struct drbd_device *,
1550− struct drbd_peer_request *, const unsigned,
1551− const unsigned, const int);
1550+ struct drbd_peer_request *, blk_opf_t, int);
15521551 extern int drbd_free_peer_reqs(struct drbd_device *, struct list_head *);
15531552 extern struct drbd_peer_request *drbd_alloc_peer_req(struct drbd_peer_device *, u64,
15541553 sector_t, unsigned int,
drivers/block/drbd/drbd_main.c+2−2
@@ -2207,7 +2207,7 @@ void drbd_destroy_device(struct kref *kref) · 2206 unmodified lines
22072207 if (device->bitmap) /* should no longer be there. */
22082208 drbd_bm_cleanup(device);
22092209 __free_page(device->md_io.page);
2210− blk_cleanup_disk(device->vdisk);
2210+ put_disk(device->vdisk);
22112211 kfree(device->rs_plan_s);
22122212
22132213 /* not for_each_connection(connection, resource):
@@ -2807,7 +2807,7 @@ out_no_minor_idr: · 593 unmodified lines
28072807 out_no_bitmap:
28082808 __free_page(device->md_io.page);
28092809 out_no_io_page:
2810− blk_cleanup_disk(disk);
2810+ put_disk(disk);
28112811 out_no_disk:
28122812 kref_put(&resource->kref, drbd_destroy_resource);
28132813 kfree(device);
drivers/block/drbd/drbd_receiver.c+12−12
@@ -1621,8 +1621,7 @@ static void drbd_issue_peer_discard_or_zero_out(struct drbd_device *device, stru · 1620 unmodified lines
16211621 /* TODO allocate from our own bio_set. */
16221622 int drbd_submit_peer_request(struct drbd_device *device,
16231623 struct drbd_peer_request *peer_req,
1624− const unsigned op, const unsigned op_flags,
1625− const int fault_type)
1624+ const blk_opf_t opf, const int fault_type)
16261625 {
16271626 struct bio *bios = NULL;
16281627 struct bio *bio;
@@ -1668,8 +1667,7 @@ int drbd_submit_peer_request(struct drbd_device *device, · 39 unmodified lines
16681667 * generated bio, but a bio allocated on behalf of the peer.
16691668 */
16701669 next_bio:
1671− bio = bio_alloc(device->ldev->backing_bdev, nr_pages, op | op_flags,
1672− GFP_NOIO);
1670+ bio = bio_alloc(device->ldev->backing_bdev, nr_pages, opf, GFP_NOIO);
16731671 /* > peer_req->i.sector, unless this is the first bio */
16741672 bio->bi_iter.bi_sector = sector;
16751673 bio->bi_private = peer_req;
@@ -2060,7 +2058,7 @@ static int recv_resync_read(struct drbd_peer_device *peer_device, sector_t secto · 384 unmodified lines
20602058 spin_unlock_irq(&device->resource->req_lock);
20612059
20622060 atomic_add(pi->size >> 9, &device->rs_sect_ev);
2063− if (drbd_submit_peer_request(device, peer_req, REQ_OP_WRITE, 0,
2061+ if (drbd_submit_peer_request(device, peer_req, REQ_OP_WRITE,
20642062 DRBD_FAULT_RS_WR) == 0)
20652063 return 0;
20662064
@@ -2383,14 +2381,14 @@ static int wait_for_and_update_peer_seq(struct drbd_peer_device *peer_device, co · 316 unmodified lines
23832381 /* see also bio_flags_to_wire()
23842382 * DRBD_REQ_*, because we need to semantically map the flags to data packet
23852383 * flags and back. We may replicate to other kernel versions. */
2386−static unsigned long wire_flags_to_bio_flags(u32 dpf)
2384+static blk_opf_t wire_flags_to_bio_flags(u32 dpf)
23872385 {
23882386 return (dpf & DP_RW_SYNC ? REQ_SYNC : 0) |
23892387 (dpf & DP_FUA ? REQ_FUA : 0) |
23902388 (dpf & DP_FLUSH ? REQ_PREFLUSH : 0);
23912389 }
23922390
2393−static unsigned long wire_flags_to_bio_op(u32 dpf)
2391+static enum req_op wire_flags_to_bio_op(u32 dpf)
23942392 {
23952393 if (dpf & DP_ZEROES)
23962394 return REQ_OP_WRITE_ZEROES;
@@ -2543,7 +2541,8 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * · 146 unmodified lines
25432541 struct drbd_peer_request *peer_req;
25442542 struct p_data *p = pi->data;
25452543 u32 peer_seq = be32_to_cpu(p->seq_num);
2546− int op, op_flags;
2544+ enum req_op op;
2545+ blk_opf_t op_flags;
25472546 u32 dp_flags;
25482547 int err, tp;
25492548
@@ -2681,7 +2680,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * · 131 unmodified lines
26812680 peer_req->flags |= EE_CALL_AL_COMPLETE_IO;
26822681 }
26832682
2684− err = drbd_submit_peer_request(device, peer_req, op, op_flags,
2683+ err = drbd_submit_peer_request(device, peer_req, op | op_flags,
26852684 DRBD_FAULT_DT_WR);
26862685 if (!err)
26872686 return 0;
@@ -2979,7 +2978,7 @@ submit_for_resync: · 291 unmodified lines
29792978 submit:
29802979 update_receiver_timing_details(connection, drbd_submit_peer_request);
29812980 inc_unacked(device);
2982− if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0,
2981+ if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ,
29832982 fault_type) == 0)
29842983 return 0;
29852984
@@ -4951,7 +4950,7 @@ static int receive_rs_deallocated(struct drbd_connection *connection, struct pac · 1965 unmodified lines
49514950
49524951 if (get_ldev(device)) {
49534952 struct drbd_peer_request *peer_req;
4954− const int op = REQ_OP_WRITE_ZEROES;
4953+ const enum req_op op = REQ_OP_WRITE_ZEROES;
49554954
49564955 peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER, sector,
49574956 size, 0, GFP_NOIO);
@@ -4969,7 +4968,8 @@ static int receive_rs_deallocated(struct drbd_connection *connection, struct pac · 11 unmodified lines
49694968 spin_unlock_irq(&device->resource->req_lock);
49704969
49714970 atomic_add(pi->size >> 9, &device->rs_sect_ev);
4972− err = drbd_submit_peer_request(device, peer_req, op, 0, DRBD_FAULT_RS_WR);
4971+ err = drbd_submit_peer_request(device, peer_req, op,
4972+ DRBD_FAULT_RS_WR);
49734973
49744974 if (err) {
49754975 spin_lock_irq(&device->resource->req_lock);
drivers/block/drbd/drbd_req.c+2−4
@@ -523,16 +523,14 @@ static void mod_rq_state(struct drbd_request *req, struct bio_and_error *m, · 522 unmodified lines
523523
524524 static void drbd_report_io_error(struct drbd_device *device, struct drbd_request *req)
525525 {
526− char b[BDEVNAME_SIZE];
527−
528526 if (!__ratelimit(&drbd_ratelimit_state))
529527 return;
530528
531− drbd_warn(device, "local %s IO error sector %llu+%u on %s\n",
529+ drbd_warn(device, "local %s IO error sector %llu+%u on %pg\n",
532530 (req->rq_state & RQ_WRITE) ? "WRITE" : "READ",
533531 (unsigned long long)req->i.sector,
534532 req->i.size >> 9,
535− bdevname(device->ldev->backing_bdev, b));
533+ device->ldev->backing_bdev);
536534 }
537535
538536 /* Helper for HANDED_OVER_TO_NETWORK.
drivers/block/drbd/drbd_worker.c+1−1
@@ -405,7 +405,7 @@ static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, · 404 unmodified lines
405405 spin_unlock_irq(&device->resource->req_lock);
406406
407407 atomic_add(size >> 9, &device->rs_sect_ev);
408− if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0,
408+ if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ,
409409 DRBD_FAULT_RS_RD) == 0)
410410 return 0;
411411
drivers/block/floppy.c+4−4
@@ -2859,7 +2859,7 @@ static blk_status_t floppy_queue_rq(struct blk_mq_hw_ctx *hctx, · 2858 unmodified lines
28592859 if (WARN(atomic_read(&usage_count) == 0,
28602860 "warning: usage count=0, current_req=%p sect=%ld flags=%llx\n",
28612861 current_req, (long)blk_rq_pos(current_req),
2862− (unsigned long long) current_req->cmd_flags))
2862+ (__force unsigned long long) current_req->cmd_flags))
28632863 return BLK_STS_IOERR;
28642864
28652865 if (test_and_set_bit(0, &fdc_busy)) {
@@ -4557,7 +4557,7 @@ out: · 1691 unmodified lines
45574557 return;
45584558
45594559 cleanup_disk:
4560− blk_cleanup_disk(disks[drive][type]);
4560+ put_disk(disks[drive][type]);
45614561 disks[drive][type] = NULL;
45624562 mutex_unlock(&floppy_probe_lock);
45634563 }
@@ -4753,7 +4753,7 @@ out_put_disk: · 189 unmodified lines
47534753 if (!disks[drive][0])
47544754 break;
47554755 del_timer_sync(&motor_off_timer[drive]);
4756− blk_cleanup_disk(disks[drive][0]);
4756+ put_disk(disks[drive][0]);
47574757 blk_mq_free_tag_set(&tag_sets[drive]);
47584758 }
47594759 return err;
@@ -4985,7 +4985,7 @@ static void __exit floppy_module_exit(void) · 225 unmodified lines
49854985 }
49864986 for (i = 0; i < ARRAY_SIZE(floppy_type); i++) {
49874987 if (disks[drive][i])
4988− blk_cleanup_disk(disks[drive][i]);
4988+ put_disk(disks[drive][i]);
49894989 }
49904990 blk_mq_free_tag_set(&tag_sets[drive]);
49914991 }
drivers/block/loop.c+1−2
@@ -2040,7 +2040,7 @@ static int loop_add(int i) · 2039 unmodified lines
20402040 return i;
20412041
20422042 out_cleanup_disk:
2043− blk_cleanup_disk(disk);
2043+ put_disk(disk);
20442044 out_cleanup_tags:
20452045 blk_mq_free_tag_set(&lo->tag_set);
20462046 out_free_idr:
@@ -2057,7 +2057,6 @@ static void loop_remove(struct loop_device *lo) · 10 unmodified lines
20572057 {
20582058 /* Make this loop device unreachable from pathname. */
20592059 del_gendisk(lo->lo_disk);
2060− blk_cleanup_queue(lo->lo_disk->queue);
20612060 blk_mq_free_tag_set(&lo->tag_set);
20622061
20632062 mutex_lock(&loop_ctl_mutex);
drivers/block/mtip32xx/mtip32xx.c+48−259
@@ -94,17 +94,12 @@ · 93 unmodified lines
9494 /* Device instance number, incremented each time a device is probed. */
9595 static int instance;
9696
97−static LIST_HEAD(online_list);
98−static LIST_HEAD(removing_list);
99−static DEFINE_SPINLOCK(dev_lock);
100−
10197 /*
10298 * Global variable used to hold the major block device number
10399 * allocated in mtip_init().
104100 */
105101 static int mtip_major;
106102 static struct dentry *dfs_parent;
107−static struct dentry *dfs_device_status;
108103
109104 static u32 cpu_use[NR_CPUS];
110105
@@ -146,11 +141,8 @@ static bool mtip_check_surprise_removal(struct driver_data *dd) · 35 unmodified lines
146141 pci_read_config_word(dd->pdev, 0x00, &vendor_id);
147142 if (vendor_id == 0xFFFF) {
148143 dd->sr = true;
149− if (dd->queue)
150− blk_queue_flag_set(QUEUE_FLAG_DEAD, dd->queue);
151− else
152− dev_warn(&dd->pdev->dev,
153− "%s: dd->queue is NULL\n", __func__);
144+ if (dd->disk)
145+ blk_mark_disk_dead(dd->disk);
154146 return true; /* device removed */
155147 }
156148
@@ -2170,106 +2162,6 @@ static const struct attribute_group *mtip_disk_attr_groups[] = { · 2013 unmodified lines
21702162 NULL,
21712163 };
21722164
2173−/* debugsfs entries */
2174−
2175−static ssize_t show_device_status(struct device_driver *drv, char *buf)
2176−{
2177− int size = 0;
2178− struct driver_data *dd, *tmp;
2179− unsigned long flags;
2180− char id_buf[42];
2181− u16 status = 0;
2182−
2183− spin_lock_irqsave(&dev_lock, flags);
2184− size += sprintf(&buf[size], "Devices Present:\n");
2185− list_for_each_entry_safe(dd, tmp, &online_list, online_list) {
2186− if (dd->pdev) {
2187− if (dd->port &&
2188− dd->port->identify &&
2189− dd->port->identify_valid) {
2190− strlcpy(id_buf,
2191− (char *) (dd->port->identify + 10), 21);
2192− status = *(dd->port->identify + 141);
2193− } else {
2194− memset(id_buf, 0, 42);
2195− status = 0;
2196− }
2197−
2198− if (dd->port &&
2199− test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
2200− size += sprintf(&buf[size],
2201− " device %s %s (ftl rebuild %d %%)\n",
2202− dev_name(&dd->pdev->dev),
2203− id_buf,
2204− status);
2205− } else {
2206− size += sprintf(&buf[size],
2207− " device %s %s\n",
2208− dev_name(&dd->pdev->dev),
2209− id_buf);
2210− }
2211− }
2212− }
2213−
2214− size += sprintf(&buf[size], "Devices Being Removed:\n");
2215− list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) {
2216− if (dd->pdev) {
2217− if (dd->port &&
2218− dd->port->identify &&
2219− dd->port->identify_valid) {
2220− strlcpy(id_buf,
2221− (char *) (dd->port->identify+10), 21);
2222− status = *(dd->port->identify + 141);
2223− } else {
2224− memset(id_buf, 0, 42);
2225− status = 0;
2226− }
2227−
2228− if (dd->port &&
2229− test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
2230− size += sprintf(&buf[size],
2231− " device %s %s (ftl rebuild %d %%)\n",
2232− dev_name(&dd->pdev->dev),
2233− id_buf,
2234− status);
2235− } else {
2236− size += sprintf(&buf[size],
2237− " device %s %s\n",
2238− dev_name(&dd->pdev->dev),
2239− id_buf);
2240− }
2241− }
2242− }
2243− spin_unlock_irqrestore(&dev_lock, flags);
2244−
2245− return size;
2246−}
2247−
2248−static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf,
2249− size_t len, loff_t *offset)
2250−{
2251− int size = *offset;
2252− char *buf;
2253− int rv = 0;
2254−
2255− if (!len || *offset)
2256− return 0;
2257−
2258− buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2259− if (!buf)
2260− return -ENOMEM;
2261−
2262− size += show_device_status(NULL, buf);
2263−
2264− *offset = size <= len ? size : len;
2265− size = copy_to_user(ubuf, buf, *offset);
2266− if (size)
2267− rv = -EFAULT;
2268−
2269− kfree(buf);
2270− return rv ? rv : *offset;
2271−}
2272−
22732165 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf,
22742166 size_t len, loff_t *offset)
22752167 {
@@ -2363,13 +2255,6 @@ static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf, · 87 unmodified lines
23632255 return rv ? rv : *offset;
23642256 }
23652257
2366−static const struct file_operations mtip_device_status_fops = {
2367− .owner = THIS_MODULE,
2368− .open = simple_open,
2369− .read = mtip_hw_read_device_status,
2370− .llseek = no_llseek,
2371−};
2372−
23732258 static const struct file_operations mtip_regs_fops = {
23742259 .owner = THIS_MODULE,
23752260 .open = simple_open,
@@ -2556,7 +2441,7 @@ static void mtip_softirq_done_fn(struct request *rq) · 180 unmodified lines
25562441 blk_mq_end_request(rq, cmd->status);
25572442 }
25582443
2559−static bool mtip_abort_cmd(struct request *req, void *data, bool reserved)
2444+static bool mtip_abort_cmd(struct request *req, void *data)
25602445 {
25612446 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
25622447 struct driver_data *dd = data;
@@ -2569,7 +2454,7 @@ static bool mtip_abort_cmd(struct request *req, void *data, bool reserved) · 6 unmodified lines
25692454 return true;
25702455 }
25712456
2572−static bool mtip_queue_cmd(struct request *req, void *data, bool reserved)
2457+static bool mtip_queue_cmd(struct request *req, void *data)
25732458 {
25742459 struct driver_data *dd = data;
25752460
@@ -3297,26 +3182,12 @@ static int mtip_block_getgeo(struct block_device *dev, · 721 unmodified lines
32973182 return 0;
32983183 }
32993184
3300−static int mtip_block_open(struct block_device *dev, fmode_t mode)
3185+static void mtip_block_free_disk(struct gendisk *disk)
33013186 {
3302− struct driver_data *dd;
3187+ struct driver_data *dd = disk->private_data;
33033188
3304− if (dev && dev->bd_disk) {
3305− dd = (struct driver_data *) dev->bd_disk->private_data;
3306−
3307− if (dd) {
3308− if (test_bit(MTIP_DDF_REMOVAL_BIT,
3309− &dd->dd_flag)) {
3310− return -ENODEV;
3311− }
3312− return 0;
3313− }
3314− }
3315− return -ENODEV;
3316−}
3317−
3318−static void mtip_block_release(struct gendisk *disk, fmode_t mode)
3319−{
3189+ ida_free(&rssd_index_ida, dd->index);
3190+ kfree(dd);
33203191 }
33213192
33223193 /*
@@ -3326,13 +3197,12 @@ static void mtip_block_release(struct gendisk *disk, fmode_t mode) · 3 unmodified lines
33263197 * layer.
33273198 */
33283199 static const struct block_device_operations mtip_block_ops = {
3329− .open = mtip_block_open,
3330− .release = mtip_block_release,
33313200 .ioctl = mtip_block_ioctl,
33323201 #ifdef CONFIG_COMPAT
33333202 .compat_ioctl = mtip_block_compat_ioctl,
33343203 #endif
33353204 .getgeo = mtip_block_getgeo,
3205+ .free_disk = mtip_block_free_disk,
33363206 .owner = THIS_MODULE
33373207 };
33383208
@@ -3487,12 +3357,11 @@ static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq, · 148 unmodified lines
34873357 return 0;
34883358 }
34893359
3490−static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req,
3491− bool reserved)
3360+static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req)
34923361 {
34933362 struct driver_data *dd = req->q->queuedata;
34943363
3495− if (reserved) {
3364+ if (blk_mq_is_reserved_rq(req)) {
34963365 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
34973366
34983367 cmd->status = BLK_STS_TIMEOUT;
@@ -3664,7 +3533,7 @@ init_hw_cmds_error: · 165 unmodified lines
36643533 disk_index_error:
36653534 ida_free(&rssd_index_ida, index);
36663535 ida_get_error:
3667− blk_cleanup_disk(dd->disk);
3536+ put_disk(dd->disk);
36683537 block_queue_alloc_init_error:
36693538 blk_mq_free_tag_set(&dd->tags);
36703539 block_queue_alloc_tag_error:
@@ -3673,72 +3542,6 @@ protocol_init_error: · 2 unmodified lines
36733542 return rv;
36743543 }
36753544
3676−static bool mtip_no_dev_cleanup(struct request *rq, void *data, bool reserv)
3677−{
3678− struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3679−
3680− cmd->status = BLK_STS_IOERR;
3681− blk_mq_complete_request(rq);
3682− return true;
3683−}
3684−
3685−/*
3686− * Block layer deinitialization function.
3687− *
3688− * Called by the PCI layer as each P320 device is removed.
3689− *
3690− * @dd Pointer to the driver data structure.
3691− *
3692− * return value
3693− * 0
3694− */
3695−static int mtip_block_remove(struct driver_data *dd)
3696−{
3697− mtip_hw_debugfs_exit(dd);
3698−
3699− if (dd->mtip_svc_handler) {
3700− set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
3701− wake_up_interruptible(&dd->port->svc_wait);
3702− kthread_stop(dd->mtip_svc_handler);
3703− }
3704−
3705− if (!dd->sr) {
3706− /*
3707− * Explicitly wait here for IOs to quiesce,
3708− * as mtip_standby_drive usually won't wait for IOs.
3709− */
3710− if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS))
3711− mtip_standby_drive(dd);
3712− }
3713− else
3714− dev_info(&dd->pdev->dev, "device %s surprise removal\n",
3715− dd->disk->disk_name);
3716−
3717− blk_freeze_queue_start(dd->queue);
3718− blk_mq_quiesce_queue(dd->queue);
3719− blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd);
3720− blk_mq_unquiesce_queue(dd->queue);
3721−
3722− if (dd->disk) {
3723− if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3724− del_gendisk(dd->disk);
3725− if (dd->disk->queue) {
3726− blk_cleanup_queue(dd->queue);
3727− blk_mq_free_tag_set(&dd->tags);
3728− dd->queue = NULL;
3729− }
3730− put_disk(dd->disk);
3731− }
3732− dd->disk = NULL;
3733−
3734− ida_free(&rssd_index_ida, dd->index);
3735−
3736− /* De-initialize the protocol layer. */
3737− mtip_hw_exit(dd);
3738−
3739− return 0;
3740−}
3741−
37423545 /*
37433546 * Function called by the PCI layer when just before the
37443547 * machine shuts down.
@@ -3755,23 +3558,14 @@ static int mtip_block_shutdown(struct driver_data *dd) · 10 unmodified lines
37553558 {
37563559 mtip_hw_shutdown(dd);
37573560
3758− /* Delete our gendisk structure, and cleanup the blk queue. */
3759− if (dd->disk) {
3760− dev_info(&dd->pdev->dev,
3761− "Shutting down %s ...\n", dd->disk->disk_name);
3561+ dev_info(&dd->pdev->dev,
3562+ "Shutting down %s ...\n", dd->disk->disk_name);
37623563
3763− if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3764− del_gendisk(dd->disk);
3765− if (dd->disk->queue) {
3766− blk_cleanup_queue(dd->queue);
3767− blk_mq_free_tag_set(&dd->tags);
3768− }
3769− put_disk(dd->disk);
3770− dd->disk = NULL;
3771− dd->queue = NULL;
3772− }
3564+ if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3565+ del_gendisk(dd->disk);
37733566
3774− ida_free(&rssd_index_ida, dd->index);
3567+ blk_mq_free_tag_set(&dd->tags);
3568+ put_disk(dd->disk);
37753569 return 0;
37763570 }
37773571
@@ -3905,7 +3699,6 @@ static int mtip_pci_probe(struct pci_dev *pdev, · 127 unmodified lines
39053699 const struct cpumask *node_mask;
39063700 int cpu, i = 0, j = 0;
39073701 int my_node = NUMA_NO_NODE;
3908− unsigned long flags;
39093702
39103703 /* Allocate memory for this devices private data. */
39113704 my_node = pcibus_to_node(pdev->bus);
@@ -3952,9 +3745,6 @@ static int mtip_pci_probe(struct pci_dev *pdev, · 40 unmodified lines
39523745 dd->pdev = pdev;
39533746 dd->numa_node = my_node;
39543747
3955− INIT_LIST_HEAD(&dd->online_list);
3956− INIT_LIST_HEAD(&dd->remove_list);
3957−
39583748 memset(dd->workq_name, 0, 32);
39593749 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance);
39603750
@@ -4047,11 +3837,6 @@ static int mtip_pci_probe(struct pci_dev *pdev, · 86 unmodified lines
40473837 else
40483838 rv = 0; /* device in rebuild state, return 0 from probe */
40493839
4050− /* Add to online list even if in ftl rebuild */
4051− spin_lock_irqsave(&dev_lock, flags);
4052− list_add(&dd->online_list, &online_list);
4053− spin_unlock_irqrestore(&dev_lock, flags);
4054−
40553840 goto done;
40563841
40573842 block_initialize_err:
@@ -4085,14 +3870,7 @@ done: · 27 unmodified lines
40853870 static void mtip_pci_remove(struct pci_dev *pdev)
40863871 {
40873872 struct driver_data *dd = pci_get_drvdata(pdev);
4088− unsigned long flags, to;
4089−
4090− set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag);
4091−
4092− spin_lock_irqsave(&dev_lock, flags);
4093− list_del_init(&dd->online_list);
4094− list_add(&dd->remove_list, &removing_list);
4095− spin_unlock_irqrestore(&dev_lock, flags);
3873+ unsigned long to;
40963874
40973875 mtip_check_surprise_removal(dd);
40983876 synchronize_irq(dd->pdev->irq);
@@ -4109,11 +3887,35 @@ static void mtip_pci_remove(struct pci_dev *pdev) · 10 unmodified lines
41093887 "Completion workers still active!\n");
41103888 }
41113889
4112− blk_mark_disk_dead(dd->disk);
41133890 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag);
41143891
4115− /* Clean up the block layer. */
4116− mtip_block_remove(dd);
3892+ if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3893+ del_gendisk(dd->disk);
3894+
3895+ mtip_hw_debugfs_exit(dd);
3896+
3897+ if (dd->mtip_svc_handler) {
3898+ set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
3899+ wake_up_interruptible(&dd->port->svc_wait);
3900+ kthread_stop(dd->mtip_svc_handler);
3901+ }
3902+
3903+ if (!dd->sr) {
3904+ /*
3905+ * Explicitly wait here for IOs to quiesce,
3906+ * as mtip_standby_drive usually won't wait for IOs.
3907+ */
3908+ if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS))
3909+ mtip_standby_drive(dd);
3910+ }
3911+ else
3912+ dev_info(&dd->pdev->dev, "device %s surprise removal\n",
3913+ dd->disk->disk_name);
3914+
3915+ blk_mq_free_tag_set(&dd->tags);
3916+
3917+ /* De-initialize the protocol layer. */
3918+ mtip_hw_exit(dd);
41173919
41183920 if (dd->isr_workq) {
41193921 destroy_workqueue(dd->isr_workq);
@@ -4124,14 +3926,10 @@ static void mtip_pci_remove(struct pci_dev *pdev) · 4 unmodified lines
41243926
41253927 pci_disable_msi(pdev);
41263928
4127− spin_lock_irqsave(&dev_lock, flags);
4128− list_del_init(&dd->remove_list);
4129− spin_unlock_irqrestore(&dev_lock, flags);
4130−
4131− kfree(dd);
4132−
41333929 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
41343930 pci_set_drvdata(pdev, NULL);
3931+
3932+ put_disk(dd->disk);
41353933 }
41363934
41373935 /*
@@ -4250,15 +4048,6 @@ static int __init mtip_init(void) · 112 unmodified lines
42504048 pr_warn("Error creating debugfs parent\n");
42514049 dfs_parent = NULL;
42524050 }
4253− if (dfs_parent) {
4254− dfs_device_status = debugfs_create_file("device_status",
4255− 0444, dfs_parent, NULL,
4256− &mtip_device_status_fops);
4257− if (IS_ERR_OR_NULL(dfs_device_status)) {
4258− pr_err("Error creating device_status node\n");
4259− dfs_device_status = NULL;
4260− }
4261− }
42624051
42634052 /* Register our PCI operations. */
42644053 error = pci_register_driver(&mtip_pci_driver);
drivers/block/mtip32xx/mtip32xx.h+0−5
@@ -149,7 +149,6 @@ enum { · 148 unmodified lines
149149 MTIP_DDF_RESUME_BIT = 6,
150150 MTIP_DDF_INIT_DONE_BIT = 7,
151151 MTIP_DDF_REBUILD_FAILED_BIT = 8,
152− MTIP_DDF_REMOVAL_BIT = 9,
153152
154153 MTIP_DDF_STOP_IO = ((1 << MTIP_DDF_REMOVE_PENDING_BIT) |
155154 (1 << MTIP_DDF_SEC_LOCK_BIT) |
@@ -462,10 +461,6 @@ struct driver_data { · 306 unmodified lines
462461
463462 int isr_binding;
464463
465− struct list_head online_list; /* linkage for online list */
466−
467− struct list_head remove_list; /* linkage for removing list */
468−
469464 int unal_qdepth; /* qdepth of unaligned IO queue */
470465 };
471466
drivers/block/n64cart.c+1−1
@@ -157,7 +157,7 @@ static int __init n64cart_probe(struct platform_device *pdev) · 156 unmodified lines
157157 return 0;
158158
159159 out_cleanup_disk:
160− blk_cleanup_disk(disk);
160+ put_disk(disk);
161161 out:
162162 return err;
163163 }
drivers/block/nbd.c+4−5
@@ -250,7 +250,7 @@ static void nbd_dev_remove(struct nbd_device *nbd) · 249 unmodified lines
250250 struct gendisk *disk = nbd->disk;
251251
252252 del_gendisk(disk);
253− blk_cleanup_disk(disk);
253+ put_disk(disk);
254254 blk_mq_free_tag_set(&nbd->tag_set);
255255
256256 /*
@@ -393,8 +393,7 @@ static u32 req_to_nbd_cmd_type(struct request *req) · 136 unmodified lines
393393 }
394394 }
395395
396−static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
397− bool reserved)
396+static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req)
398397 {
399398 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);
400399 struct nbd_device *nbd = cmd->nbd;
@@ -880,7 +879,7 @@ static void recv_work(struct work_struct *work) · 479 unmodified lines
880879 kfree(args);
881880 }
882881
883−static bool nbd_clear_req(struct request *req, void *data, bool reserved)
882+static bool nbd_clear_req(struct request *req, void *data)
884883 {
885884 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);
886885
@@ -1833,7 +1832,7 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs) · 946 unmodified lines
18331832 out_free_work:
18341833 destroy_workqueue(nbd->recv_workq);
18351834 out_err_disk:
1836− blk_cleanup_disk(disk);
1835+ put_disk(disk);
18371836 out_free_idr:
18381837 mutex_lock(&nbd_index_mutex);
18391838 idr_remove(&nbd_index_idr, index);
drivers/block/null_blk/main.c+7−8
@@ -1310,7 +1310,7 @@ static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd, · 1309 unmodified lines
13101310 }
13111311
13121312 static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd,
1313− enum req_opf op,
1313+ enum req_op op,
13141314 sector_t sector,
13151315 sector_t nr_sectors)
13161316 {
@@ -1381,9 +1381,8 @@ static inline void nullb_complete_cmd(struct nullb_cmd *cmd) · 64 unmodified lines
13811381 }
13821382 }
13831383
1384−blk_status_t null_process_cmd(struct nullb_cmd *cmd,
1385− enum req_opf op, sector_t sector,
1386− unsigned int nr_sectors)
1384+blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
1385+ sector_t sector, unsigned int nr_sectors)
13871386 {
13881387 struct nullb_device *dev = cmd->nq->dev;
13891388 blk_status_t ret;
@@ -1401,7 +1400,7 @@ blk_status_t null_process_cmd(struct nullb_cmd *cmd, · 11 unmodified lines
14011400 }
14021401
14031402 static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
1404− sector_t nr_sectors, enum req_opf op)
1403+ sector_t nr_sectors, enum req_op op)
14051404 {
14061405 struct nullb_device *dev = cmd->nq->dev;
14071406 struct nullb *nullb = dev->nullb;
@@ -1578,7 +1577,7 @@ static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) · 170 unmodified lines
15781577 return nr;
15791578 }
15801579
1581−static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
1580+static enum blk_eh_timer_return null_timeout_rq(struct request *rq)
15821581 {
15831582 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
15841583 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
@@ -1737,7 +1736,7 @@ static void null_del_dev(struct nullb *nullb) · 152 unmodified lines
17371736 null_restart_queue_async(nullb);
17381737 }
17391738
1740− blk_cleanup_disk(nullb->disk);
1739+ put_disk(nullb->disk);
17411740 if (dev->queue_mode == NULL_Q_MQ &&
17421741 nullb->tag_set == &nullb->__tag_set)
17431742 blk_mq_free_tag_set(nullb->tag_set);
@@ -2082,7 +2081,7 @@ static int null_add_dev(struct nullb_device *dev) · 338 unmodified lines
20822081 out_cleanup_zone:
20832082 null_free_zoned_dev(dev);
20842083 out_cleanup_disk:
2085− blk_cleanup_disk(nullb->disk);
2084+ put_disk(nullb->disk);
20862085 out_cleanup_tags:
20872086 if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
20882087 blk_mq_free_tag_set(nullb->tag_set);
drivers/block/null_blk/null_blk.h+5−7
@@ -136,9 +136,8 @@ struct nullb { · 135 unmodified lines
136136
137137 blk_status_t null_handle_discard(struct nullb_device *dev, sector_t sector,
138138 sector_t nr_sectors);
139−blk_status_t null_process_cmd(struct nullb_cmd *cmd,
140− enum req_opf op, sector_t sector,
141− unsigned int nr_sectors);
139+blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
140+ sector_t sector, unsigned int nr_sectors);
142141
143142 #ifdef CONFIG_BLK_DEV_ZONED
144143 int null_init_zoned_dev(struct nullb_device *dev, struct request_queue *q);
@@ -146,9 +145,8 @@ int null_register_zoned_dev(struct nullb *nullb); · 1 unmodified line
146145 void null_free_zoned_dev(struct nullb_device *dev);
147146 int null_report_zones(struct gendisk *disk, sector_t sector,
148147 unsigned int nr_zones, report_zones_cb cb, void *data);
149−blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd,
150− enum req_opf op, sector_t sector,
151− sector_t nr_sectors);
148+blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_op op,
149+ sector_t sector, sector_t nr_sectors);
152150 size_t null_zone_valid_read_len(struct nullb *nullb,
153151 sector_t sector, unsigned int len);
154152 #else
@@ -164,7 +162,7 @@ static inline int null_register_zoned_dev(struct nullb *nullb) · 9 unmodified lines
164162 }
165163 static inline void null_free_zoned_dev(struct nullb_device *dev) {}
166164 static inline blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd,
167− enum req_opf op, sector_t sector, sector_t nr_sectors)
165+ enum req_op op, sector_t sector, sector_t nr_sectors)
168166 {
169167 return BLK_STS_NOTSUPP;
170168 }
drivers/block/null_blk/trace.h+1−1
@@ -36,7 +36,7 @@ TRACE_EVENT(nullb_zone_op, · 35 unmodified lines
3636 TP_ARGS(cmd, zone_no, zone_cond),
3737 TP_STRUCT__entry(
3838 __array(char, disk, DISK_NAME_LEN)
39− __field(enum req_opf, op)
39+ __field(enum req_op, op)
4040 __field(unsigned int, zone_no)
4141 __field(unsigned int, zone_cond)
4242 ),
drivers/block/null_blk/zoned.c+6−6
@@ -159,7 +159,7 @@ int null_register_zoned_dev(struct nullb *nullb) · 158 unmodified lines
159159 struct nullb_device *dev = nullb->dev;
160160 struct request_queue *q = nullb->q;
161161
162− blk_queue_set_zoned(nullb->disk, BLK_ZONED_HM);
162+ disk_set_zoned(nullb->disk, BLK_ZONED_HM);
163163 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
164164 blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE);
165165
@@ -170,12 +170,12 @@ int null_register_zoned_dev(struct nullb *nullb) · 4 unmodified lines
170170 return ret;
171171 } else {
172172 blk_queue_chunk_sectors(q, dev->zone_size_sects);
173− q->nr_zones = blkdev_nr_zones(nullb->disk);
173+ nullb->disk->nr_zones = bdev_nr_zones(nullb->disk->part0);
174174 }
175175
176176 blk_queue_max_zone_append_sectors(q, dev->zone_size_sects);
177− blk_queue_max_open_zones(q, dev->zone_max_open);
178− blk_queue_max_active_zones(q, dev->zone_max_active);
177+ disk_set_max_open_zones(nullb->disk, dev->zone_max_open);
178+ disk_set_max_active_zones(nullb->disk, dev->zone_max_active);
179179
180180 return 0;
181181 }
@@ -600,7 +600,7 @@ static blk_status_t null_reset_zone(struct nullb_device *dev, · 418 unmodified lines
600600 return BLK_STS_OK;
601601 }
602602
603−static blk_status_t null_zone_mgmt(struct nullb_cmd *cmd, enum req_opf op,
603+static blk_status_t null_zone_mgmt(struct nullb_cmd *cmd, enum req_op op,
604604 sector_t sector)
605605 {
606606 struct nullb_device *dev = cmd->nq->dev;
@@ -653,7 +653,7 @@ static blk_status_t null_zone_mgmt(struct nullb_cmd *cmd, enum req_opf op, · 46 unmodified lines
653653 return ret;
654654 }
655655
656−blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_opf op,
656+blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_op op,
657657 sector_t sector, sector_t nr_sectors)
658658 {
659659 struct nullb_device *dev;
drivers/block/paride/pcd.c+2−2
@@ -956,7 +956,7 @@ out_unreg_cdrom: · 955 unmodified lines
956956 out_pi_release:
957957 pi_release(cd->pi);
958958 out_free_disk:
959− blk_cleanup_disk(cd->disk);
959+ put_disk(cd->disk);
960960 out_free_tag_set:
961961 blk_mq_free_tag_set(&cd->tag_set);
962962 return ret;
@@ -1029,7 +1029,7 @@ static void __exit pcd_exit(void) · 66 unmodified lines
10291029 unregister_cdrom(&cd->info);
10301030 del_gendisk(cd->disk);
10311031 pi_release(cd->pi);
1032− blk_cleanup_disk(cd->disk);
1032+ put_disk(cd->disk);
10331033
10341034 blk_mq_free_tag_set(&cd->tag_set);
10351035 }
drivers/block/paride/pd.c+4−2
@@ -501,6 +501,8 @@ static enum action do_pd_io_start(void) · 500 unmodified lines
501501 return do_pd_read_start();
502502 else
503503 return do_pd_write_start();
504+ default:
505+ break;
504506 }
505507 return Fail;
506508 }
@@ -943,7 +945,7 @@ static int pd_probe_drive(struct pd_unit *disk, int autoprobe, int port, · 436 unmodified lines
943945 goto cleanup_disk;
944946 return 0;
945947 cleanup_disk:
946− blk_cleanup_disk(disk->gd);
948+ put_disk(disk->gd);
947949 put_disk:
948950 put_disk(p);
949951 disk->gd = NULL;
@@ -1018,7 +1020,7 @@ static void __exit pd_exit(void) · 68 unmodified lines
10181020 if (p) {
10191021 disk->gd = NULL;
10201022 del_gendisk(p);
1021− blk_cleanup_disk(p);
1023+ put_disk(p);
10221024 blk_mq_free_tag_set(&disk->tag_set);
10231025 pi_release(disk->pi);
10241026 }
drivers/block/paride/pf.c+2−2
@@ -975,7 +975,7 @@ static int __init pf_init_unit(struct pf_unit *pf, bool autoprobe, int port, · 974 unmodified lines
975975 out_pi_release:
976976 pi_release(pf->pi);
977977 out_free_disk:
978− blk_cleanup_disk(pf->disk);
978+ put_disk(pf->disk);
979979 out_free_tag_set:
980980 blk_mq_free_tag_set(&pf->tag_set);
981981 return ret;
@@ -1044,7 +1044,7 @@ static void __exit pf_exit(void) · 62 unmodified lines
10441044 if (!pf->present)
10451045 continue;
10461046 del_gendisk(pf->disk);
1047− blk_cleanup_disk(pf->disk);
1047+ put_disk(pf->disk);
10481048 blk_mq_free_tag_set(&pf->tag_set);
10491049 pi_release(pf->pi);
10501050 }
drivers/block/pktcdvd.c+5−9
@@ -2460,11 +2460,9 @@ static int pkt_seq_show(struct seq_file *m, void *p) · 2459 unmodified lines
24602460 {
24612461 struct pktcdvd_device *pd = m->private;
24622462 char *msg;
2463− char bdev_buf[BDEVNAME_SIZE];
24642463 int states[PACKET_NUM_STATES];
24652464
2466− seq_printf(m, "Writer %s mapped to %s:\n", pd->name,
2467− bdevname(pd->bdev, bdev_buf));
2465+ seq_printf(m, "Writer %s mapped to %pg:\n", pd->name, pd->bdev);
24682466
24692467 seq_printf(m, "\nSettings:\n");
24702468 seq_printf(m, "\tpacket size:\t\t%dkB\n", pd->settings.size / 2);
@@ -2521,7 +2519,6 @@ static int pkt_seq_show(struct seq_file *m, void *p) · 50 unmodified lines
25212519 static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev)
25222520 {
25232521 int i;
2524− char b[BDEVNAME_SIZE];
25252522 struct block_device *bdev;
25262523 struct scsi_device *sdev;
25272524
@@ -2534,8 +2531,7 @@ static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev) · 6 unmodified lines
25342531 if (!pd2)
25352532 continue;
25362533 if (pd2->bdev->bd_dev == dev) {
2537− pkt_err(pd, "%s already setup\n",
2538− bdevname(pd2->bdev, b));
2534+ pkt_err(pd, "%pg already setup\n", pd2->bdev);
25392535 return -EBUSY;
25402536 }
25412537 if (pd2->pkt_dev == dev) {
@@ -2570,7 +2566,7 @@ static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev) · 28 unmodified lines
25702566 }
25712567
25722568 proc_create_single_data(pd->name, 0, pkt_proc, pkt_seq_show, pd);
2573− pkt_dbg(1, pd, "writer mapped to %s\n", bdevname(bdev, b));
2569+ pkt_dbg(1, pd, "writer mapped to %pg\n", bdev);
25742570 return 0;
25752571
25762572 out_mem:
@@ -2733,7 +2729,7 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev) · 156 unmodified lines
27332729 return 0;
27342730
27352731 out_mem2:
2736− blk_cleanup_disk(disk);
2732+ put_disk(disk);
27372733 out_mem:
27382734 mempool_exit(&pd->rb_pool);
27392735 kfree(pd);
@@ -2783,7 +2779,7 @@ static int pkt_remove_dev(dev_t pkt_dev) · 43 unmodified lines
27832779 pkt_dbg(1, pd, "writer unmapped\n");
27842780
27852781 del_gendisk(pd->disk);
2786− blk_cleanup_disk(pd->disk);
2782+ put_disk(pd->disk);
27872783
27882784 mempool_exit(&pd->rb_pool);
27892785 kfree(pd);
drivers/block/ps3disk.c+2−2
@@ -473,7 +473,7 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev) · 472 unmodified lines
473473
474474 return 0;
475475 fail_cleanup_disk:
476− blk_cleanup_disk(gendisk);
476+ put_disk(gendisk);
477477 fail_free_tag_set:
478478 blk_mq_free_tag_set(&priv->tag_set);
479479 fail_teardown:
@@ -500,7 +500,7 @@ static void ps3disk_remove(struct ps3_system_bus_device *_dev) · 20 unmodified lines
500500 &ps3disk_mask);
501501 mutex_unlock(&ps3disk_mask_mutex);
502502 del_gendisk(priv->gendisk);
503− blk_cleanup_disk(priv->gendisk);
503+ put_disk(priv->gendisk);
504504 blk_mq_free_tag_set(&priv->tag_set);
505505 dev_notice(&dev->sbd.core, "Synchronizing disk cache\n");
506506 ps3disk_sync_cache(dev);
drivers/block/ps3vram.c+2−2
@@ -761,7 +761,7 @@ static int ps3vram_probe(struct ps3_system_bus_device *dev) · 760 unmodified lines
761761 return 0;
762762
763763 out_cleanup_disk:
764− blk_cleanup_disk(gendisk);
764+ put_disk(gendisk);
765765 out_cache_cleanup:
766766 remove_proc_entry(DEVICE_NAME, NULL);
767767 ps3vram_cache_cleanup(dev);
@@ -792,7 +792,7 @@ static void ps3vram_remove(struct ps3_system_bus_device *dev) · 24 unmodified lines
792792 struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev);
793793
794794 del_gendisk(priv->gendisk);
795− blk_cleanup_disk(priv->gendisk);
795+ put_disk(priv->gendisk);
796796 remove_proc_entry(DEVICE_NAME, NULL);
797797 ps3vram_cache_cleanup(dev);
798798 iounmap(priv->reports);
drivers/block/rbd.c+1−1
@@ -4729,7 +4729,7 @@ static blk_status_t rbd_queue_rq(struct blk_mq_hw_ctx *hctx, · 4728 unmodified lines
47294729
47304730 static void rbd_free_disk(struct rbd_device *rbd_dev)
47314731 {
4732− blk_cleanup_disk(rbd_dev->disk);
4732+ put_disk(rbd_dev->disk);
47334733 blk_mq_free_tag_set(&rbd_dev->tag_set);
47344734 rbd_dev->disk = NULL;
47354735 }
drivers/block/rnbd/rnbd-clt.c+3−3
@@ -1408,7 +1408,7 @@ static int rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx) · 1407 unmodified lines
14081408 blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue);
14091409 err = add_disk(dev->gd);
14101410 if (err)
1411− blk_cleanup_disk(dev->gd);
1411+ put_disk(dev->gd);
14121412
14131413 return err;
14141414 }
@@ -1630,7 +1630,7 @@ put_sess: · 215 unmodified lines
16301630 static void destroy_gen_disk(struct rnbd_clt_dev *dev)
16311631 {
16321632 del_gendisk(dev->gd);
1633− blk_cleanup_disk(dev->gd);
1633+ put_disk(dev->gd);
16341634 }
16351635
16361636 static void destroy_sysfs(struct rnbd_clt_dev *dev,
@@ -1755,7 +1755,7 @@ static void rnbd_destroy_sessions(void) · 118 unmodified lines
17551755 list_for_each_entry_safe(dev, tn, &sess->devs_list, list) {
17561756 /*
17571757 * Here unmap happens in parallel for only one reason:
1758− * blk_cleanup_queue() takes around half a second, so
1758+ * del_gendisk() takes around half a second, so
17591759 * on huge amount of devices the whole module unload
17601760 * procedure takes minutes.
17611761 */
drivers/block/rnbd/rnbd-proto.h+4−3
@@ -229,9 +229,9 @@ static inline bool rnbd_flags_supported(u32 flags) · 228 unmodified lines
229229 return true;
230230 }
231231
232−static inline u32 rnbd_to_bio_flags(u32 rnbd_opf)
232+static inline blk_opf_t rnbd_to_bio_flags(u32 rnbd_opf)
233233 {
234− u32 bio_opf;
234+ blk_opf_t bio_opf;
235235
236236 switch (rnbd_op(rnbd_opf)) {
237237 case RNBD_OP_READ:
@@ -286,7 +286,8 @@ static inline u32 rq_to_rnbd_flags(struct request *rq) · 48 unmodified lines
286286 break;
287287 default:
288288 WARN(1, "Unknown request type %d (flags %llu)\n",
289− req_op(rq), (unsigned long long)rq->cmd_flags);
289+ (__force u32)req_op(rq),
290+ (__force unsigned long long)rq->cmd_flags);
290291 rnbd_opf = 0;
291292 }
292293
drivers/block/rnbd/rnbd-srv-dev.c+0−1
@@ -28,7 +28,6 @@ struct rnbd_dev *rnbd_dev_open(const char *path, fmode_t flags) · 27 unmodified lines
2828 goto err;
2929
3030 dev->blk_open_flags = flags;
31− bdevname(dev->bdev, dev->name);
3231
3332 return dev;
3433
drivers/block/rnbd/rnbd-srv-dev.h+0−1
@@ -15,7 +15,6 @@ · 14 unmodified lines
1515 struct rnbd_dev {
1616 struct block_device *bdev;
1717 fmode_t blk_open_flags;
18− char name[BDEVNAME_SIZE];
1918 };
2019
2120 /**
drivers/block/rnbd/rnbd-srv-sysfs.c+2−3
@@ -38,14 +38,13 @@ static struct kobj_type dev_ktype = { · 37 unmodified lines
3838 };
3939
4040 int rnbd_srv_create_dev_sysfs(struct rnbd_srv_dev *dev,
41− struct block_device *bdev,
42− const char *dev_name)
41+ struct block_device *bdev)
4342 {
4443 struct kobject *bdev_kobj;
4544 int ret;
4645
4746 ret = kobject_init_and_add(&dev->dev_kobj, &dev_ktype,
48− rnbd_devs_kobj, dev_name);
47+ rnbd_devs_kobj, "%pg", bdev);
4948 if (ret) {
5049 kobject_put(&dev->dev_kobj);
5150 return ret;
drivers/block/rnbd/rnbd-srv.c+4−5
@@ -419,7 +419,7 @@ static struct rnbd_srv_sess_dev · 418 unmodified lines
419419 return sess_dev;
420420 }
421421
422−static struct rnbd_srv_dev *rnbd_srv_init_srv_dev(const char *id)
422+static struct rnbd_srv_dev *rnbd_srv_init_srv_dev(struct block_device *bdev)
423423 {
424424 struct rnbd_srv_dev *dev;
425425
@@ -427,7 +427,7 @@ static struct rnbd_srv_dev *rnbd_srv_init_srv_dev(const char *id) · 1 unmodified line
427427 if (!dev)
428428 return ERR_PTR(-ENOMEM);
429429
430− strscpy(dev->id, id, sizeof(dev->id));
430+ snprintf(dev->id, sizeof(dev->id), "%pg", bdev);
431431 kref_init(&dev->kref);
432432 INIT_LIST_HEAD(&dev->sess_dev_list);
433433 mutex_init(&dev->lock);
@@ -512,7 +512,7 @@ rnbd_srv_get_or_create_srv_dev(struct rnbd_dev *rnbd_dev, · 78 unmodified lines
512512 int ret;
513513 struct rnbd_srv_dev *new_dev, *dev;
514514
515− new_dev = rnbd_srv_init_srv_dev(rnbd_dev->name);
515+ new_dev = rnbd_srv_init_srv_dev(rnbd_dev->bdev);
516516 if (IS_ERR(new_dev))
517517 return new_dev;
518518
@@ -758,8 +758,7 @@ static int process_msg_open(struct rnbd_srv_session *srv_sess, · 239 unmodified lines
758758 */
759759 mutex_lock(&srv_dev->lock);
760760 if (!srv_dev->dev_kobj.state_in_sysfs) {
761− ret = rnbd_srv_create_dev_sysfs(srv_dev, rnbd_dev->bdev,
762− rnbd_dev->name);
761+ ret = rnbd_srv_create_dev_sysfs(srv_dev, rnbd_dev->bdev);
763762 if (ret) {
764763 mutex_unlock(&srv_dev->lock);
765764 rnbd_srv_err(srv_sess_dev,
drivers/block/rnbd/rnbd-srv.h+1−2
@@ -68,8 +68,7 @@ void rnbd_srv_sess_dev_force_close(struct rnbd_srv_sess_dev *sess_dev, · 67 unmodified lines
6868 /* rnbd-srv-sysfs.c */
6969
7070 int rnbd_srv_create_dev_sysfs(struct rnbd_srv_dev *dev,
71− struct block_device *bdev,
72− const char *dir_name);
71+ struct block_device *bdev);
7372 void rnbd_srv_destroy_dev_sysfs(struct rnbd_srv_dev *dev);
7473 int rnbd_srv_create_dev_session_sysfs(struct rnbd_srv_sess_dev *sess_dev);
7574 void rnbd_srv_destroy_dev_session_sysfs(struct rnbd_srv_sess_dev *sess_dev);
drivers/block/sunvdc.c+2−2
@@ -886,7 +886,7 @@ static int probe_disk(struct vdc_port *port) · 885 unmodified lines
886886 return 0;
887887
888888 out_cleanup_disk:
889− blk_cleanup_disk(g);
889+ put_disk(g);
890890 out_free_tag:
891891 blk_mq_free_tag_set(&port->tag_set);
892892 return err;
@@ -1070,7 +1070,7 @@ static void vdc_port_remove(struct vio_dev *vdev) · 177 unmodified lines
10701070 del_timer_sync(&port->vio.timer);
10711071
10721072 del_gendisk(port->disk);
1073− blk_cleanup_disk(port->disk);
1073+ put_disk(port->disk);
10741074 blk_mq_free_tag_set(&port->tag_set);
10751075
10761076 vdc_free_tx_ring(port);
drivers/block/swim.c+1−1
@@ -783,7 +783,7 @@ static void swim_cleanup_floppy_disk(struct floppy_state *fs) · 782 unmodified lines
783783 if (fs->registered)
784784 del_gendisk(fs->disk);
785785
786− blk_cleanup_disk(disk);
786+ put_disk(disk);
787787 blk_mq_free_tag_set(&fs->tag_set);
788788 }
789789
drivers/block/swim3.c+1−1
@@ -1238,7 +1238,7 @@ static int swim3_attach(struct macio_dev *mdev, · 1237 unmodified lines
12381238 return 0;
12391239
12401240 out_cleanup_disk:
1241− blk_cleanup_disk(disk);
1241+ put_disk(disk);
12421242 out_free_tag_set:
12431243 blk_mq_free_tag_set(&fs->tag_set);
12441244 out_unregister:
drivers/block/sx8.c+3−3
@@ -1377,7 +1377,7 @@ static void carm_free_disk(struct carm_host *host, unsigned int port_no) · 1376 unmodified lines
13771377
13781378 if (host->state > HST_DEV_ACTIVATE)
13791379 del_gendisk(disk);
1380− blk_cleanup_disk(disk);
1380+ put_disk(disk);
13811381 }
13821382
13831383 static int carm_init_shm(struct carm_host *host)
@@ -1536,7 +1536,7 @@ err_out_free_majors: · 152 unmodified lines
15361536 clear_bit(0, &carm_major_alloc);
15371537 else if (host->major == 161)
15381538 clear_bit(1, &carm_major_alloc);
1539− blk_cleanup_queue(host->oob_q);
1539+ blk_mq_destroy_queue(host->oob_q);
15401540 blk_mq_free_tag_set(&host->tag_set);
15411541 err_out_dma_free:
15421542 dma_free_coherent(&pdev->dev, CARM_SHM_SIZE, host->shm, host->shm_dma);
@@ -1570,7 +1570,7 @@ static void carm_remove_one (struct pci_dev *pdev) · 27 unmodified lines
15701570 clear_bit(0, &carm_major_alloc);
15711571 else if (host->major == 161)
15721572 clear_bit(1, &carm_major_alloc);
1573− blk_cleanup_queue(host->oob_q);
1573+ blk_mq_destroy_queue(host->oob_q);
15741574 blk_mq_free_tag_set(&host->tag_set);
15751575 dma_free_coherent(&pdev->dev, CARM_SHM_SIZE, host->shm, host->shm_dma);
15761576 iounmap(host->mmio);
drivers/block/ublk_drv.c+1545−0
@@ -0,0 +1,1545 @@
1+// SPDX-License-Identifier: GPL-2.0-or-later
2+/*
3+ * Userspace block device - block device which IO is handled from userspace
4+ *
5+ * Take full use of io_uring passthrough command for communicating with
6+ * ublk userspace daemon(ublksrvd) for handling basic IO request.
7+ *
8+ * Copyright 2022 Ming Lei <ming.lei@redhat.com>
9+ *
10+ * (part of code stolen from loop.c)
11+ */
12+#include <linux/module.h>
13+#include <linux/moduleparam.h>
14+#include <linux/sched.h>
15+#include <linux/fs.h>
16+#include <linux/pagemap.h>
17+#include <linux/file.h>
18+#include <linux/stat.h>
19+#include <linux/errno.h>
20+#include <linux/major.h>
21+#include <linux/wait.h>
22+#include <linux/blkdev.h>
23+#include <linux/init.h>
24+#include <linux/swap.h>
25+#include <linux/slab.h>
26+#include <linux/compat.h>
27+#include <linux/mutex.h>
28+#include <linux/writeback.h>
29+#include <linux/completion.h>
30+#include <linux/highmem.h>
31+#include <linux/sysfs.h>
32+#include <linux/miscdevice.h>
33+#include <linux/falloc.h>
34+#include <linux/uio.h>
35+#include <linux/ioprio.h>
36+#include <linux/sched/mm.h>
37+#include <linux/uaccess.h>
38+#include <linux/cdev.h>
39+#include <linux/io_uring.h>
40+#include <linux/blk-mq.h>
41+#include <linux/delay.h>
42+#include <linux/mm.h>
43+#include <asm/page.h>
44+#include <linux/task_work.h>
45+#include <uapi/linux/ublk_cmd.h>
46+
47+#define UBLK_MINORS (1U << MINORBITS)
48+
49+/* All UBLK_F_* have to be included into UBLK_F_ALL */
50+#define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_URING_CMD_COMP_IN_TASK)
51+
52+struct ublk_rq_data {
53+ struct callback_head work;
54+};
55+
56+struct ublk_uring_cmd_pdu {
57+ struct request *req;
58+};
59+
60+/*
61+ * io command is active: sqe cmd is received, and its cqe isn't done
62+ *
63+ * If the flag is set, the io command is owned by ublk driver, and waited
64+ * for incoming blk-mq request from the ublk block device.
65+ *
66+ * If the flag is cleared, the io command will be completed, and owned by
67+ * ublk server.
68+ */
69+#define UBLK_IO_FLAG_ACTIVE 0x01
70+
71+/*
72+ * IO command is completed via cqe, and it is being handled by ublksrv, and
73+ * not committed yet
74+ *
75+ * Basically exclusively with UBLK_IO_FLAG_ACTIVE, so can be served for
76+ * cross verification
77+ */
78+#define UBLK_IO_FLAG_OWNED_BY_SRV 0x02
79+
80+/*
81+ * IO command is aborted, so this flag is set in case of
82+ * !UBLK_IO_FLAG_ACTIVE.
83+ *
84+ * After this flag is observed, any pending or new incoming request
85+ * associated with this io command will be failed immediately
86+ */
87+#define UBLK_IO_FLAG_ABORTED 0x04
88+
89+struct ublk_io {
90+ /* userspace buffer address from io cmd */
91+ __u64 addr;
92+ unsigned int flags;
93+ int res;
94+
95+ struct io_uring_cmd *cmd;
96+};
97+
98+struct ublk_queue {
99+ int q_id;
100+ int q_depth;
101+
102+ unsigned long flags;
103+ struct task_struct *ubq_daemon;
104+ char *io_cmd_buf;
105+
106+ unsigned long io_addr; /* mapped vm address */
107+ unsigned int max_io_sz;
108+ bool abort_work_pending;
109+ unsigned short nr_io_ready; /* how many ios setup */
110+ struct ublk_device *dev;
111+ struct ublk_io ios[0];
112+};
113+
114+#define UBLK_DAEMON_MONITOR_PERIOD (5 * HZ)
115+
116+struct ublk_device {
117+ struct gendisk *ub_disk;
118+
119+ char *__queues;
120+
121+ unsigned short queue_size;
122+ unsigned short bs_shift;
123+ struct ublksrv_ctrl_dev_info dev_info;
124+
125+ struct blk_mq_tag_set tag_set;
126+
127+ struct cdev cdev;
128+ struct device cdev_dev;
129+
130+#define UB_STATE_OPEN 0
131+#define UB_STATE_USED 1
132+ unsigned long state;
133+ int ub_number;
134+
135+ struct mutex mutex;
136+
137+ spinlock_t mm_lock;
138+ struct mm_struct *mm;
139+
140+ struct completion completion;
141+ unsigned int nr_queues_ready;
142+ atomic_t nr_aborted_queues;
143+
144+ /*
145+ * Our ubq->daemon may be killed without any notification, so
146+ * monitor each queue's daemon periodically
147+ */
148+ struct delayed_work monitor_work;
149+ struct work_struct stop_work;
150+};
151+
152+static dev_t ublk_chr_devt;
153+static struct class *ublk_chr_class;
154+
155+static DEFINE_IDR(ublk_index_idr);
156+static DEFINE_SPINLOCK(ublk_idr_lock);
157+static wait_queue_head_t ublk_idr_wq; /* wait until one idr is freed */
158+
159+static DEFINE_MUTEX(ublk_ctl_mutex);
160+
161+static struct miscdevice ublk_misc;
162+
163+static inline bool ublk_can_use_task_work(const struct ublk_queue *ubq)
164+{
165+ if (IS_BUILTIN(CONFIG_BLK_DEV_UBLK) &&
166+ !(ubq->flags & UBLK_F_URING_CMD_COMP_IN_TASK))
167+ return true;
168+ return false;
169+}
170+
171+static struct ublk_device *ublk_get_device(struct ublk_device *ub)
172+{
173+ if (kobject_get_unless_zero(&ub->cdev_dev.kobj))
174+ return ub;
175+ return NULL;
176+}
177+
178+static void ublk_put_device(struct ublk_device *ub)
179+{
180+ put_device(&ub->cdev_dev);
181+}
182+
183+static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev,
184+ int qid)
185+{
186+ return (struct ublk_queue *)&(dev->__queues[qid * dev->queue_size]);
187+}
188+
189+static inline bool ublk_rq_has_data(const struct request *rq)
190+{
191+ return rq->bio && bio_has_data(rq->bio);
192+}
193+
194+static inline struct ublksrv_io_desc *ublk_get_iod(struct ublk_queue *ubq,
195+ int tag)
196+{
197+ return (struct ublksrv_io_desc *)
198+ &(ubq->io_cmd_buf[tag * sizeof(struct ublksrv_io_desc)]);
199+}
200+
201+static inline char *ublk_queue_cmd_buf(struct ublk_device *ub, int q_id)
202+{
203+ return ublk_get_queue(ub, q_id)->io_cmd_buf;
204+}
205+
206+static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub, int q_id)
207+{
208+ struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
209+
210+ return round_up(ubq->q_depth * sizeof(struct ublksrv_io_desc),
211+ PAGE_SIZE);
212+}
213+
214+static void ublk_free_disk(struct gendisk *disk)
215+{
216+ struct ublk_device *ub = disk->private_data;
217+
218+ clear_bit(UB_STATE_USED, &ub->state);
219+ put_device(&ub->cdev_dev);
220+}
221+
222+static const struct block_device_operations ub_fops = {
223+ .owner = THIS_MODULE,
224+ .free_disk = ublk_free_disk,
225+};
226+
227+#define UBLK_MAX_PIN_PAGES 32
228+
229+struct ublk_map_data {
230+ const struct ublk_queue *ubq;
231+ const struct request *rq;
232+ const struct ublk_io *io;
233+ unsigned max_bytes;
234+};
235+
236+struct ublk_io_iter {
237+ struct page *pages[UBLK_MAX_PIN_PAGES];
238+ unsigned pg_off; /* offset in the 1st page in pages */
239+ int nr_pages; /* how many page pointers in pages */
240+ struct bio *bio;
241+ struct bvec_iter iter;
242+};
243+
244+static inline unsigned ublk_copy_io_pages(struct ublk_io_iter *data,
245+ unsigned max_bytes, bool to_vm)
246+{
247+ const unsigned total = min_t(unsigned, max_bytes,
248+ PAGE_SIZE - data->pg_off +
249+ ((data->nr_pages - 1) << PAGE_SHIFT));
250+ unsigned done = 0;
251+ unsigned pg_idx = 0;
252+
253+ while (done < total) {
254+ struct bio_vec bv = bio_iter_iovec(data->bio, data->iter);
255+ const unsigned int bytes = min3(bv.bv_len, total - done,
256+ (unsigned)(PAGE_SIZE - data->pg_off));
257+ void *bv_buf = bvec_kmap_local(&bv);
258+ void *pg_buf = kmap_local_page(data->pages[pg_idx]);
259+
260+ if (to_vm)
261+ memcpy(pg_buf + data->pg_off, bv_buf, bytes);
262+ else
263+ memcpy(bv_buf, pg_buf + data->pg_off, bytes);
264+
265+ kunmap_local(pg_buf);
266+ kunmap_local(bv_buf);
267+
268+ /* advance page array */
269+ data->pg_off += bytes;
270+ if (data->pg_off == PAGE_SIZE) {
271+ pg_idx += 1;
272+ data->pg_off = 0;
273+ }
274+
275+ done += bytes;
276+
277+ /* advance bio */
278+ bio_advance_iter_single(data->bio, &data->iter, bytes);
279+ if (!data->iter.bi_size) {
280+ data->bio = data->bio->bi_next;
281+ if (data->bio == NULL)
282+ break;
283+ data->iter = data->bio->bi_iter;
284+ }
285+ }
286+
287+ return done;
288+}
289+
290+static inline int ublk_copy_user_pages(struct ublk_map_data *data,
291+ bool to_vm)
292+{
293+ const unsigned int gup_flags = to_vm ? FOLL_WRITE : 0;
294+ const unsigned long start_vm = data->io->addr;
295+ unsigned int done = 0;
296+ struct ublk_io_iter iter = {
297+ .pg_off = start_vm & (PAGE_SIZE - 1),
298+ .bio = data->rq->bio,
299+ .iter = data->rq->bio->bi_iter,
300+ };
301+ const unsigned int nr_pages = round_up(data->max_bytes +
302+ (start_vm & (PAGE_SIZE - 1)), PAGE_SIZE) >> PAGE_SHIFT;
303+
304+ while (done < nr_pages) {
305+ const unsigned to_pin = min_t(unsigned, UBLK_MAX_PIN_PAGES,
306+ nr_pages - done);
307+ unsigned i, len;
308+
309+ iter.nr_pages = get_user_pages_fast(start_vm +
310+ (done << PAGE_SHIFT), to_pin, gup_flags,
311+ iter.pages);
312+ if (iter.nr_pages <= 0)
313+ return done == 0 ? iter.nr_pages : done;
314+ len = ublk_copy_io_pages(&iter, data->max_bytes, to_vm);
315+ for (i = 0; i < iter.nr_pages; i++) {
316+ if (to_vm)
317+ set_page_dirty(iter.pages[i]);
318+ put_page(iter.pages[i]);
319+ }
320+ data->max_bytes -= len;
321+ done += iter.nr_pages;
322+ }
323+
324+ return done;
325+}
326+
327+static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req,
328+ struct ublk_io *io)
329+{
330+ const unsigned int rq_bytes = blk_rq_bytes(req);
331+ /*
332+ * no zero copy, we delay copy WRITE request data into ublksrv
333+ * context and the big benefit is that pinning pages in current
334+ * context is pretty fast, see ublk_pin_user_pages
335+ */
336+ if (req_op(req) != REQ_OP_WRITE && req_op(req) != REQ_OP_FLUSH)
337+ return rq_bytes;
338+
339+ if (ublk_rq_has_data(req)) {
340+ struct ublk_map_data data = {
341+ .ubq = ubq,
342+ .rq = req,
343+ .io = io,
344+ .max_bytes = rq_bytes,
345+ };
346+
347+ ublk_copy_user_pages(&data, true);
348+
349+ return rq_bytes - data.max_bytes;
350+ }
351+ return rq_bytes;
352+}
353+
354+static int ublk_unmap_io(const struct ublk_queue *ubq,
355+ const struct request *req,
356+ struct ublk_io *io)
357+{
358+ const unsigned int rq_bytes = blk_rq_bytes(req);
359+
360+ if (req_op(req) == REQ_OP_READ && ublk_rq_has_data(req)) {
361+ struct ublk_map_data data = {
362+ .ubq = ubq,
363+ .rq = req,
364+ .io = io,
365+ .max_bytes = io->res,
366+ };
367+
368+ WARN_ON_ONCE(io->res > rq_bytes);
369+
370+ ublk_copy_user_pages(&data, false);
371+
372+ return io->res - data.max_bytes;
373+ }
374+ return rq_bytes;
375+}
376+
377+static inline unsigned int ublk_req_build_flags(struct request *req)
378+{
379+ unsigned flags = 0;
380+
381+ if (req->cmd_flags & REQ_FAILFAST_DEV)
382+ flags |= UBLK_IO_F_FAILFAST_DEV;
383+
384+ if (req->cmd_flags & REQ_FAILFAST_TRANSPORT)
385+ flags |= UBLK_IO_F_FAILFAST_TRANSPORT;
386+
387+ if (req->cmd_flags & REQ_FAILFAST_DRIVER)
388+ flags |= UBLK_IO_F_FAILFAST_DRIVER;
389+
390+ if (req->cmd_flags & REQ_META)
391+ flags |= UBLK_IO_F_META;
392+
393+ if (req->cmd_flags & REQ_FUA)
394+ flags |= UBLK_IO_F_FUA;
395+
396+ if (req->cmd_flags & REQ_NOUNMAP)
397+ flags |= UBLK_IO_F_NOUNMAP;
398+
399+ if (req->cmd_flags & REQ_SWAP)
400+ flags |= UBLK_IO_F_SWAP;
401+
402+ return flags;
403+}
404+
405+static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req)
406+{
407+ struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag);
408+ struct ublk_io *io = &ubq->ios[req->tag];
409+ u32 ublk_op;
410+
411+ switch (req_op(req)) {
412+ case REQ_OP_READ:
413+ ublk_op = UBLK_IO_OP_READ;
414+ break;
415+ case REQ_OP_WRITE:
416+ ublk_op = UBLK_IO_OP_WRITE;
417+ break;
418+ case REQ_OP_FLUSH:
419+ ublk_op = UBLK_IO_OP_FLUSH;
420+ break;
421+ case REQ_OP_DISCARD:
422+ ublk_op = UBLK_IO_OP_DISCARD;
423+ break;
424+ case REQ_OP_WRITE_ZEROES:
425+ ublk_op = UBLK_IO_OP_WRITE_ZEROES;
426+ break;
427+ default:
428+ return BLK_STS_IOERR;
429+ }
430+
431+ /* need to translate since kernel may change */
432+ iod->op_flags = ublk_op | ublk_req_build_flags(req);
433+ iod->nr_sectors = blk_rq_sectors(req);
434+ iod->start_sector = blk_rq_pos(req);
435+ iod->addr = io->addr;
436+
437+ return BLK_STS_OK;
438+}
439+
440+static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu(
441+ struct io_uring_cmd *ioucmd)
442+{
443+ return (struct ublk_uring_cmd_pdu *)&ioucmd->pdu;
444+}
445+
446+static bool ubq_daemon_is_dying(struct ublk_queue *ubq)
447+{
448+ return ubq->ubq_daemon->flags & PF_EXITING;
449+}
450+
451+/* todo: handle partial completion */
452+static void ublk_complete_rq(struct request *req)
453+{
454+ struct ublk_queue *ubq = req->mq_hctx->driver_data;
455+ struct ublk_io *io = &ubq->ios[req->tag];
456+ unsigned int unmapped_bytes;
457+
458+ /* failed read IO if nothing is read */
459+ if (!io->res && req_op(req) == REQ_OP_READ)
460+ io->res = -EIO;
461+
462+ if (io->res < 0) {
463+ blk_mq_end_request(req, errno_to_blk_status(io->res));
464+ return;
465+ }
466+
467+ /*
468+ * FLUSH or DISCARD usually won't return bytes returned, so end them
469+ * directly.
470+ *
471+ * Both the two needn't unmap.
472+ */
473+ if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE) {
474+ blk_mq_end_request(req, BLK_STS_OK);
475+ return;
476+ }
477+
478+ /* for READ request, writing data in iod->addr to rq buffers */
479+ unmapped_bytes = ublk_unmap_io(ubq, req, io);
480+
481+ /*
482+ * Extremely impossible since we got data filled in just before
483+ *
484+ * Re-read simply for this unlikely case.
485+ */
486+ if (unlikely(unmapped_bytes < io->res))
487+ io->res = unmapped_bytes;
488+
489+ if (blk_update_request(req, BLK_STS_OK, io->res))
490+ blk_mq_requeue_request(req, true);
491+ else
492+ __blk_mq_end_request(req, BLK_STS_OK);
493+}
494+
495+/*
496+ * __ublk_fail_req() may be called from abort context or ->ubq_daemon
497+ * context during exiting, so lock is required.
498+ *
499+ * Also aborting may not be started yet, keep in mind that one failed
500+ * request may be issued by block layer again.
501+ */
502+static void __ublk_fail_req(struct ublk_io *io, struct request *req)
503+{
504+ WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_ACTIVE);
505+
506+ if (!(io->flags & UBLK_IO_FLAG_ABORTED)) {
507+ io->flags |= UBLK_IO_FLAG_ABORTED;
508+ blk_mq_end_request(req, BLK_STS_IOERR);
509+ }
510+}
511+
512+#define UBLK_REQUEUE_DELAY_MS 3
513+
514+static inline void __ublk_rq_task_work(struct request *req)
515+{
516+ struct ublk_queue *ubq = req->mq_hctx->driver_data;
517+ struct ublk_device *ub = ubq->dev;
518+ int tag = req->tag;
519+ struct ublk_io *io = &ubq->ios[tag];
520+ bool task_exiting = current != ubq->ubq_daemon ||
521+ (current->flags & PF_EXITING);
522+ unsigned int mapped_bytes;
523+
524+ pr_devel("%s: complete: op %d, qid %d tag %d io_flags %x addr %llx\n",
525+ __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags,
526+ ublk_get_iod(ubq, req->tag)->addr);
527+
528+ if (unlikely(task_exiting)) {
529+ blk_mq_end_request(req, BLK_STS_IOERR);
530+ mod_delayed_work(system_wq, &ub->monitor_work, 0);
531+ return;
532+ }
533+
534+ mapped_bytes = ublk_map_io(ubq, req, io);
535+
536+ /* partially mapped, update io descriptor */
537+ if (unlikely(mapped_bytes != blk_rq_bytes(req))) {
538+ /*
539+ * Nothing mapped, retry until we succeed.
540+ *
541+ * We may never succeed in mapping any bytes here because
542+ * of OOM. TODO: reserve one buffer with single page pinned
543+ * for providing forward progress guarantee.
544+ */
545+ if (unlikely(!mapped_bytes)) {
546+ blk_mq_requeue_request(req, false);
547+ blk_mq_delay_kick_requeue_list(req->q,
548+ UBLK_REQUEUE_DELAY_MS);
549+ return;
550+ }
551+
552+ ublk_get_iod(ubq, req->tag)->nr_sectors =
553+ mapped_bytes >> 9;
554+ }
555+
556+ /* mark this cmd owned by ublksrv */
557+ io->flags |= UBLK_IO_FLAG_OWNED_BY_SRV;
558+
559+ /*
560+ * clear ACTIVE since we are done with this sqe/cmd slot
561+ * We can only accept io cmd in case of being not active.
562+ */
563+ io->flags &= ~UBLK_IO_FLAG_ACTIVE;
564+
565+ /* tell ublksrv one io request is coming */
566+ io_uring_cmd_done(io->cmd, UBLK_IO_RES_OK, 0);
567+}
568+
569+static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd)
570+{
571+ struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
572+
573+ __ublk_rq_task_work(pdu->req);
574+}
575+
576+static void ublk_rq_task_work_fn(struct callback_head *work)
577+{
578+ struct ublk_rq_data *data = container_of(work,
579+ struct ublk_rq_data, work);
580+ struct request *req = blk_mq_rq_from_pdu(data);
581+
582+ __ublk_rq_task_work(req);
583+}
584+
585+static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
586+ const struct blk_mq_queue_data *bd)
587+{
588+ struct ublk_queue *ubq = hctx->driver_data;
589+ struct request *rq = bd->rq;
590+ blk_status_t res;
591+
592+ /* fill iod to slot in io cmd buffer */
593+ res = ublk_setup_iod(ubq, rq);
594+ if (unlikely(res != BLK_STS_OK))
595+ return BLK_STS_IOERR;
596+
597+ blk_mq_start_request(bd->rq);
598+
599+ if (unlikely(ubq_daemon_is_dying(ubq))) {
600+ fail:
601+ mod_delayed_work(system_wq, &ubq->dev->monitor_work, 0);
602+ return BLK_STS_IOERR;
603+ }
604+
605+ if (ublk_can_use_task_work(ubq)) {
606+ struct ublk_rq_data *data = blk_mq_rq_to_pdu(rq);
607+ enum task_work_notify_mode notify_mode = bd->last ?
608+ TWA_SIGNAL_NO_IPI : TWA_NONE;
609+
610+ if (task_work_add(ubq->ubq_daemon, &data->work, notify_mode))
611+ goto fail;
612+ } else {
613+ struct io_uring_cmd *cmd = ubq->ios[rq->tag].cmd;
614+ struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
615+
616+ pdu->req = rq;
617+ io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb);
618+ }
619+
620+ return BLK_STS_OK;
621+}
622+
623+static void ublk_commit_rqs(struct blk_mq_hw_ctx *hctx)
624+{
625+ struct ublk_queue *ubq = hctx->driver_data;
626+
627+ if (ublk_can_use_task_work(ubq))
628+ __set_notify_signal(ubq->ubq_daemon);
629+}
630+
631+static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
632+ unsigned int hctx_idx)
633+{
634+ struct ublk_device *ub = driver_data;
635+ struct ublk_queue *ubq = ublk_get_queue(ub, hctx->queue_num);
636+
637+ hctx->driver_data = ubq;
638+ return 0;
639+}
640+
641+static int ublk_init_rq(struct blk_mq_tag_set *set, struct request *req,
642+ unsigned int hctx_idx, unsigned int numa_node)
643+{
644+ struct ublk_rq_data *data = blk_mq_rq_to_pdu(req);
645+
646+ init_task_work(&data->work, ublk_rq_task_work_fn);
647+ return 0;
648+}
649+
650+static const struct blk_mq_ops ublk_mq_ops = {
651+ .queue_rq = ublk_queue_rq,
652+ .commit_rqs = ublk_commit_rqs,
653+ .init_hctx = ublk_init_hctx,
654+ .init_request = ublk_init_rq,
655+};
656+
657+static int ublk_ch_open(struct inode *inode, struct file *filp)
658+{
659+ struct ublk_device *ub = container_of(inode->i_cdev,
660+ struct ublk_device, cdev);
661+
662+ if (test_and_set_bit(UB_STATE_OPEN, &ub->state))
663+ return -EBUSY;
664+ filp->private_data = ub;
665+ return 0;
666+}
667+
668+static int ublk_ch_release(struct inode *inode, struct file *filp)
669+{
670+ struct ublk_device *ub = filp->private_data;
671+
672+ clear_bit(UB_STATE_OPEN, &ub->state);
673+ return 0;
674+}
675+
676+/* map pre-allocated per-queue cmd buffer to ublksrv daemon */
677+static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma)
678+{
679+ struct ublk_device *ub = filp->private_data;
680+ size_t sz = vma->vm_end - vma->vm_start;
681+ unsigned max_sz = UBLK_MAX_QUEUE_DEPTH * sizeof(struct ublksrv_io_desc);
682+ unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT;
683+ int q_id, ret = 0;
684+
685+ spin_lock(&ub->mm_lock);
686+ if (!ub->mm)
687+ ub->mm = current->mm;
688+ if (current->mm != ub->mm)
689+ ret = -EINVAL;
690+ spin_unlock(&ub->mm_lock);
691+
692+ if (ret)
693+ return ret;
694+
695+ if (vma->vm_flags & VM_WRITE)
696+ return -EPERM;
697+
698+ end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz;
699+ if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end)
700+ return -EINVAL;
701+
702+ q_id = (phys_off - UBLKSRV_CMD_BUF_OFFSET) / max_sz;
703+ pr_devel("%s: qid %d, pid %d, addr %lx pg_off %lx sz %lu\n",
704+ __func__, q_id, current->pid, vma->vm_start,
705+ phys_off, (unsigned long)sz);
706+
707+ if (sz != ublk_queue_cmd_buf_size(ub, q_id))
708+ return -EINVAL;
709+
710+ pfn = virt_to_phys(ublk_queue_cmd_buf(ub, q_id)) >> PAGE_SHIFT;
711+ return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
712+}
713+
714+static void ublk_commit_completion(struct ublk_device *ub,
715+ struct ublksrv_io_cmd *ub_cmd)
716+{
717+ u32 qid = ub_cmd->q_id, tag = ub_cmd->tag;
718+ struct ublk_queue *ubq = ublk_get_queue(ub, qid);
719+ struct ublk_io *io = &ubq->ios[tag];
720+ struct request *req;
721+
722+ /* now this cmd slot is owned by nbd driver */
723+ io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV;
724+ io->res = ub_cmd->result;
725+
726+ /* find the io request and complete */
727+ req = blk_mq_tag_to_rq(ub->tag_set.tags[qid], tag);
728+
729+ if (req && likely(!blk_should_fake_timeout(req->q)))
730+ ublk_complete_rq(req);
731+}
732+
733+/*
734+ * When ->ubq_daemon is exiting, either new request is ended immediately,
735+ * or any queued io command is drained, so it is safe to abort queue
736+ * lockless
737+ */
738+static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)
739+{
740+ int i;
741+
742+ if (!ublk_get_device(ub))
743+ return;
744+
745+ for (i = 0; i < ubq->q_depth; i++) {
746+ struct ublk_io *io = &ubq->ios[i];
747+
748+ if (!(io->flags & UBLK_IO_FLAG_ACTIVE)) {
749+ struct request *rq;
750+
751+ /*
752+ * Either we fail the request or ublk_rq_task_work_fn
753+ * will do it
754+ */
755+ rq = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], i);
756+ if (rq)
757+ __ublk_fail_req(io, rq);
758+ }
759+ }
760+ ublk_put_device(ub);
761+}
762+
763+static void ublk_daemon_monitor_work(struct work_struct *work)
764+{
765+ struct ublk_device *ub =
766+ container_of(work, struct ublk_device, monitor_work.work);
767+ int i;
768+
769+ for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
770+ struct ublk_queue *ubq = ublk_get_queue(ub, i);
771+
772+ if (ubq_daemon_is_dying(ubq)) {
773+ schedule_work(&ub->stop_work);
774+
775+ /* abort queue is for making forward progress */
776+ ublk_abort_queue(ub, ubq);
777+ }
778+ }
779+
780+ /*
781+ * We can't schedule monitor work after ublk_remove() is started.
782+ *
783+ * No need ub->mutex, monitor work are canceled after state is marked
784+ * as DEAD, so DEAD state is observed reliably.
785+ */
786+ if (ub->dev_info.state != UBLK_S_DEV_DEAD)
787+ schedule_delayed_work(&ub->monitor_work,
788+ UBLK_DAEMON_MONITOR_PERIOD);
789+}
790+
791+static void ublk_cancel_queue(struct ublk_queue *ubq)
792+{
793+ int i;
794+
795+ for (i = 0; i < ubq->q_depth; i++) {
796+ struct ublk_io *io = &ubq->ios[i];
797+
798+ if (io->flags & UBLK_IO_FLAG_ACTIVE)
799+ io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, 0);
800+ }
801+}
802+
803+/* Cancel all pending commands, must be called after del_gendisk() returns */
804+static void ublk_cancel_dev(struct ublk_device *ub)
805+{
806+ int i;
807+
808+ for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
809+ ublk_cancel_queue(ublk_get_queue(ub, i));
810+}
811+
812+static void ublk_stop_dev(struct ublk_device *ub)
813+{
814+ mutex_lock(&ub->mutex);
815+ if (ub->dev_info.state != UBLK_S_DEV_LIVE)
816+ goto unlock;
817+
818+ del_gendisk(ub->ub_disk);
819+ ub->dev_info.state = UBLK_S_DEV_DEAD;
820+ ub->dev_info.ublksrv_pid = -1;
821+ ublk_cancel_dev(ub);
822+ put_disk(ub->ub_disk);
823+ ub->ub_disk = NULL;
824+ unlock:
825+ mutex_unlock(&ub->mutex);
826+ cancel_delayed_work_sync(&ub->monitor_work);
827+}
828+
829+static inline bool ublk_queue_ready(struct ublk_queue *ubq)
830+{
831+ return ubq->nr_io_ready == ubq->q_depth;
832+}
833+
834+/* device can only be started after all IOs are ready */
835+static void ublk_mark_io_ready(struct ublk_device *ub, struct ublk_queue *ubq)
836+{
837+ mutex_lock(&ub->mutex);
838+ ubq->nr_io_ready++;
839+ if (ublk_queue_ready(ubq)) {
840+ ubq->ubq_daemon = current;
841+ get_task_struct(ubq->ubq_daemon);
842+ ub->nr_queues_ready++;
843+ }
844+ if (ub->nr_queues_ready == ub->dev_info.nr_hw_queues)
845+ complete_all(&ub->completion);
846+ mutex_unlock(&ub->mutex);
847+}
848+
849+static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
850+{
851+ struct ublksrv_io_cmd *ub_cmd = (struct ublksrv_io_cmd *)cmd->cmd;
852+ struct ublk_device *ub = cmd->file->private_data;
853+ struct ublk_queue *ubq;
854+ struct ublk_io *io;
855+ u32 cmd_op = cmd->cmd_op;
856+ unsigned tag = ub_cmd->tag;
857+ int ret = -EINVAL;
858+
859+ pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n",
860+ __func__, cmd->cmd_op, ub_cmd->q_id, tag,
861+ ub_cmd->result);
862+
863+ if (!(issue_flags & IO_URING_F_SQE128))
864+ goto out;
865+
866+ if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues)
867+ goto out;
868+
869+ ubq = ublk_get_queue(ub, ub_cmd->q_id);
870+ if (!ubq || ub_cmd->q_id != ubq->q_id)
871+ goto out;
872+
873+ if (ubq->ubq_daemon && ubq->ubq_daemon != current)
874+ goto out;
875+
876+ if (tag >= ubq->q_depth)
877+ goto out;
878+
879+ io = &ubq->ios[tag];
880+
881+ /* there is pending io cmd, something must be wrong */
882+ if (io->flags & UBLK_IO_FLAG_ACTIVE) {
883+ ret = -EBUSY;
884+ goto out;
885+ }
886+
887+ switch (cmd_op) {
888+ case UBLK_IO_FETCH_REQ:
889+ /* UBLK_IO_FETCH_REQ is only allowed before queue is setup */
890+ if (ublk_queue_ready(ubq)) {
891+ ret = -EBUSY;
892+ goto out;
893+ }
894+ /*
895+ * The io is being handled by server, so COMMIT_RQ is expected
896+ * instead of FETCH_REQ
897+ */
898+ if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)
899+ goto out;
900+ /* FETCH_RQ has to provide IO buffer */
901+ if (!ub_cmd->addr)
902+ goto out;
903+ io->cmd = cmd;
904+ io->flags |= UBLK_IO_FLAG_ACTIVE;
905+ io->addr = ub_cmd->addr;
906+
907+ ublk_mark_io_ready(ub, ubq);
908+ break;
909+ case UBLK_IO_COMMIT_AND_FETCH_REQ:
910+ /* FETCH_RQ has to provide IO buffer */
911+ if (!ub_cmd->addr)
912+ goto out;
913+ if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV))
914+ goto out;
915+ io->addr = ub_cmd->addr;
916+ io->flags |= UBLK_IO_FLAG_ACTIVE;
917+ io->cmd = cmd;
918+ ublk_commit_completion(ub, ub_cmd);
919+ break;
920+ default:
921+ goto out;
922+ }
923+ return -EIOCBQUEUED;
924+
925+ out:
926+ io_uring_cmd_done(cmd, ret, 0);
927+ pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n",
928+ __func__, cmd_op, tag, ret, io->flags);
929+ return -EIOCBQUEUED;
930+}
931+
932+static const struct file_operations ublk_ch_fops = {
933+ .owner = THIS_MODULE,
934+ .open = ublk_ch_open,
935+ .release = ublk_ch_release,
936+ .llseek = no_llseek,
937+ .uring_cmd = ublk_ch_uring_cmd,
938+ .mmap = ublk_ch_mmap,
939+};
940+
941+static void ublk_deinit_queue(struct ublk_device *ub, int q_id)
942+{
943+ int size = ublk_queue_cmd_buf_size(ub, q_id);
944+ struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
945+
946+ if (ubq->ubq_daemon)
947+ put_task_struct(ubq->ubq_daemon);
948+ if (ubq->io_cmd_buf)
949+ free_pages((unsigned long)ubq->io_cmd_buf, get_order(size));
950+}
951+
952+static int ublk_init_queue(struct ublk_device *ub, int q_id)
953+{
954+ struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
955+ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
956+ void *ptr;
957+ int size;
958+
959+ ubq->flags = ub->dev_info.flags;
960+ ubq->q_id = q_id;
961+ ubq->q_depth = ub->dev_info.queue_depth;
962+ size = ublk_queue_cmd_buf_size(ub, q_id);
963+
964+ ptr = (void *) __get_free_pages(gfp_flags, get_order(size));
965+ if (!ptr)
966+ return -ENOMEM;
967+
968+ ubq->io_cmd_buf = ptr;
969+ ubq->dev = ub;
970+ return 0;
971+}
972+
973+static void ublk_deinit_queues(struct ublk_device *ub)
974+{
975+ int nr_queues = ub->dev_info.nr_hw_queues;
976+ int i;
977+
978+ if (!ub->__queues)
979+ return;
980+
981+ for (i = 0; i < nr_queues; i++)
982+ ublk_deinit_queue(ub, i);
983+ kfree(ub->__queues);
984+}
985+
986+static int ublk_init_queues(struct ublk_device *ub)
987+{
988+ int nr_queues = ub->dev_info.nr_hw_queues;
989+ int depth = ub->dev_info.queue_depth;
990+ int ubq_size = sizeof(struct ublk_queue) + depth * sizeof(struct ublk_io);
991+ int i, ret = -ENOMEM;
992+
993+ ub->queue_size = ubq_size;
994+ ub->__queues = kcalloc(nr_queues, ubq_size, GFP_KERNEL);
995+ if (!ub->__queues)
996+ return ret;
997+
998+ for (i = 0; i < nr_queues; i++) {
999+ if (ublk_init_queue(ub, i))
1000+ goto fail;
1001+ }
1002+
1003+ init_completion(&ub->completion);
1004+ return 0;
1005+
1006+ fail:
1007+ ublk_deinit_queues(ub);
1008+ return ret;
1009+}
1010+
1011+static int ublk_alloc_dev_number(struct ublk_device *ub, int idx)
1012+{
1013+ int i = idx;
1014+ int err;
1015+
1016+ spin_lock(&ublk_idr_lock);
1017+ /* allocate id, if @id >= 0, we're requesting that specific id */
1018+ if (i >= 0) {
1019+ err = idr_alloc(&ublk_index_idr, ub, i, i + 1, GFP_NOWAIT);
1020+ if (err == -ENOSPC)
1021+ err = -EEXIST;
1022+ } else {
1023+ err = idr_alloc(&ublk_index_idr, ub, 0, 0, GFP_NOWAIT);
1024+ }
1025+ spin_unlock(&ublk_idr_lock);
1026+
1027+ if (err >= 0)
1028+ ub->ub_number = err;
1029+
1030+ return err;
1031+}
1032+
1033+static void ublk_free_dev_number(struct ublk_device *ub)
1034+{
1035+ spin_lock(&ublk_idr_lock);
1036+ idr_remove(&ublk_index_idr, ub->ub_number);
1037+ wake_up_all(&ublk_idr_wq);
1038+ spin_unlock(&ublk_idr_lock);
1039+}
1040+
1041+static void ublk_cdev_rel(struct device *dev)
1042+{
1043+ struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev);
1044+
1045+ blk_mq_free_tag_set(&ub->tag_set);
1046+ ublk_deinit_queues(ub);
1047+ ublk_free_dev_number(ub);
1048+ mutex_destroy(&ub->mutex);
1049+ kfree(ub);
1050+}
1051+
1052+static int ublk_add_chdev(struct ublk_device *ub)
1053+{
1054+ struct device *dev = &ub->cdev_dev;
1055+ int minor = ub->ub_number;
1056+ int ret;
1057+
1058+ dev->parent = ublk_misc.this_device;
1059+ dev->devt = MKDEV(MAJOR(ublk_chr_devt), minor);
1060+ dev->class = ublk_chr_class;
1061+ dev->release = ublk_cdev_rel;
1062+ device_initialize(dev);
1063+
1064+ ret = dev_set_name(dev, "ublkc%d", minor);
1065+ if (ret)
1066+ goto fail;
1067+
1068+ cdev_init(&ub->cdev, &ublk_ch_fops);
1069+ ret = cdev_device_add(&ub->cdev, dev);
1070+ if (ret)
1071+ goto fail;
1072+ return 0;
1073+ fail:
1074+ put_device(dev);
1075+ return ret;
1076+}
1077+
1078+static void ublk_stop_work_fn(struct work_struct *work)
1079+{
1080+ struct ublk_device *ub =
1081+ container_of(work, struct ublk_device, stop_work);
1082+
1083+ ublk_stop_dev(ub);
1084+}
1085+
1086+/* align maximum I/O size to PAGE_SIZE */
1087+static void ublk_align_max_io_size(struct ublk_device *ub)
1088+{
1089+ unsigned int max_rq_bytes = ub->dev_info.rq_max_blocks << ub->bs_shift;
1090+
1091+ ub->dev_info.rq_max_blocks =
1092+ round_down(max_rq_bytes, PAGE_SIZE) >> ub->bs_shift;
1093+}
1094+
1095+static int ublk_add_tag_set(struct ublk_device *ub)
1096+{
1097+ ub->tag_set.ops = &ublk_mq_ops;
1098+ ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues;
1099+ ub->tag_set.queue_depth = ub->dev_info.queue_depth;
1100+ ub->tag_set.numa_node = NUMA_NO_NODE;
1101+ ub->tag_set.cmd_size = sizeof(struct ublk_rq_data);
1102+ ub->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1103+ ub->tag_set.driver_data = ub;
1104+ return blk_mq_alloc_tag_set(&ub->tag_set);
1105+}
1106+
1107+static void ublk_remove(struct ublk_device *ub)
1108+{
1109+ ublk_stop_dev(ub);
1110+ cancel_work_sync(&ub->stop_work);
1111+ cdev_device_del(&ub->cdev, &ub->cdev_dev);
1112+ put_device(&ub->cdev_dev);
1113+}
1114+
1115+static struct ublk_device *ublk_get_device_from_id(int idx)
1116+{
1117+ struct ublk_device *ub = NULL;
1118+
1119+ if (idx < 0)
1120+ return NULL;
1121+
1122+ spin_lock(&ublk_idr_lock);
1123+ ub = idr_find(&ublk_index_idr, idx);
1124+ if (ub)
1125+ ub = ublk_get_device(ub);
1126+ spin_unlock(&ublk_idr_lock);
1127+
1128+ return ub;
1129+}
1130+
1131+static int ublk_ctrl_start_dev(struct io_uring_cmd *cmd)
1132+{
1133+ struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1134+ int ublksrv_pid = (int)header->data[0];
1135+ unsigned long dev_blocks = header->data[1];
1136+ struct ublk_device *ub;
1137+ struct gendisk *disk;
1138+ int ret = -EINVAL;
1139+
1140+ if (ublksrv_pid <= 0)
1141+ return -EINVAL;
1142+
1143+ ub = ublk_get_device_from_id(header->dev_id);
1144+ if (!ub)
1145+ return -EINVAL;
1146+
1147+ wait_for_completion_interruptible(&ub->completion);
1148+
1149+ schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD);
1150+
1151+ mutex_lock(&ub->mutex);
1152+ if (ub->dev_info.state == UBLK_S_DEV_LIVE ||
1153+ test_bit(UB_STATE_USED, &ub->state)) {
1154+ ret = -EEXIST;
1155+ goto out_unlock;
1156+ }
1157+
1158+ /* We may get disk size updated */
1159+ if (dev_blocks)
1160+ ub->dev_info.dev_blocks = dev_blocks;
1161+
1162+ disk = blk_mq_alloc_disk(&ub->tag_set, ub);
1163+ if (IS_ERR(disk)) {
1164+ ret = PTR_ERR(disk);
1165+ goto out_unlock;
1166+ }
1167+ sprintf(disk->disk_name, "ublkb%d", ub->ub_number);
1168+ disk->fops = &ub_fops;
1169+ disk->private_data = ub;
1170+
1171+ blk_queue_logical_block_size(disk->queue, ub->dev_info.block_size);
1172+ blk_queue_physical_block_size(disk->queue, ub->dev_info.block_size);
1173+ blk_queue_io_min(disk->queue, ub->dev_info.block_size);
1174+ blk_queue_max_hw_sectors(disk->queue,
1175+ ub->dev_info.rq_max_blocks << (ub->bs_shift - 9));
1176+ disk->queue->limits.discard_granularity = PAGE_SIZE;
1177+ blk_queue_max_discard_sectors(disk->queue, UINT_MAX >> 9);
1178+ blk_queue_max_write_zeroes_sectors(disk->queue, UINT_MAX >> 9);
1179+
1180+ set_capacity(disk, ub->dev_info.dev_blocks << (ub->bs_shift - 9));
1181+
1182+ ub->dev_info.ublksrv_pid = ublksrv_pid;
1183+ ub->ub_disk = disk;
1184+ get_device(&ub->cdev_dev);
1185+ ret = add_disk(disk);
1186+ if (ret) {
1187+ put_disk(disk);
1188+ goto out_unlock;
1189+ }
1190+ set_bit(UB_STATE_USED, &ub->state);
1191+ ub->dev_info.state = UBLK_S_DEV_LIVE;
1192+out_unlock:
1193+ mutex_unlock(&ub->mutex);
1194+ ublk_put_device(ub);
1195+ return ret;
1196+}
1197+
1198+static int ublk_ctrl_get_queue_affinity(struct io_uring_cmd *cmd)
1199+{
1200+ struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1201+ void __user *argp = (void __user *)(unsigned long)header->addr;
1202+ struct ublk_device *ub;
1203+ cpumask_var_t cpumask;
1204+ unsigned long queue;
1205+ unsigned int retlen;
1206+ unsigned int i;
1207+ int ret = -EINVAL;
1208+
1209+ if (header->len * BITS_PER_BYTE < nr_cpu_ids)
1210+ return -EINVAL;
1211+ if (header->len & (sizeof(unsigned long)-1))
1212+ return -EINVAL;
1213+ if (!header->addr)
1214+ return -EINVAL;
1215+
1216+ ub = ublk_get_device_from_id(header->dev_id);
1217+ if (!ub)
1218+ return -EINVAL;
1219+
1220+ queue = header->data[0];
1221+ if (queue >= ub->dev_info.nr_hw_queues)
1222+ goto out_put_device;
1223+
1224+ ret = -ENOMEM;
1225+ if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
1226+ goto out_put_device;
1227+
1228+ for_each_possible_cpu(i) {
1229+ if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[i] == queue)
1230+ cpumask_set_cpu(i, cpumask);
1231+ }
1232+
1233+ ret = -EFAULT;
1234+ retlen = min_t(unsigned short, header->len, cpumask_size());
1235+ if (copy_to_user(argp, cpumask, retlen))
1236+ goto out_free_cpumask;
1237+ if (retlen != header->len &&
1238+ clear_user(argp + retlen, header->len - retlen))
1239+ goto out_free_cpumask;
1240+
1241+ ret = 0;
1242+out_free_cpumask:
1243+ free_cpumask_var(cpumask);
1244+out_put_device:
1245+ ublk_put_device(ub);
1246+ return ret;
1247+}
1248+
1249+static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info)
1250+{
1251+ pr_devel("%s: dev id %d flags %llx\n", __func__,
1252+ info->dev_id, info->flags);
1253+ pr_devel("\t nr_hw_queues %d queue_depth %d block size %d dev_capacity %lld\n",
1254+ info->nr_hw_queues, info->queue_depth,
1255+ info->block_size, info->dev_blocks);
1256+}
1257+
1258+static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
1259+{
1260+ struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1261+ void __user *argp = (void __user *)(unsigned long)header->addr;
1262+ struct ublksrv_ctrl_dev_info info;
1263+ struct ublk_device *ub;
1264+ int ret = -EINVAL;
1265+
1266+ if (header->len < sizeof(info) || !header->addr)
1267+ return -EINVAL;
1268+ if (header->queue_id != (u16)-1) {
1269+ pr_warn("%s: queue_id is wrong %x\n",
1270+ __func__, header->queue_id);
1271+ return -EINVAL;
1272+ }
1273+ if (copy_from_user(&info, argp, sizeof(info)))
1274+ return -EFAULT;
1275+ ublk_dump_dev_info(&info);
1276+ if (header->dev_id != info.dev_id) {
1277+ pr_warn("%s: dev id not match %u %u\n",
1278+ __func__, header->dev_id, info.dev_id);
1279+ return -EINVAL;
1280+ }
1281+
1282+ ret = mutex_lock_killable(&ublk_ctl_mutex);
1283+ if (ret)
1284+ return ret;
1285+
1286+ ret = -ENOMEM;
1287+ ub = kzalloc(sizeof(*ub), GFP_KERNEL);
1288+ if (!ub)
1289+ goto out_unlock;
1290+ mutex_init(&ub->mutex);
1291+ spin_lock_init(&ub->mm_lock);
1292+ INIT_WORK(&ub->stop_work, ublk_stop_work_fn);
1293+ INIT_DELAYED_WORK(&ub->monitor_work, ublk_daemon_monitor_work);
1294+
1295+ ret = ublk_alloc_dev_number(ub, header->dev_id);
1296+ if (ret < 0)
1297+ goto out_free_ub;
1298+
1299+ memcpy(&ub->dev_info, &info, sizeof(info));
1300+
1301+ /* update device id */
1302+ ub->dev_info.dev_id = ub->ub_number;
1303+
1304+ /*
1305+ * 64bit flags will be copied back to userspace as feature
1306+ * negotiation result, so have to clear flags which driver
1307+ * doesn't support yet, then userspace can get correct flags
1308+ * (features) to handle.
1309+ */
1310+ ub->dev_info.flags &= UBLK_F_ALL;
1311+
1312+ /* We are not ready to support zero copy */
1313+ ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY;
1314+
1315+ ub->bs_shift = ilog2(ub->dev_info.block_size);
1316+ ub->dev_info.nr_hw_queues = min_t(unsigned int,
1317+ ub->dev_info.nr_hw_queues, nr_cpu_ids);
1318+ ublk_align_max_io_size(ub);
1319+
1320+ ret = ublk_init_queues(ub);
1321+ if (ret)
1322+ goto out_free_dev_number;
1323+
1324+ ret = ublk_add_tag_set(ub);
1325+ if (ret)
1326+ goto out_deinit_queues;
1327+
1328+ ret = -EFAULT;
1329+ if (copy_to_user(argp, &ub->dev_info, sizeof(info)))
1330+ goto out_free_tag_set;
1331+
1332+ /*
1333+ * Add the char dev so that ublksrv daemon can be setup.
1334+ * ublk_add_chdev() will cleanup everything if it fails.
1335+ */
1336+ ret = ublk_add_chdev(ub);
1337+ goto out_unlock;
1338+
1339+out_free_tag_set:
1340+ blk_mq_free_tag_set(&ub->tag_set);
1341+out_deinit_queues:
1342+ ublk_deinit_queues(ub);
1343+out_free_dev_number:
1344+ ublk_free_dev_number(ub);
1345+out_free_ub:
1346+ mutex_destroy(&ub->mutex);
1347+ kfree(ub);
1348+out_unlock:
1349+ mutex_unlock(&ublk_ctl_mutex);
1350+ return ret;
1351+}
1352+
1353+static inline bool ublk_idr_freed(int id)
1354+{
1355+ void *ptr;
1356+
1357+ spin_lock(&ublk_idr_lock);
1358+ ptr = idr_find(&ublk_index_idr, id);
1359+ spin_unlock(&ublk_idr_lock);
1360+
1361+ return ptr == NULL;
1362+}
1363+
1364+static int ublk_ctrl_del_dev(int idx)
1365+{
1366+ struct ublk_device *ub;
1367+ int ret;
1368+
1369+ ret = mutex_lock_killable(&ublk_ctl_mutex);
1370+ if (ret)
1371+ return ret;
1372+
1373+ ub = ublk_get_device_from_id(idx);
1374+ if (ub) {
1375+ ublk_remove(ub);
1376+ ublk_put_device(ub);
1377+ ret = 0;
1378+ } else {
1379+ ret = -ENODEV;
1380+ }
1381+
1382+ /*
1383+ * Wait until the idr is removed, then it can be reused after
1384+ * DEL_DEV command is returned.
1385+ */
1386+ if (!ret)
1387+ wait_event(ublk_idr_wq, ublk_idr_freed(idx));
1388+ mutex_unlock(&ublk_ctl_mutex);
1389+
1390+ return ret;
1391+}
1392+
1393+static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd)
1394+{
1395+ struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1396+
1397+ pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n",
1398+ __func__, cmd->cmd_op, header->dev_id, header->queue_id,
1399+ header->data[0], header->addr, header->len);
1400+}
1401+
1402+static int ublk_ctrl_stop_dev(struct io_uring_cmd *cmd)
1403+{
1404+ struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1405+ struct ublk_device *ub;
1406+
1407+ ub = ublk_get_device_from_id(header->dev_id);
1408+ if (!ub)
1409+ return -EINVAL;
1410+
1411+ ublk_stop_dev(ub);
1412+ cancel_work_sync(&ub->stop_work);
1413+
1414+ ublk_put_device(ub);
1415+ return 0;
1416+}
1417+
1418+static int ublk_ctrl_get_dev_info(struct io_uring_cmd *cmd)
1419+{
1420+ struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1421+ void __user *argp = (void __user *)(unsigned long)header->addr;
1422+ struct ublk_device *ub;
1423+ int ret = 0;
1424+
1425+ if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr)
1426+ return -EINVAL;
1427+
1428+ ub = ublk_get_device_from_id(header->dev_id);
1429+ if (!ub)
1430+ return -EINVAL;
1431+
1432+ if (copy_to_user(argp, &ub->dev_info, sizeof(ub->dev_info)))
1433+ ret = -EFAULT;
1434+ ublk_put_device(ub);
1435+
1436+ return ret;
1437+}
1438+
1439+static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
1440+ unsigned int issue_flags)
1441+{
1442+ struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1443+ int ret = -EINVAL;
1444+
1445+ ublk_ctrl_cmd_dump(cmd);
1446+
1447+ if (!(issue_flags & IO_URING_F_SQE128))
1448+ goto out;
1449+
1450+ ret = -EPERM;
1451+ if (!capable(CAP_SYS_ADMIN))
1452+ goto out;
1453+
1454+ ret = -ENODEV;
1455+ switch (cmd->cmd_op) {
1456+ case UBLK_CMD_START_DEV:
1457+ ret = ublk_ctrl_start_dev(cmd);
1458+ break;
1459+ case UBLK_CMD_STOP_DEV:
1460+ ret = ublk_ctrl_stop_dev(cmd);
1461+ break;
1462+ case UBLK_CMD_GET_DEV_INFO:
1463+ ret = ublk_ctrl_get_dev_info(cmd);
1464+ break;
1465+ case UBLK_CMD_ADD_DEV:
1466+ ret = ublk_ctrl_add_dev(cmd);
1467+ break;
1468+ case UBLK_CMD_DEL_DEV:
1469+ ret = ublk_ctrl_del_dev(header->dev_id);
1470+ break;
1471+ case UBLK_CMD_GET_QUEUE_AFFINITY:
1472+ ret = ublk_ctrl_get_queue_affinity(cmd);
1473+ break;
1474+ default:
1475+ break;
1476+ }
1477+ out:
1478+ io_uring_cmd_done(cmd, ret, 0);
1479+ pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n",
1480+ __func__, ret, cmd->cmd_op, header->dev_id, header->queue_id);
1481+ return -EIOCBQUEUED;
1482+}
1483+
1484+static const struct file_operations ublk_ctl_fops = {
1485+ .open = nonseekable_open,
1486+ .uring_cmd = ublk_ctrl_uring_cmd,
1487+ .owner = THIS_MODULE,
1488+ .llseek = noop_llseek,
1489+};
1490+
1491+static struct miscdevice ublk_misc = {
1492+ .minor = MISC_DYNAMIC_MINOR,
1493+ .name = "ublk-control",
1494+ .fops = &ublk_ctl_fops,
1495+};
1496+
1497+static int __init ublk_init(void)
1498+{
1499+ int ret;
1500+
1501+ init_waitqueue_head(&ublk_idr_wq);
1502+
1503+ ret = misc_register(&ublk_misc);
1504+ if (ret)
1505+ return ret;
1506+
1507+ ret = alloc_chrdev_region(&ublk_chr_devt, 0, UBLK_MINORS, "ublk-char");
1508+ if (ret)
1509+ goto unregister_mis;
1510+
1511+ ublk_chr_class = class_create(THIS_MODULE, "ublk-char");
1512+ if (IS_ERR(ublk_chr_class)) {
1513+ ret = PTR_ERR(ublk_chr_class);
1514+ goto free_chrdev_region;
1515+ }
1516+ return 0;
1517+
1518+free_chrdev_region:
1519+ unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
1520+unregister_mis:
1521+ misc_deregister(&ublk_misc);
1522+ return ret;
1523+}
1524+
1525+static void __exit ublk_exit(void)
1526+{
1527+ struct ublk_device *ub;
1528+ int id;
1529+
1530+ class_destroy(ublk_chr_class);
1531+
1532+ misc_deregister(&ublk_misc);
1533+
1534+ idr_for_each_entry(&ublk_index_idr, ub, id)
1535+ ublk_remove(ub);
1536+
1537+ idr_destroy(&ublk_index_idr);
1538+ unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
1539+}
1540+
1541+module_init(ublk_init);
1542+module_exit(ublk_exit);
1543+
1544+MODULE_AUTHOR("Ming Lei <ming.lei@redhat.com>");
1545+MODULE_LICENSE("GPL");
drivers/block/virtio_blk.c+1−2
@@ -1089,7 +1089,7 @@ static int virtblk_probe(struct virtio_device *vdev) · 1088 unmodified lines
10891089 return 0;
10901090
10911091 out_cleanup_disk:
1092− blk_cleanup_disk(vblk->disk);
1092+ put_disk(vblk->disk);
10931093 out_free_tags:
10941094 blk_mq_free_tag_set(&vblk->tag_set);
10951095 out_free_vq:
@@ -1111,7 +1111,6 @@ static void virtblk_remove(struct virtio_device *vdev) · 15 unmodified lines
11111111 flush_work(&vblk->config_work);
11121112
11131113 del_gendisk(vblk->disk);
1114− blk_cleanup_queue(vblk->disk->queue);
11151114 blk_mq_free_tag_set(&vblk->tag_set);
11161115
11171116 mutex_lock(&vblk->vdev_mutex);
drivers/block/xen-blkback/blkback.c+3−3
@@ -442,7 +442,7 @@ static void free_req(struct xen_blkif_ring *ring, struct pending_req *req) · 441 unmodified lines
442442 * Routines for managing virtual block devices (vbds).
443443 */
444444 static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif,
445− int operation)
445+ enum req_op operation)
446446 {
447447 struct xen_vbd *vbd = &blkif->vbd;
448448 int rc = -EACCES;
@@ -1193,8 +1193,8 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, · 744 unmodified lines
11931193 struct bio *bio = NULL;
11941194 struct bio **biolist = pending_req->biolist;
11951195 int i, nbio = 0;
1196− int operation;
1197− int operation_flags = 0;
1196+ enum req_op operation;
1197+ blk_opf_t operation_flags = 0;
11981198 struct blk_plug plug;
11991199 bool drain = false;
12001200 struct grant_page **pages = pending_req->segments;
drivers/block/xen-blkfront.c+2−2
@@ -2397,7 +2397,7 @@ static void blkfront_connect(struct blkfront_info *info) · 2396 unmodified lines
23972397
23982398 err = device_add_disk(&info->xbdev->dev, info->gd, NULL);
23992399 if (err) {
2400− blk_cleanup_disk(info->gd);
2400+ put_disk(info->gd);
24012401 blk_mq_free_tag_set(&info->tag_set);
24022402 info->rq = NULL;
24032403 goto fail;
@@ -2482,7 +2482,7 @@ static int blkfront_remove(struct xenbus_device *xbdev) · 78 unmodified lines
24822482 blkif_free(info, 0);
24832483 if (info->gd) {
24842484 xlbd_release_minors(info->gd->first_minor, info->gd->minors);
2485− blk_cleanup_disk(info->gd);
2485+ put_disk(info->gd);
24862486 blk_mq_free_tag_set(&info->tag_set);
24872487 }
24882488
drivers/block/z2ram.c+1−2
@@ -337,7 +337,7 @@ static int z2ram_register_disk(int minor) · 336 unmodified lines
337337 z2ram_gendisk[minor] = disk;
338338 err = add_disk(disk);
339339 if (err)
340− blk_cleanup_disk(disk);
340+ put_disk(disk);
341341 return err;
342342 }
343343
@@ -384,7 +384,6 @@ static void __exit z2_exit(void) · 40 unmodified lines
384384
385385 for (i = 0; i < Z2MINOR_COUNT; i++) {
386386 del_gendisk(z2ram_gendisk[i]);
387− blk_cleanup_queue(z2ram_gendisk[i]->queue);
388387 put_disk(z2ram_gendisk[i]);
389388 }
390389 blk_mq_free_tag_set(&tag_set);
drivers/block/zram/zram_drv.c+4−4
@@ -1523,7 +1523,7 @@ static void zram_bio_discard(struct zram *zram, u32 index, · 1522 unmodified lines
15231523 * Returns 1 if IO request was successfully submitted.
15241524 */
15251525 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
1526− int offset, unsigned int op, struct bio *bio)
1526+ int offset, enum req_op op, struct bio *bio)
15271527 {
15281528 int ret;
15291529
@@ -1631,7 +1631,7 @@ static void zram_slot_free_notify(struct block_device *bdev, · 101 unmodified lines
16311631 }
16321632
16331633 static int zram_rw_page(struct block_device *bdev, sector_t sector,
1634− struct page *page, unsigned int op)
1634+ struct page *page, enum req_op op)
16351635 {
16361636 int offset, ret;
16371637 u32 index;
@@ -1957,7 +1957,7 @@ static int zram_add(void) · 319 unmodified lines
19571957 return device_id;
19581958
19591959 out_cleanup_disk:
1960− blk_cleanup_disk(zram->disk);
1960+ put_disk(zram->disk);
19611961 out_free_idr:
19621962 idr_remove(&zram_index_idr, device_id);
19631963 out_free_dev:
@@ -2008,7 +2008,7 @@ static int zram_remove(struct zram *zram) · 44 unmodified lines
20082008 */
20092009 zram_reset_device(zram);
20102010
2011− blk_cleanup_disk(zram->disk);
2011+ put_disk(zram->disk);
20122012 kfree(zram);
20132013 return 0;
20142014 }
drivers/cdrom/gdrom.c+1−2
@@ -817,7 +817,7 @@ probe_fail_free_irqs: · 816 unmodified lines
817817 free_irq(HW_EVENT_GDROM_DMA, &gd);
818818 free_irq(HW_EVENT_GDROM_CMD, &gd);
819819 probe_fail_cleanup_disk:
820− blk_cleanup_disk(gd.disk);
820+ put_disk(gd.disk);
821821 probe_fail_free_tag_set:
822822 blk_mq_free_tag_set(&gd.tag_set);
823823 probe_fail_free_cd_info:
@@ -831,7 +831,6 @@ probe_fail_no_mem: · 7 unmodified lines
831831
832832 static int remove_gdrom(struct platform_device *devptr)
833833 {
834− blk_cleanup_queue(gd.gdrom_rq);
835834 blk_mq_free_tag_set(&gd.tag_set);
836835 free_irq(HW_EVENT_GDROM_CMD, &gd);
837836 free_irq(HW_EVENT_GDROM_DMA, &gd);
drivers/infiniband/ulp/srp/ib_srp.c+1−2
@@ -1282,8 +1282,7 @@ struct srp_terminate_context { · 1281 unmodified lines
12821282 int scsi_result;
12831283 };
12841284
1285−static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr,
1286− bool reserved)
1285+static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr)
12871286 {
12881287 struct srp_terminate_context *context = context_ptr;
12891288 struct srp_target_port *target = context->srp_target;
drivers/md/bcache/super.c+13−14
@@ -414,8 +414,8 @@ static void uuid_io_unlock(struct closure *cl) · 413 unmodified lines
414414 up(&c->uuid_write_mutex);
415415 }
416416
417−static void uuid_io(struct cache_set *c, int op, unsigned long op_flags,
418− struct bkey *k, struct closure *parent)
417+static void uuid_io(struct cache_set *c, blk_opf_t opf, struct bkey *k,
418+ struct closure *parent)
419419 {
420420 struct closure *cl = &c->uuid_write;
421421 struct uuid_entry *u;
@@ -429,22 +429,22 @@ static void uuid_io(struct cache_set *c, int op, unsigned long op_flags, · 7 unmodified lines
429429 for (i = 0; i < KEY_PTRS(k); i++) {
430430 struct bio *bio = bch_bbio_alloc(c);
431431
432− bio->bi_opf = REQ_SYNC | REQ_META | op_flags;
432+ bio->bi_opf = opf | REQ_SYNC | REQ_META;
433433 bio->bi_iter.bi_size = KEY_SIZE(k) << 9;
434434
435435 bio->bi_end_io = uuid_endio;
436436 bio->bi_private = cl;
437− bio_set_op_attrs(bio, op, REQ_SYNC|REQ_META|op_flags);
438437 bch_bio_map(bio, c->uuids);
439438
440439 bch_submit_bbio(bio, c, k, i);
441440
442− if (op != REQ_OP_WRITE)
441+ if ((opf & REQ_OP_MASK) != REQ_OP_WRITE)
443442 break;
444443 }
445444
446445 bch_extent_to_text(buf, sizeof(buf), k);
447− pr_debug("%s UUIDs at %s\n", op == REQ_OP_WRITE ? "wrote" : "read", buf);
446+ pr_debug("%s UUIDs at %s\n", (opf & REQ_OP_MASK) == REQ_OP_WRITE ?
447+ "wrote" : "read", buf);
448448
449449 for (u = c->uuids; u < c->uuids + c->nr_uuids; u++)
450450 if (!bch_is_zero(u->uuid, 16))
@@ -463,7 +463,7 @@ static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl) · 12 unmodified lines
463463 return "bad uuid pointer";
464464
465465 bkey_copy(&c->uuid_bucket, k);
466− uuid_io(c, REQ_OP_READ, 0, k, cl);
466+ uuid_io(c, REQ_OP_READ, k, cl);
467467
468468 if (j->version < BCACHE_JSET_VERSION_UUIDv1) {
469469 struct uuid_entry_v0 *u0 = (void *) c->uuids;
@@ -511,7 +511,7 @@ static int __uuid_write(struct cache_set *c) · 41 unmodified lines
511511
512512 size = meta_bucket_pages(&ca->sb) * PAGE_SECTORS;
513513 SET_KEY_SIZE(&k.key, size);
514− uuid_io(c, REQ_OP_WRITE, 0, &k.key, &cl);
514+ uuid_io(c, REQ_OP_WRITE, &k.key, &cl);
515515 closure_sync(&cl);
516516
517517 /* Only one bucket used for uuid write */
@@ -587,8 +587,7 @@ static void prio_endio(struct bio *bio) · 69 unmodified lines
587587 closure_put(&ca->prio);
588588 }
589589
590−static void prio_io(struct cache *ca, uint64_t bucket, int op,
591− unsigned long op_flags)
590+static void prio_io(struct cache *ca, uint64_t bucket, blk_opf_t opf)
592591 {
593592 struct closure *cl = &ca->prio;
594593 struct bio *bio = bch_bbio_alloc(ca->set);
@@ -601,7 +600,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, int op, · 6 unmodified lines
601600
602601 bio->bi_end_io = prio_endio;
603602 bio->bi_private = ca;
604− bio_set_op_attrs(bio, op, REQ_SYNC|REQ_META|op_flags);
603+ bio->bi_opf = opf | REQ_SYNC | REQ_META;
605604 bch_bio_map(bio, ca->disk_buckets);
606605
607606 closure_bio_submit(ca->set, bio, &ca->prio);
@@ -661,7 +660,7 @@ int bch_prio_write(struct cache *ca, bool wait) · 53 unmodified lines
661660 BUG_ON(bucket == -1);
662661
663662 mutex_unlock(&ca->set->bucket_lock);
664− prio_io(ca, bucket, REQ_OP_WRITE, 0);
663+ prio_io(ca, bucket, REQ_OP_WRITE);
665664 mutex_lock(&ca->set->bucket_lock);
666665
667666 ca->prio_buckets[i] = bucket;
@@ -705,7 +704,7 @@ static int prio_read(struct cache *ca, uint64_t bucket) · 37 unmodified lines
705704 ca->prio_last_buckets[bucket_nr] = bucket;
706705 bucket_nr++;
707706
708− prio_io(ca, bucket, REQ_OP_READ, 0);
707+ prio_io(ca, bucket, REQ_OP_READ);
709708
710709 if (p->csum !=
711710 bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8)) {
@@ -884,7 +883,7 @@ static void bcache_device_free(struct bcache_device *d) · 172 unmodified lines
884883 if (disk) {
885884 ida_simple_remove(&bcache_device_idx,
886885 first_minor_to_idx(disk->first_minor));
887− blk_cleanup_disk(disk);
886+ put_disk(disk);
888887 }
889888
890889 bioset_exit(&d->bio_split);
drivers/md/dm-bufio.c+12−14
@@ -577,13 +577,12 @@ static void dmio_complete(unsigned long error, void *context) · 576 unmodified lines
577577 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0);
578578 }
579579
580−static void use_dmio(struct dm_buffer *b, int rw, sector_t sector,
580+static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
581581 unsigned n_sectors, unsigned offset)
582582 {
583583 int r;
584584 struct dm_io_request io_req = {
585− .bi_op = rw,
586− .bi_op_flags = 0,
585+ .bi_opf = op,
587586 .notify.fn = dmio_complete,
588587 .notify.context = b,
589588 .client = b->c->dm_io,
@@ -616,7 +615,7 @@ static void bio_complete(struct bio *bio) · 26 unmodified lines
616615 b->end_io(b, status);
617616 }
618617
619−static void use_bio(struct dm_buffer *b, int rw, sector_t sector,
618+static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector,
620619 unsigned n_sectors, unsigned offset)
621620 {
622621 struct bio *bio;
@@ -630,10 +629,10 @@ static void use_bio(struct dm_buffer *b, int rw, sector_t sector, · 7 unmodified lines
630629 bio = bio_kmalloc(vec_size, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN);
631630 if (!bio) {
632631 dmio:
633− use_dmio(b, rw, sector, n_sectors, offset);
632+ use_dmio(b, op, sector, n_sectors, offset);
634633 return;
635634 }
636− bio_init(bio, b->c->bdev, bio->bi_inline_vecs, vec_size, rw);
635+ bio_init(bio, b->c->bdev, bio->bi_inline_vecs, vec_size, op);
637636 bio->bi_iter.bi_sector = sector;
638637 bio->bi_end_io = bio_complete;
639638 bio->bi_private = b;
@@ -669,7 +668,8 @@ static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block · 29 unmodified lines
669668 return sector;
670669 }
671670
672−static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buffer *, blk_status_t))
671+static void submit_io(struct dm_buffer *b, enum req_op op,
672+ void (*end_io)(struct dm_buffer *, blk_status_t))
673673 {
674674 unsigned n_sectors;
675675 sector_t sector;
@@ -679,7 +679,7 @@ static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buff · 3 unmodified lines
679679
680680 sector = block_to_sector(b->c, b->block);
681681
682− if (rw != REQ_OP_WRITE) {
682+ if (op != REQ_OP_WRITE) {
683683 n_sectors = b->c->block_size >> SECTOR_SHIFT;
684684 offset = 0;
685685 } else {
@@ -698,9 +698,9 @@ static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buff · 12 unmodified lines
698698 }
699699
700700 if (b->data_mode != DATA_MODE_VMALLOC)
701− use_bio(b, rw, sector, n_sectors, offset);
701+ use_bio(b, op, sector, n_sectors, offset);
702702 else
703− use_dmio(b, rw, sector, n_sectors, offset);
703+ use_dmio(b, op, sector, n_sectors, offset);
704704 }
705705
706706 /*----------------------------------------------------------------
@@ -1341,8 +1341,7 @@ EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers); · 634 unmodified lines
13411341 int dm_bufio_issue_flush(struct dm_bufio_client *c)
13421342 {
13431343 struct dm_io_request io_req = {
1344− .bi_op = REQ_OP_WRITE,
1345− .bi_op_flags = REQ_PREFLUSH | REQ_SYNC,
1344+ .bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC,
13461345 .mem.type = DM_IO_KMEM,
13471346 .mem.ptr.addr = NULL,
13481347 .client = c->dm_io,
@@ -1365,8 +1364,7 @@ EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); · 16 unmodified lines
13651364 int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count)
13661365 {
13671366 struct dm_io_request io_req = {
1368− .bi_op = REQ_OP_DISCARD,
1369− .bi_op_flags = REQ_SYNC,
1367+ .bi_opf = REQ_OP_DISCARD | REQ_SYNC,
13701368 .mem.type = DM_IO_KMEM,
13711369 .mem.ptr.addr = NULL,
13721370 .client = c->dm_io,
drivers/md/dm-ebs-target.c+8−7
@@ -61,7 +61,8 @@ static inline bool __ebs_check_bs(unsigned int bs) · 60 unmodified lines
6161 *
6262 * copy blocks between bufio blocks and bio vector's (partial/overlapping) pages.
6363 */
64−static int __ebs_rw_bvec(struct ebs_c *ec, int rw, struct bio_vec *bv, struct bvec_iter *iter)
64+static int __ebs_rw_bvec(struct ebs_c *ec, enum req_op op, struct bio_vec *bv,
65+ struct bvec_iter *iter)
6566 {
6667 int r = 0;
6768 unsigned char *ba, *pa;
@@ -81,7 +82,7 @@ static int __ebs_rw_bvec(struct ebs_c *ec, int rw, struct bio_vec *bv, struct bv · 13 unmodified lines
8182 cur_len = min(dm_bufio_get_block_size(ec->bufio) - buf_off, bv_len);
8283
8384 /* Avoid reading for writes in case bio vector's page overwrites block completely. */
84− if (rw == READ || buf_off || bv_len < dm_bufio_get_block_size(ec->bufio))
85+ if (op == REQ_OP_READ || buf_off || bv_len < dm_bufio_get_block_size(ec->bufio))
8586 ba = dm_bufio_read(ec->bufio, block, &b);
8687 else
8788 ba = dm_bufio_new(ec->bufio, block, &b);
@@ -95,7 +96,7 @@ static int __ebs_rw_bvec(struct ebs_c *ec, int rw, struct bio_vec *bv, struct bv · 7 unmodified lines
9596 } else {
9697 /* Copy data to/from bio to buffer if read/new was successful above. */
9798 ba += buf_off;
98− if (rw == READ) {
99+ if (op == REQ_OP_READ) {
99100 memcpy(pa, ba, cur_len);
100101 flush_dcache_page(bv->bv_page);
101102 } else {
@@ -117,14 +118,14 @@ static int __ebs_rw_bvec(struct ebs_c *ec, int rw, struct bio_vec *bv, struct bv · 15 unmodified lines
117118 }
118119
119120 /* READ/WRITE: iterate bio vector's copying between (partial) pages and bufio blocks. */
120−static int __ebs_rw_bio(struct ebs_c *ec, int rw, struct bio *bio)
121+static int __ebs_rw_bio(struct ebs_c *ec, enum req_op op, struct bio *bio)
121122 {
122123 int r = 0, rr;
123124 struct bio_vec bv;
124125 struct bvec_iter iter;
125126
126127 bio_for_each_bvec(bv, bio, iter) {
127− rr = __ebs_rw_bvec(ec, rw, &bv, &iter);
128+ rr = __ebs_rw_bvec(ec, op, &bv, &iter);
128129 if (rr)
129130 r = rr;
130131 }
@@ -205,10 +206,10 @@ static void __ebs_process_bios(struct work_struct *ws) · 74 unmodified lines
205206 bio_list_for_each(bio, &bios) {
206207 r = -EIO;
207208 if (bio_op(bio) == REQ_OP_READ)
208− r = __ebs_rw_bio(ec, READ, bio);
209+ r = __ebs_rw_bio(ec, REQ_OP_READ, bio);
209210 else if (bio_op(bio) == REQ_OP_WRITE) {
210211 write = true;
211− r = __ebs_rw_bio(ec, WRITE, bio);
212+ r = __ebs_rw_bio(ec, REQ_OP_WRITE, bio);
212213 } else if (bio_op(bio) == REQ_OP_DISCARD) {
213214 __ebs_forget_bio(ec, bio);
214215 r = __ebs_discard_bio(ec, bio);
drivers/md/dm-flakey.c+6−2
@@ -32,7 +32,7 @@ struct flakey_c { · 31 unmodified lines
3232 unsigned corrupt_bio_byte;
3333 unsigned corrupt_bio_rw;
3434 unsigned corrupt_bio_value;
35− unsigned corrupt_bio_flags;
35+ blk_opf_t corrupt_bio_flags;
3636 };
3737
3838 enum feature_flag_bits {
@@ -145,7 +145,11 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc, · 106 unmodified lines
145145 /*
146146 * Only corrupt bios with these flags set.
147147 */
148− r = dm_read_arg(_args + 3, as, &fc->corrupt_bio_flags, &ti->error);
148+ BUILD_BUG_ON(sizeof(fc->corrupt_bio_flags) !=
149+ sizeof(unsigned int));
150+ r = dm_read_arg(_args + 3, as,
151+ (__force unsigned *)&fc->corrupt_bio_flags,
152+ &ti->error);
149153 if (r)
150154 return r;
151155 argc--;
drivers/md/dm-integrity.c+38−38
@@ -298,7 +298,7 @@ struct dm_integrity_io { · 297 unmodified lines
298298 struct work_struct work;
299299
300300 struct dm_integrity_c *ic;
301− enum req_opf op;
301+ enum req_op op;
302302 bool fua;
303303
304304 struct dm_integrity_range range;
@@ -551,14 +551,14 @@ static int sb_mac(struct dm_integrity_c *ic, bool wr) · 246 unmodified lines
551551 return 0;
552552 }
553553
554−static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags)
554+static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf)
555555 {
556556 struct dm_io_request io_req;
557557 struct dm_io_region io_loc;
558+ const enum req_op op = opf & REQ_OP_MASK;
558559 int r;
559560
560− io_req.bi_op = op;
561− io_req.bi_op_flags = op_flags;
561+ io_req.bi_opf = opf;
562562 io_req.mem.type = DM_IO_KMEM;
563563 io_req.mem.ptr.addr = ic->sb;
564564 io_req.notify.fn = NULL;
@@ -1050,8 +1050,9 @@ static void complete_journal_io(unsigned long error, void *context) · 485 unmodified lines
10501050 complete_journal_op(comp);
10511051 }
10521052
1053−static void rw_journal_sectors(struct dm_integrity_c *ic, int op, int op_flags,
1054− unsigned sector, unsigned n_sectors, struct journal_completion *comp)
1053+static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf,
1054+ unsigned sector, unsigned n_sectors,
1055+ struct journal_completion *comp)
10551056 {
10561057 struct dm_io_request io_req;
10571058 struct dm_io_region io_loc;
@@ -1067,8 +1068,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, int op, int op_flags, · 9 unmodified lines
10671068 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
10681069 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
10691070
1070− io_req.bi_op = op;
1071− io_req.bi_op_flags = op_flags;
1071+ io_req.bi_opf = opf;
10721072 io_req.mem.type = DM_IO_PAGE_LIST;
10731073 if (ic->journal_io)
10741074 io_req.mem.ptr.pl = &ic->journal_io[pl_index];
@@ -1088,7 +1088,8 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, int op, int op_flags, · 13 unmodified lines
10881088
10891089 r = dm_io(&io_req, 1, &io_loc, NULL);
10901090 if (unlikely(r)) {
1091− dm_integrity_io_error(ic, op == REQ_OP_READ ? "reading journal" : "writing journal", r);
1091+ dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ?
1092+ "reading journal" : "writing journal", r);
10921093 if (comp) {
10931094 WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
10941095 complete_journal_io(-1UL, comp);
@@ -1096,15 +1097,16 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, int op, int op_flags, · 1 unmodified line
10961097 }
10971098 }
10981099
1099−static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned section,
1100− unsigned n_sections, struct journal_completion *comp)
1100+static void rw_journal(struct dm_integrity_c *ic, blk_opf_t opf,
1101+ unsigned section, unsigned n_sections,
1102+ struct journal_completion *comp)
11011103 {
11021104 unsigned sector, n_sectors;
11031105
11041106 sector = section * ic->journal_section_sectors;
11051107 n_sectors = n_sections * ic->journal_section_sectors;
11061108
1107− rw_journal_sectors(ic, op, op_flags, sector, n_sectors, comp);
1109+ rw_journal_sectors(ic, opf, sector, n_sectors, comp);
11081110 }
11091111
11101112 static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsigned commit_sections)
@@ -1129,7 +1131,7 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi · 18 unmodified lines
11291131 for (i = 0; i < commit_sections; i++)
11301132 rw_section_mac(ic, commit_start + i, true);
11311133 }
1132− rw_journal(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, commit_start,
1134+ rw_journal(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, commit_start,
11331135 commit_sections, &io_comp);
11341136 } else {
11351137 unsigned to_end;
@@ -1141,7 +1143,8 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi · 5 unmodified lines
11411143 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
11421144 encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
11431145 if (try_wait_for_completion(&crypt_comp_1.comp)) {
1144− rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
1146+ rw_journal(ic, REQ_OP_WRITE | REQ_FUA,
1147+ commit_start, to_end, &io_comp);
11451148 reinit_completion(&crypt_comp_1.comp);
11461149 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
11471150 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
@@ -1152,17 +1155,17 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi · 4 unmodified lines
11521155 crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
11531156 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
11541157 wait_for_completion_io(&crypt_comp_1.comp);
1155− rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
1158+ rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp);
11561159 wait_for_completion_io(&crypt_comp_2.comp);
11571160 }
11581161 } else {
11591162 for (i = 0; i < to_end; i++)
11601163 rw_section_mac(ic, commit_start + i, true);
1161− rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
1164+ rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp);
11621165 for (i = 0; i < commit_sections - to_end; i++)
11631166 rw_section_mac(ic, i, true);
11641167 }
1165− rw_journal(ic, REQ_OP_WRITE, REQ_FUA, 0, commit_sections - to_end, &io_comp);
1168+ rw_journal(ic, REQ_OP_WRITE | REQ_FUA, 0, commit_sections - to_end, &io_comp);
11661169 }
11671170
11681171 wait_for_completion_io(&io_comp.comp);
@@ -1188,8 +1191,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsig · 19 unmodified lines
11881191 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
11891192 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
11901193
1191− io_req.bi_op = REQ_OP_WRITE;
1192− io_req.bi_op_flags = 0;
1194+ io_req.bi_opf = REQ_OP_WRITE;
11931195 io_req.mem.type = DM_IO_PAGE_LIST;
11941196 io_req.mem.ptr.pl = &ic->journal[pl_index];
11951197 io_req.mem.offset = pl_offset;
@@ -1516,8 +1518,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat · 320 unmodified lines
15161518 if (!ic->meta_dev)
15171519 flush_data = false;
15181520 if (flush_data) {
1519− fr.io_req.bi_op = REQ_OP_WRITE,
1520− fr.io_req.bi_op_flags = REQ_PREFLUSH | REQ_SYNC,
1521+ fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC,
15211522 fr.io_req.mem.type = DM_IO_KMEM,
15221523 fr.io_req.mem.ptr.addr = NULL,
15231524 fr.io_req.notify.fn = flush_notify,
@@ -2626,7 +2627,7 @@ static void recalc_write_super(struct dm_integrity_c *ic) · 1102 unmodified lines
26262627 if (dm_integrity_failed(ic))
26272628 return;
26282629
2629− r = sync_rw_sb(ic, REQ_OP_WRITE, 0);
2630+ r = sync_rw_sb(ic, REQ_OP_WRITE);
26302631 if (unlikely(r))
26312632 dm_integrity_io_error(ic, "writing superblock", r);
26322633 }
@@ -2706,8 +2707,7 @@ next_chunk: · 73 unmodified lines
27062707 if (unlikely(dm_integrity_failed(ic)))
27072708 goto err;
27082709
2709− io_req.bi_op = REQ_OP_READ;
2710− io_req.bi_op_flags = 0;
2710+ io_req.bi_opf = REQ_OP_READ;
27112711 io_req.mem.type = DM_IO_VMA;
27122712 io_req.mem.ptr.addr = ic->recalc_buffer;
27132713 io_req.notify.fn = NULL;
@@ -2800,7 +2800,7 @@ static void bitmap_block_work(struct work_struct *w) · 86 unmodified lines
28002800 if (bio_list_empty(&waiting))
28012801 return;
28022802
2803− rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC,
2803+ rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC,
28042804 bbs->idx * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT),
28052805 BITMAP_BLOCK_SIZE >> SECTOR_SHIFT, NULL);
28062806
@@ -2846,7 +2846,7 @@ static void bitmap_flush_work(struct work_struct *work) · 39 unmodified lines
28462846 block_bitmap_op(ic, ic->journal, 0, limit, BITMAP_OP_CLEAR);
28472847 block_bitmap_op(ic, ic->may_write_bitmap, 0, limit, BITMAP_OP_CLEAR);
28482848
2849− rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, 0,
2849+ rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
28502850 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
28512851
28522852 spin_lock_irq(&ic->endio_wait.lock);
@@ -2918,7 +2918,7 @@ static void replay_journal(struct dm_integrity_c *ic) · 65 unmodified lines
29182918
29192919 if (!ic->just_formatted) {
29202920 DEBUG_print("reading journal\n");
2921− rw_journal(ic, REQ_OP_READ, 0, 0, ic->journal_sections, NULL);
2921+ rw_journal(ic, REQ_OP_READ, 0, ic->journal_sections, NULL);
29222922 if (ic->journal_io)
29232923 DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal");
29242924 if (ic->journal_io) {
@@ -3113,7 +3113,7 @@ static void dm_integrity_postsuspend(struct dm_target *ti) · 188 unmodified lines
31133113 /* set to 0 to test bitmap replay code */
31143114 init_journal(ic, 0, ic->journal_sections, 0);
31153115 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3116− r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA);
3116+ r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
31173117 if (unlikely(r))
31183118 dm_integrity_io_error(ic, "writing superblock", r);
31193119 #endif
@@ -3136,23 +3136,23 @@ static void dm_integrity_resume(struct dm_target *ti) · 16 unmodified lines
31363136 if (ic->provided_data_sectors > old_provided_data_sectors &&
31373137 ic->mode == 'B' &&
31383138 ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) {
3139− rw_journal_sectors(ic, REQ_OP_READ, 0, 0,
3139+ rw_journal_sectors(ic, REQ_OP_READ, 0,
31403140 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
31413141 block_bitmap_op(ic, ic->journal, old_provided_data_sectors,
31423142 ic->provided_data_sectors - old_provided_data_sectors, BITMAP_OP_SET);
3143− rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, 0,
3143+ rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
31443144 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
31453145 }
31463146
31473147 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
3148− r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA);
3148+ r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
31493149 if (unlikely(r))
31503150 dm_integrity_io_error(ic, "writing superblock", r);
31513151 }
31523152
31533153 if (ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) {
31543154 DEBUG_print("resume dirty_bitmap\n");
3155− rw_journal_sectors(ic, REQ_OP_READ, 0, 0,
3155+ rw_journal_sectors(ic, REQ_OP_READ, 0,
31563156 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
31573157 if (ic->mode == 'B') {
31583158 if (ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit &&
@@ -3171,7 +3171,7 @@ static void dm_integrity_resume(struct dm_target *ti) · 12 unmodified lines
31713171 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET);
31723172 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET);
31733173 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3174− rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, 0,
3174+ rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
31753175 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
31763176 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
31773177 ic->sb->recalc_sector = cpu_to_le64(0);
@@ -3187,7 +3187,7 @@ static void dm_integrity_resume(struct dm_target *ti) · 9 unmodified lines
31873187 replay_journal(ic);
31883188 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
31893189 }
3190− r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA);
3190+ r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
31913191 if (unlikely(r))
31923192 dm_integrity_io_error(ic, "writing superblock", r);
31933193 } else {
@@ -3199,7 +3199,7 @@ static void dm_integrity_resume(struct dm_target *ti) · 5 unmodified lines
31993199 if (ic->mode == 'B') {
32003200 ic->sb->flags |= cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
32013201 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit;
3202− r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA);
3202+ r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
32033203 if (unlikely(r))
32043204 dm_integrity_io_error(ic, "writing superblock", r);
32053205
@@ -3215,7 +3215,7 @@ static void dm_integrity_resume(struct dm_target *ti) · 9 unmodified lines
32153215 block_bitmap_op(ic, ic->may_write_bitmap, le64_to_cpu(ic->sb->recalc_sector),
32163216 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
32173217 }
3218− rw_journal_sectors(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, 0,
3218+ rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
32193219 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
32203220 }
32213221 }
@@ -4256,7 +4256,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) · 1034 unmodified lines
42564256 goto bad;
42574257 }
42584258
4259− r = sync_rw_sb(ic, REQ_OP_READ, 0);
4259+ r = sync_rw_sb(ic, REQ_OP_READ);
42604260 if (r) {
42614261 ti->error = "Error reading superblock";
42624262 goto bad;
@@ -4500,7 +4500,7 @@ try_smaller_buffer: · 237 unmodified lines
45004500 ti->error = "Error initializing journal";
45014501 goto bad;
45024502 }
4503− r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA);
4503+ r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
45044504 if (r) {
45054505 ti->error = "Error initializing superblock";
45064506 goto bad;
drivers/md/dm-io.c+19−19
@@ -293,7 +293,7 @@ static void km_dp_init(struct dpages *dp, void *data) · 292 unmodified lines
293293 /*-----------------------------------------------------------------
294294 * IO routines that accept a list of pages.
295295 *---------------------------------------------------------------*/
296−static void do_region(int op, int op_flags, unsigned region,
296+static void do_region(const blk_opf_t opf, unsigned region,
297297 struct dm_io_region *where, struct dpages *dp,
298298 struct io *io)
299299 {
@@ -306,6 +306,7 @@ static void do_region(int op, int op_flags, unsigned region, · 6 unmodified lines
306306 struct request_queue *q = bdev_get_queue(where->bdev);
307307 sector_t num_sectors;
308308 unsigned int special_cmd_max_sectors;
309+ const enum req_op op = opf & REQ_OP_MASK;
309310
310311 /*
311312 * Reject unsupported discard and write same requests.
@@ -339,8 +340,8 @@ static void do_region(int op, int op_flags, unsigned region, · 27 unmodified lines
339340 (PAGE_SIZE >> SECTOR_SHIFT)));
340341 }
341342
342− bio = bio_alloc_bioset(where->bdev, num_bvecs, op | op_flags,
343− GFP_NOIO, &io->client->bios);
343+ bio = bio_alloc_bioset(where->bdev, num_bvecs, opf, GFP_NOIO,
344+ &io->client->bios);
344345 bio->bi_iter.bi_sector = where->sector + (where->count - remaining);
345346 bio->bi_end_io = endio;
346347 store_io_and_region_in_bio(bio, io, region);
@@ -368,7 +369,7 @@ static void do_region(int op, int op_flags, unsigned region, · 21 unmodified lines
368369 } while (remaining);
369370 }
370371
371−static void dispatch_io(int op, int op_flags, unsigned int num_regions,
372+static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
372373 struct dm_io_region *where, struct dpages *dp,
373374 struct io *io, int sync)
374375 {
@@ -378,7 +379,7 @@ static void dispatch_io(int op, int op_flags, unsigned int num_regions, · 3 unmodified lines
378379 BUG_ON(num_regions > DM_IO_MAX_REGIONS);
379380
380381 if (sync)
381− op_flags |= REQ_SYNC;
382+ opf |= REQ_SYNC;
382383
383384 /*
384385 * For multiple regions we need to be careful to rewind
@@ -386,8 +387,8 @@ static void dispatch_io(int op, int op_flags, unsigned int num_regions, · 1 unmodified line
386387 */
387388 for (i = 0; i < num_regions; i++) {
388389 *dp = old_pages;
389− if (where[i].count || (op_flags & REQ_PREFLUSH))
390− do_region(op, op_flags, i, where + i, dp, io);
390+ if (where[i].count || (opf & REQ_PREFLUSH))
391+ do_region(opf, i, where + i, dp, io);
391392 }
392393
393394 /*
@@ -411,13 +412,13 @@ static void sync_io_complete(unsigned long error, void *context) · 17 unmodified lines
411412 }
412413
413414 static int sync_io(struct dm_io_client *client, unsigned int num_regions,
414− struct dm_io_region *where, int op, int op_flags,
415− struct dpages *dp, unsigned long *error_bits)
415+ struct dm_io_region *where, blk_opf_t opf, struct dpages *dp,
416+ unsigned long *error_bits)
416417 {
417418 struct io *io;
418419 struct sync_io sio;
419420
420− if (num_regions > 1 && !op_is_write(op)) {
421+ if (num_regions > 1 && !op_is_write(opf)) {
421422 WARN_ON(1);
422423 return -EIO;
423424 }
@@ -434,7 +435,7 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, · 10 unmodified lines
434435 io->vma_invalidate_address = dp->vma_invalidate_address;
435436 io->vma_invalidate_size = dp->vma_invalidate_size;
436437
437− dispatch_io(op, op_flags, num_regions, where, dp, io, 1);
438+ dispatch_io(opf, num_regions, where, dp, io, 1);
438439
439440 wait_for_completion_io(&sio.wait);
440441
@@ -445,12 +446,12 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, · 4 unmodified lines
445446 }
446447
447448 static int async_io(struct dm_io_client *client, unsigned int num_regions,
448− struct dm_io_region *where, int op, int op_flags,
449+ struct dm_io_region *where, blk_opf_t opf,
449450 struct dpages *dp, io_notify_fn fn, void *context)
450451 {
451452 struct io *io;
452453
453− if (num_regions > 1 && !op_is_write(op)) {
454+ if (num_regions > 1 && !op_is_write(opf)) {
454455 WARN_ON(1);
455456 fn(1, context);
456457 return -EIO;
@@ -466,7 +467,7 @@ static int async_io(struct dm_io_client *client, unsigned int num_regions, · 9 unmodified lines
466467 io->vma_invalidate_address = dp->vma_invalidate_address;
467468 io->vma_invalidate_size = dp->vma_invalidate_size;
468469
469− dispatch_io(op, op_flags, num_regions, where, dp, io, 0);
470+ dispatch_io(opf, num_regions, where, dp, io, 0);
470471 return 0;
471472 }
472473
@@ -489,7 +490,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, · 16 unmodified lines
489490
490491 case DM_IO_VMA:
491492 flush_kernel_vmap_range(io_req->mem.ptr.vma, size);
492− if (io_req->bi_op == REQ_OP_READ) {
493+ if ((io_req->bi_opf & REQ_OP_MASK) == REQ_OP_READ) {
493494 dp->vma_invalidate_address = io_req->mem.ptr.vma;
494495 dp->vma_invalidate_size = size;
495496 }
@@ -519,11 +520,10 @@ int dm_io(struct dm_io_request *io_req, unsigned num_regions, · 23 unmodified lines
519520
520521 if (!io_req->notify.fn)
521522 return sync_io(io_req->client, num_regions, where,
522− io_req->bi_op, io_req->bi_op_flags, &dp,
523− sync_error_bits);
523+ io_req->bi_opf, &dp, sync_error_bits);
524524
525− return async_io(io_req->client, num_regions, where, io_req->bi_op,
526− io_req->bi_op_flags, &dp, io_req->notify.fn,
525+ return async_io(io_req->client, num_regions, where,
526+ io_req->bi_opf, &dp, io_req->notify.fn,
527527 io_req->notify.context);
528528 }
529529 EXPORT_SYMBOL(dm_io);
drivers/md/dm-kcopyd.c+13−13
@@ -350,9 +350,9 @@ struct kcopyd_job { · 349 unmodified lines
350350 unsigned long write_err;
351351
352352 /*
353− * Either READ or WRITE
353+ * REQ_OP_READ, REQ_OP_WRITE or REQ_OP_WRITE_ZEROES.
354354 */
355− int rw;
355+ enum req_op op;
356356 struct dm_io_region source;
357357
358358 /*
@@ -418,7 +418,8 @@ static struct kcopyd_job *pop_io_job(struct list_head *jobs, · 59 unmodified lines
418418 * constraint and sequential writes that are at the right position.
419419 */
420420 list_for_each_entry(job, jobs, list) {
421− if (job->rw == READ || !(job->flags & BIT(DM_KCOPYD_WRITE_SEQ))) {
421+ if (job->op == REQ_OP_READ ||
422+ !(job->flags & BIT(DM_KCOPYD_WRITE_SEQ))) {
422423 list_del(&job->list);
423424 return job;
424425 }
@@ -518,7 +519,7 @@ static void complete_io(unsigned long error, void *context) · 93 unmodified lines
518519 io_job_finish(kc->throttle);
519520
520521 if (error) {
521− if (op_is_write(job->rw))
522+ if (op_is_write(job->op))
522523 job->write_err |= error;
523524 else
524525 job->read_err = 1;
@@ -530,11 +531,11 @@ static void complete_io(unsigned long error, void *context) · 5 unmodified lines
530531 }
531532 }
532533
533− if (op_is_write(job->rw))
534+ if (op_is_write(job->op))
534535 push(&kc->complete_jobs, job);
535536
536537 else {
537− job->rw = WRITE;
538+ job->op = REQ_OP_WRITE;
538539 push(&kc->io_jobs, job);
539540 }
540541
@@ -549,8 +550,7 @@ static int run_io_job(struct kcopyd_job *job) · 8 unmodified lines
549550 {
550551 int r;
551552 struct dm_io_request io_req = {
552− .bi_op = job->rw,
553− .bi_op_flags = 0,
553+ .bi_opf = job->op,
554554 .mem.type = DM_IO_PAGE_LIST,
555555 .mem.ptr.pl = job->pages,
556556 .mem.offset = 0,
@@ -571,7 +571,7 @@ static int run_io_job(struct kcopyd_job *job) · 14 unmodified lines
571571
572572 io_job_start(job->kc->throttle);
573573
574− if (job->rw == READ)
574+ if (job->op == REQ_OP_READ)
575575 r = dm_io(&io_req, 1, &job->source, NULL);
576576 else
577577 r = dm_io(&io_req, job->num_dests, job->dests, NULL);
@@ -614,7 +614,7 @@ static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc, · 36 unmodified lines
614614
615615 if (r < 0) {
616616 /* error this rogue job */
617− if (op_is_write(job->rw))
617+ if (op_is_write(job->op))
618618 job->write_err = (unsigned long) -1L;
619619 else
620620 job->read_err = 1;
@@ -817,7 +817,7 @@ void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, · 196 unmodified lines
817817 if (from) {
818818 job->source = *from;
819819 job->pages = NULL;
820− job->rw = READ;
820+ job->op = REQ_OP_READ;
821821 } else {
822822 memset(&job->source, 0, sizeof job->source);
823823 job->source.count = job->dests[0].count;
@@ -826,10 +826,10 @@ void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, · 2 unmodified lines
826826 /*
827827 * Use WRITE ZEROES to optimize zeroing if all dests support it.
828828 */
829− job->rw = REQ_OP_WRITE_ZEROES;
829+ job->op = REQ_OP_WRITE_ZEROES;
830830 for (i = 0; i < job->num_dests; i++)
831831 if (!bdev_write_zeroes_sectors(job->dests[i].bdev)) {
832− job->rw = WRITE;
832+ job->op = REQ_OP_WRITE;
833833 break;
834834 }
835835 }
drivers/md/dm-log.c+3−5
@@ -291,10 +291,9 @@ static void header_from_disk(struct log_header_core *core, struct log_header_dis · 290 unmodified lines
291291 core->nr_regions = le64_to_cpu(disk->nr_regions);
292292 }
293293
294−static int rw_header(struct log_c *lc, int op)
294+static int rw_header(struct log_c *lc, enum req_op op)
295295 {
296− lc->io_req.bi_op = op;
297− lc->io_req.bi_op_flags = 0;
296+ lc->io_req.bi_opf = op;
298297
299298 return dm_io(&lc->io_req, 1, &lc->header_location, NULL);
300299 }
@@ -307,8 +306,7 @@ static int flush_header(struct log_c *lc) · 6 unmodified lines
307306 .count = 0,
308307 };
309308
310− lc->io_req.bi_op = REQ_OP_WRITE;
311− lc->io_req.bi_op_flags = REQ_PREFLUSH;
309+ lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
312310
313311 return dm_io(&lc->io_req, 1, &null_location, NULL);
314312 }
drivers/md/dm-raid.c+1−1
@@ -2038,7 +2038,7 @@ static int read_disk_sb(struct md_rdev *rdev, int size, bool force_reload) · 2037 unmodified lines
20382038
20392039 rdev->sb_loaded = 0;
20402040
2041− if (!sync_page_io(rdev, 0, size, rdev->sb_page, REQ_OP_READ, 0, true)) {
2041+ if (!sync_page_io(rdev, 0, size, rdev->sb_page, REQ_OP_READ, true)) {
20422042 DMERR("Failed to read superblock of device at position %d",
20432043 rdev->raid_disk);
20442044 md_error(rdev->mddev, rdev);
drivers/md/dm-raid1.c+5−7
@@ -260,8 +260,7 @@ static int mirror_flush(struct dm_target *ti) · 259 unmodified lines
260260 struct dm_io_region io[MAX_NR_MIRRORS];
261261 struct mirror *m;
262262 struct dm_io_request io_req = {
263− .bi_op = REQ_OP_WRITE,
264− .bi_op_flags = REQ_PREFLUSH | REQ_SYNC,
263+ .bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC,
265264 .mem.type = DM_IO_KMEM,
266265 .mem.ptr.addr = NULL,
267266 .client = ms->io_client,
@@ -535,8 +534,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio) · 267 unmodified lines
535534 {
536535 struct dm_io_region io;
537536 struct dm_io_request io_req = {
538− .bi_op = REQ_OP_READ,
539− .bi_op_flags = 0,
537+ .bi_opf = REQ_OP_READ,
540538 .mem.type = DM_IO_BIO,
541539 .mem.ptr.bio = bio,
542540 .notify.fn = read_callback,
@@ -648,9 +646,9 @@ static void do_write(struct mirror_set *ms, struct bio *bio) · 105 unmodified lines
648646 unsigned int i;
649647 struct dm_io_region io[MAX_NR_MIRRORS], *dest = io;
650648 struct mirror *m;
649+ blk_opf_t op_flags = bio->bi_opf & (REQ_FUA | REQ_PREFLUSH);
651650 struct dm_io_request io_req = {
652− .bi_op = REQ_OP_WRITE,
653− .bi_op_flags = bio->bi_opf & (REQ_FUA | REQ_PREFLUSH),
651+ .bi_opf = REQ_OP_WRITE | op_flags,
654652 .mem.type = DM_IO_BIO,
655653 .mem.ptr.bio = bio,
656654 .notify.fn = write_callback,
@@ -659,7 +657,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio) · 2 unmodified lines
659657 };
660658
661659 if (bio_op(bio) == REQ_OP_DISCARD) {
662− io_req.bi_op = REQ_OP_DISCARD;
660+ io_req.bi_opf = REQ_OP_DISCARD | op_flags;
663661 io_req.mem.type = DM_IO_KMEM;
664662 io_req.mem.ptr.addr = NULL;
665663 }
drivers/md/dm-snap-persistent.c+12−13
@@ -226,8 +226,8 @@ static void do_metadata(struct work_struct *work) · 225 unmodified lines
226226 /*
227227 * Read or write a chunk aligned and sized block of data from a device.
228228 */
229−static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, int op,
230− int op_flags, int metadata)
229+static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf,
230+ int metadata)
231231 {
232232 struct dm_io_region where = {
233233 .bdev = dm_snap_cow(ps->store->snap)->bdev,
@@ -235,8 +235,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, int op, · 1 unmodified line
235235 .count = ps->store->chunk_size,
236236 };
237237 struct dm_io_request io_req = {
238− .bi_op = op,
239− .bi_op_flags = op_flags,
238+ .bi_opf = opf,
240239 .mem.type = DM_IO_VMA,
241240 .mem.ptr.vma = area,
242241 .client = ps->io_client,
@@ -282,11 +281,11 @@ static void skip_metadata(struct pstore *ps) · 39 unmodified lines
282281 * Read or write a metadata area. Remembering to skip the first
283282 * chunk which holds the header.
284283 */
285−static int area_io(struct pstore *ps, int op, int op_flags)
284+static int area_io(struct pstore *ps, blk_opf_t opf)
286285 {
287286 chunk_t chunk = area_location(ps, ps->current_area);
288287
289− return chunk_io(ps, ps->area, chunk, op, op_flags, 0);
288+ return chunk_io(ps, ps->area, chunk, opf, 0);
290289 }
291290
292291 static void zero_memory_area(struct pstore *ps)
@@ -297,7 +296,7 @@ static void zero_memory_area(struct pstore *ps) · 4 unmodified lines
297296 static int zero_disk_area(struct pstore *ps, chunk_t area)
298297 {
299298 return chunk_io(ps, ps->zero_area, area_location(ps, area),
300− REQ_OP_WRITE, 0, 0);
299+ REQ_OP_WRITE, 0);
301300 }
302301
303302 static int read_header(struct pstore *ps, int *new_snapshot)
@@ -329,7 +328,7 @@ static int read_header(struct pstore *ps, int *new_snapshot) · 25 unmodified lines
329328 if (r)
330329 return r;
331330
332− r = chunk_io(ps, ps->header_area, 0, REQ_OP_READ, 0, 1);
331+ r = chunk_io(ps, ps->header_area, 0, REQ_OP_READ, 1);
333332 if (r)
334333 goto bad;
335334
@@ -390,7 +389,7 @@ static int write_header(struct pstore *ps) · 54 unmodified lines
390389 dh->version = cpu_to_le32(ps->version);
391390 dh->chunk_size = cpu_to_le32(ps->store->chunk_size);
392391
393− return chunk_io(ps, ps->header_area, 0, REQ_OP_WRITE, 0, 1);
392+ return chunk_io(ps, ps->header_area, 0, REQ_OP_WRITE, 1);
394393 }
395394
396395 /*
@@ -734,8 +733,8 @@ static void persistent_commit_exception(struct dm_exception_store *store, · 337 unmodified lines
734733 /*
735734 * Commit exceptions to disk.
736735 */
737− if (ps->valid && area_io(ps, REQ_OP_WRITE,
738− REQ_PREFLUSH | REQ_FUA | REQ_SYNC))
736+ if (ps->valid && area_io(ps, REQ_OP_WRITE | REQ_PREFLUSH | REQ_FUA |
737+ REQ_SYNC))
739738 ps->valid = 0;
740739
741740 /*
@@ -775,7 +774,7 @@ static int persistent_prepare_merge(struct dm_exception_store *store, · 33 unmodified lines
775774 return 0;
776775
777776 ps->current_area--;
778− r = area_io(ps, REQ_OP_READ, 0);
777+ r = area_io(ps, REQ_OP_READ);
779778 if (r < 0)
780779 return r;
781780 ps->current_committed = ps->exceptions_per_area;
@@ -812,7 +811,7 @@ static int persistent_commit_merge(struct dm_exception_store *store, · 30 unmodified lines
812811 for (i = 0; i < nr_merged; i++)
813812 clear_exception(ps, ps->current_committed - 1 - i);
814813
815− r = area_io(ps, REQ_OP_WRITE, REQ_PREFLUSH | REQ_FUA);
814+ r = area_io(ps, REQ_OP_WRITE | REQ_PREFLUSH | REQ_FUA);
816815 if (r < 0)
817816 return r;
818817
drivers/md/dm-table.c+2−4
@@ -1620,13 +1620,11 @@ static bool dm_table_supports_zoned_model(struct dm_table *t, · 1619 unmodified lines
16201620 static int device_not_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev,
16211621 sector_t start, sector_t len, void *data)
16221622 {
1623− struct request_queue *q = bdev_get_queue(dev->bdev);
16241623 unsigned int *zone_sectors = data;
16251624
1626− if (!blk_queue_is_zoned(q))
1625+ if (!bdev_is_zoned(dev->bdev))
16271626 return 0;
1628−
1629− return blk_queue_zone_sectors(q) != *zone_sectors;
1627+ return bdev_zone_sectors(dev->bdev) != *zone_sectors;
16301628 }
16311629
16321630 /*
drivers/md/dm-writecache.c+4−8
@@ -523,8 +523,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) · 522 unmodified lines
523523
524524 region.sector += wc->start_sector;
525525 atomic_inc(&endio.count);
526− req.bi_op = REQ_OP_WRITE;
527− req.bi_op_flags = REQ_SYNC;
526+ req.bi_opf = REQ_OP_WRITE | REQ_SYNC;
528527 req.mem.type = DM_IO_VMA;
529528 req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;
530529 req.client = wc->dm_io;
@@ -562,8 +561,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc) · 31 unmodified lines
562561
563562 region.sector += wc->start_sector;
564563
565− req.bi_op = REQ_OP_WRITE;
566− req.bi_op_flags = REQ_SYNC | REQ_FUA;
564+ req.bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_FUA;
567565 req.mem.type = DM_IO_VMA;
568566 req.mem.ptr.vma = (char *)wc->memory_map;
569567 req.client = wc->dm_io;
@@ -592,8 +590,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) · 22 unmodified lines
592590 region.bdev = dev->bdev;
593591 region.sector = 0;
594592 region.count = 0;
595− req.bi_op = REQ_OP_WRITE;
596− req.bi_op_flags = REQ_PREFLUSH;
593+ req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
597594 req.mem.type = DM_IO_KMEM;
598595 req.mem.ptr.addr = NULL;
599596 req.client = wc->dm_io;
@@ -981,8 +978,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors · 381 unmodified lines
981978 region.bdev = wc->ssd_dev->bdev;
982979 region.sector = wc->start_sector;
983980 region.count = n_sectors;
984− req.bi_op = REQ_OP_READ;
985− req.bi_op_flags = REQ_SYNC;
981+ req.bi_opf = REQ_OP_READ | REQ_SYNC;
986982 req.mem.type = DM_IO_VMA;
987983 req.mem.ptr.vma = (char *)wc->memory_map;
988984 req.client = wc->dm_io;
drivers/md/dm-zone.c+43−45
@@ -139,13 +139,11 @@ bool dm_is_zone_write(struct mapped_device *md, struct bio *bio) · 138 unmodified lines
139139
140140 void dm_cleanup_zoned_dev(struct mapped_device *md)
141141 {
142− struct request_queue *q = md->queue;
143−
144− if (q) {
145− kfree(q->conv_zones_bitmap);
146− q->conv_zones_bitmap = NULL;
147− kfree(q->seq_zones_wlock);
148− q->seq_zones_wlock = NULL;
142+ if (md->disk) {
143+ kfree(md->disk->conv_zones_bitmap);
144+ md->disk->conv_zones_bitmap = NULL;
145+ kfree(md->disk->seq_zones_wlock);
146+ md->disk->seq_zones_wlock = NULL;
149147 }
150148
151149 kvfree(md->zwp_offset);
@@ -179,31 +177,31 @@ static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx, · 27 unmodified lines
179177 void *data)
180178 {
181179 struct mapped_device *md = data;
182− struct request_queue *q = md->queue;
180+ struct gendisk *disk = md->disk;
183181
184182 switch (zone->type) {
185183 case BLK_ZONE_TYPE_CONVENTIONAL:
186− if (!q->conv_zones_bitmap) {
187− q->conv_zones_bitmap =
188− kcalloc(BITS_TO_LONGS(q->nr_zones),
184+ if (!disk->conv_zones_bitmap) {
185+ disk->conv_zones_bitmap =
186+ kcalloc(BITS_TO_LONGS(disk->nr_zones),
189187 sizeof(unsigned long), GFP_NOIO);
190− if (!q->conv_zones_bitmap)
188+ if (!disk->conv_zones_bitmap)
191189 return -ENOMEM;
192190 }
193− set_bit(idx, q->conv_zones_bitmap);
191+ set_bit(idx, disk->conv_zones_bitmap);
194192 break;
195193 case BLK_ZONE_TYPE_SEQWRITE_REQ:
196194 case BLK_ZONE_TYPE_SEQWRITE_PREF:
197− if (!q->seq_zones_wlock) {
198− q->seq_zones_wlock =
199− kcalloc(BITS_TO_LONGS(q->nr_zones),
195+ if (!disk->seq_zones_wlock) {
196+ disk->seq_zones_wlock =
197+ kcalloc(BITS_TO_LONGS(disk->nr_zones),
200198 sizeof(unsigned long), GFP_NOIO);
201− if (!q->seq_zones_wlock)
199+ if (!disk->seq_zones_wlock)
202200 return -ENOMEM;
203201 }
204202 if (!md->zwp_offset) {
205203 md->zwp_offset =
206− kvcalloc(q->nr_zones, sizeof(unsigned int),
204+ kvcalloc(disk->nr_zones, sizeof(unsigned int),
207205 GFP_KERNEL);
208206 if (!md->zwp_offset)
209207 return -ENOMEM;
@@ -228,7 +226,7 @@ static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx, · 18 unmodified lines
228226 */
229227 static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t)
230228 {
231− struct request_queue *q = md->queue;
229+ struct gendisk *disk = md->disk;
232230 unsigned int noio_flag;
233231 int ret;
234232
@@ -236,7 +234,7 @@ static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t) · 1 unmodified line
236234 * Check if something changed. If yes, cleanup the current resources
237235 * and reallocate everything.
238236 */
239− if (!q->nr_zones || q->nr_zones != md->nr_zones)
237+ if (!disk->nr_zones || disk->nr_zones != md->nr_zones)
240238 dm_cleanup_zoned_dev(md);
241239 if (md->nr_zones)
242240 return 0;
@@ -246,17 +244,17 @@ static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t) · 3 unmodified lines
246244 * operations in this context are done as if GFP_NOIO was specified.
247245 */
248246 noio_flag = memalloc_noio_save();
249− ret = dm_blk_do_report_zones(md, t, 0, q->nr_zones,
247+ ret = dm_blk_do_report_zones(md, t, 0, disk->nr_zones,
250248 dm_zone_revalidate_cb, md);
251249 memalloc_noio_restore(noio_flag);
252250 if (ret < 0)
253251 goto err;
254− if (ret != q->nr_zones) {
252+ if (ret != disk->nr_zones) {
255253 ret = -EIO;
256254 goto err;
257255 }
258256
259− md->nr_zones = q->nr_zones;
257+ md->nr_zones = disk->nr_zones;
260258
261259 return 0;
262260
@@ -270,7 +268,7 @@ static int device_not_zone_append_capable(struct dm_target *ti, · 7 unmodified lines
270268 struct dm_dev *dev, sector_t start,
271269 sector_t len, void *data)
272270 {
273− return !blk_queue_is_zoned(bdev_get_queue(dev->bdev));
271+ return !bdev_is_zoned(dev->bdev);
274272 }
275273
276274 static bool dm_table_supports_zone_append(struct dm_table *t)
@@ -301,7 +299,7 @@ int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q) · 24 unmodified lines
301299 * correct value to be exposed in sysfs queue/nr_zones.
302300 */
303301 WARN_ON_ONCE(queue_is_mq(q));
304− q->nr_zones = blkdev_nr_zones(md->disk);
302+ md->disk->nr_zones = bdev_nr_zones(md->disk->part0);
305303
306304 /* Check if zone append is natively supported */
307305 if (dm_table_supports_zone_append(t)) {
@@ -334,7 +332,7 @@ static int dm_update_zone_wp_offset_cb(struct blk_zone *zone, unsigned int idx, · 26 unmodified lines
334332 static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno,
335333 unsigned int *wp_ofst)
336334 {
337− sector_t sector = zno * blk_queue_zone_sectors(md->queue);
335+ sector_t sector = zno * bdev_zone_sectors(md->disk->part0);
338336 unsigned int noio_flag;
339337 struct dm_table *t;
340338 int srcu_idx, ret;
@@ -361,7 +359,7 @@ static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno, · 20 unmodified lines
361359 }
362360
363361 struct orig_bio_details {
364− unsigned int op;
362+ enum req_op op;
365363 unsigned int nr_sectors;
366364 };
367365
@@ -373,7 +371,7 @@ struct orig_bio_details { · 5 unmodified lines
373371 static bool dm_zone_map_bio_begin(struct mapped_device *md,
374372 unsigned int zno, struct bio *clone)
375373 {
376− sector_t zsectors = blk_queue_zone_sectors(md->queue);
374+ sector_t zsectors = bdev_zone_sectors(md->disk->part0);
377375 unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);
378376
379377 /*
@@ -443,7 +441,7 @@ static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int z · 63 unmodified lines
443441 return BLK_STS_OK;
444442 case REQ_OP_ZONE_FINISH:
445443 WRITE_ONCE(md->zwp_offset[zno],
446− blk_queue_zone_sectors(md->queue));
444+ bdev_zone_sectors(md->disk->part0));
447445 return BLK_STS_OK;
448446 case REQ_OP_WRITE_ZEROES:
449447 case REQ_OP_WRITE:
@@ -466,26 +464,26 @@ static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int z · 16 unmodified lines
466464 }
467465 }
468466
469−static inline void dm_zone_lock(struct request_queue *q,
470− unsigned int zno, struct bio *clone)
467+static inline void dm_zone_lock(struct gendisk *disk, unsigned int zno,
468+ struct bio *clone)
471469 {
472470 if (WARN_ON_ONCE(bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)))
473471 return;
474472
475− wait_on_bit_lock_io(q->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE);
473+ wait_on_bit_lock_io(disk->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE);
476474 bio_set_flag(clone, BIO_ZONE_WRITE_LOCKED);
477475 }
478476
479−static inline void dm_zone_unlock(struct request_queue *q,
480− unsigned int zno, struct bio *clone)
477+static inline void dm_zone_unlock(struct gendisk *disk, unsigned int zno,
478+ struct bio *clone)
481479 {
482480 if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
483481 return;
484482
485− WARN_ON_ONCE(!test_bit(zno, q->seq_zones_wlock));
486− clear_bit_unlock(zno, q->seq_zones_wlock);
483+ WARN_ON_ONCE(!test_bit(zno, disk->seq_zones_wlock));
484+ clear_bit_unlock(zno, disk->seq_zones_wlock);
487485 smp_mb__after_atomic();
488− wake_up_bit(q->seq_zones_wlock, zno);
486+ wake_up_bit(disk->seq_zones_wlock, zno);
489487
490488 bio_clear_flag(clone, BIO_ZONE_WRITE_LOCKED);
491489 }
@@ -520,7 +518,6 @@ int dm_zone_map_bio(struct dm_target_io *tio) · 28 unmodified lines
520518 struct dm_io *io = tio->io;
521519 struct dm_target *ti = tio->ti;
522520 struct mapped_device *md = io->md;
523− struct request_queue *q = md->queue;
524521 struct bio *clone = &tio->clone;
525522 struct orig_bio_details orig_bio_details;
526523 unsigned int zno;
@@ -536,7 +533,7 @@ int dm_zone_map_bio(struct dm_target_io *tio) · 9 unmodified lines
536533
537534 /* Lock the target zone */
538535 zno = bio_zone_no(clone);
539− dm_zone_lock(q, zno, clone);
536+ dm_zone_lock(md->disk, zno, clone);
540537
541538 orig_bio_details.nr_sectors = bio_sectors(clone);
542539 orig_bio_details.op = bio_op(clone);
@@ -546,7 +543,7 @@ int dm_zone_map_bio(struct dm_target_io *tio) · 3 unmodified lines
546543 * both valid, and if the bio is a zone append, remap it to a write.
547544 */
548545 if (!dm_zone_map_bio_begin(md, zno, clone)) {
549− dm_zone_unlock(q, zno, clone);
546+ dm_zone_unlock(md->disk, zno, clone);
550547 return DM_MAPIO_KILL;
551548 }
552549
@@ -570,12 +567,12 @@ int dm_zone_map_bio(struct dm_target_io *tio) · 17 unmodified lines
570567 sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
571568 *tio->len_ptr);
572569 if (sts != BLK_STS_OK)
573− dm_zone_unlock(q, zno, clone);
570+ dm_zone_unlock(md->disk, zno, clone);
574571 break;
575572 case DM_MAPIO_REQUEUE:
576573 case DM_MAPIO_KILL:
577574 default:
578− dm_zone_unlock(q, zno, clone);
575+ dm_zone_unlock(md->disk, zno, clone);
579576 sts = BLK_STS_IOERR;
580577 break;
581578 }
@@ -592,7 +589,7 @@ int dm_zone_map_bio(struct dm_target_io *tio) · 10 unmodified lines
592589 void dm_zone_endio(struct dm_io *io, struct bio *clone)
593590 {
594591 struct mapped_device *md = io->md;
595− struct request_queue *q = md->queue;
592+ struct gendisk *disk = md->disk;
596593 struct bio *orig_bio = io->orig_bio;
597594 unsigned int zwp_offset;
598595 unsigned int zno;
@@ -608,7 +605,8 @@ void dm_zone_endio(struct dm_io *io, struct bio *clone) · 9 unmodified lines
608605 */
609606 if (clone->bi_status == BLK_STS_OK &&
610607 bio_op(clone) == REQ_OP_ZONE_APPEND) {
611− sector_t mask = (sector_t)blk_queue_zone_sectors(q) - 1;
608+ sector_t mask =
609+ (sector_t)bdev_zone_sectors(disk->part0) - 1;
612610
613611 orig_bio->bi_iter.bi_sector +=
614612 clone->bi_iter.bi_sector & mask;
@@ -649,5 +647,5 @@ void dm_zone_endio(struct dm_io *io, struct bio *clone) · 34 unmodified lines
649647 zwp_offset - bio_sectors(orig_bio);
650648 }
651649
652− dm_zone_unlock(q, zno, clone);
650+ dm_zone_unlock(disk, zno, clone);
653651 }
drivers/md/dm-zoned-metadata.c+3−2
@@ -737,7 +737,7 @@ static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk, · 736 unmodified lines
737737 /*
738738 * Read/write a metadata block.
739739 */
740−static int dmz_rdwr_block(struct dmz_dev *dev, int op,
740+static int dmz_rdwr_block(struct dmz_dev *dev, enum req_op op,
741741 sector_t block, struct page *page)
742742 {
743743 struct bio *bio;
@@ -2045,7 +2045,8 @@ struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd, · 1301 unmodified lines
20452045 * allocated and used to map the chunk.
20462046 * The zone returned will be set to the active state.
20472047 */
2048−struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk, int op)
2048+struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd,
2049+ unsigned int chunk, enum req_op op)
20492050 {
20502051 struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
20512052 struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
drivers/md/dm-zoned-target.c+12−13
@@ -764,8 +764,7 @@ static void dmz_put_zoned_device(struct dm_target *ti) · 763 unmodified lines
764764 static int dmz_fixup_devices(struct dm_target *ti)
765765 {
766766 struct dmz_target *dmz = ti->private;
767− struct dmz_dev *reg_dev, *zoned_dev;
768− struct request_queue *q;
767+ struct dmz_dev *reg_dev = NULL;
769768 sector_t zone_nr_sectors = 0;
770769 int i;
771770
@@ -780,32 +779,32 @@ static int dmz_fixup_devices(struct dm_target *ti) · 8 unmodified lines
780779 return -EINVAL;
781780 }
782781 for (i = 1; i < dmz->nr_ddevs; i++) {
783− zoned_dev = &dmz->dev[i];
782+ struct dmz_dev *zoned_dev = &dmz->dev[i];
783+ struct block_device *bdev = zoned_dev->bdev;
784+
784785 if (zoned_dev->flags & DMZ_BDEV_REGULAR) {
785786 ti->error = "Secondary disk is not a zoned device";
786787 return -EINVAL;
787788 }
788− q = bdev_get_queue(zoned_dev->bdev);
789789 if (zone_nr_sectors &&
790− zone_nr_sectors != blk_queue_zone_sectors(q)) {
790+ zone_nr_sectors != bdev_zone_sectors(bdev)) {
791791 ti->error = "Zone nr sectors mismatch";
792792 return -EINVAL;
793793 }
794− zone_nr_sectors = blk_queue_zone_sectors(q);
794+ zone_nr_sectors = bdev_zone_sectors(bdev);
795795 zoned_dev->zone_nr_sectors = zone_nr_sectors;
796− zoned_dev->nr_zones =
797− blkdev_nr_zones(zoned_dev->bdev->bd_disk);
796+ zoned_dev->nr_zones = bdev_nr_zones(bdev);
798797 }
799798 } else {
800− reg_dev = NULL;
801− zoned_dev = &dmz->dev[0];
799+ struct dmz_dev *zoned_dev = &dmz->dev[0];
800+ struct block_device *bdev = zoned_dev->bdev;
801+
802802 if (zoned_dev->flags & DMZ_BDEV_REGULAR) {
803803 ti->error = "Disk is not a zoned device";
804804 return -EINVAL;
805805 }
806− q = bdev_get_queue(zoned_dev->bdev);
807− zoned_dev->zone_nr_sectors = blk_queue_zone_sectors(q);
808− zoned_dev->nr_zones = blkdev_nr_zones(zoned_dev->bdev->bd_disk);
806+ zoned_dev->zone_nr_sectors = bdev_zone_sectors(bdev);
807+ zoned_dev->nr_zones = bdev_nr_zones(bdev);
809808 }
810809
811810 if (reg_dev) {
drivers/md/dm-zoned.h+1−1
@@ -248,7 +248,7 @@ struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd, · 247 unmodified lines
248248 unsigned int dev_idx, bool idle);
249249
250250 struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd,
251− unsigned int chunk, int op);
251+ unsigned int chunk, enum req_op op);
252252 void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *zone);
253253 struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,
254254 struct dm_zone *dzone);
drivers/md/dm.c+15−18
@@ -716,7 +716,7 @@ static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) · 715 unmodified lines
716716 }
717717
718718 static inline struct dm_table *dm_get_live_table_bio(struct mapped_device *md,
719− int *srcu_idx, unsigned bio_opf)
719+ int *srcu_idx, blk_opf_t bio_opf)
720720 {
721721 if (bio_opf & REQ_NOWAIT)
722722 return dm_get_live_table_fast(md);
@@ -725,7 +725,7 @@ static inline struct dm_table *dm_get_live_table_bio(struct mapped_device *md, · 2 unmodified lines
725725 }
726726
727727 static inline void dm_put_live_table_bio(struct mapped_device *md, int srcu_idx,
728− unsigned bio_opf)
728+ blk_opf_t bio_opf)
729729 {
730730 if (bio_opf & REQ_NOWAIT)
731731 dm_put_live_table_fast(md);
@@ -1033,7 +1033,7 @@ static void clone_endio(struct bio *bio) · 301 unmodified lines
10331033 }
10341034
10351035 if (static_branch_unlikely(&zoned_enabled) &&
1036− unlikely(blk_queue_is_zoned(bdev_get_queue(bio->bi_bdev))))
1036+ unlikely(bdev_is_zoned(bio->bi_bdev)))
10371037 dm_zone_endio(io, bio);
10381038
10391039 if (endio) {
@@ -1086,23 +1086,18 @@ static sector_t max_io_len(struct dm_target *ti, sector_t sector) · 46 unmodified lines
10861086 {
10871087 sector_t target_offset = dm_target_offset(ti, sector);
10881088 sector_t len = max_io_len_target_boundary(ti, target_offset);
1089− sector_t max_len;
10901089
10911090 /*
10921091 * Does the target need to split IO even further?
10931092 * - varied (per target) IO splitting is a tenet of DM; this
10941093 * explains why stacked chunk_sectors based splitting via
1095− * blk_max_size_offset() isn't possible here. So pass in
1096− * ti->max_io_len to override stacked chunk_sectors.
1094+ * blk_queue_split() isn't possible here.
10971095 */
1098− if (ti->max_io_len) {
1099− max_len = blk_max_size_offset(ti->table->md->queue,
1100− target_offset, ti->max_io_len);
1101− if (len > max_len)
1102− len = max_len;
1103− }
1104−
1105− return len;
1096+ if (!ti->max_io_len)
1097+ return len;
1098+ return min_t(sector_t, len,
1099+ min(queue_max_sectors(ti->table->md->queue),
1100+ blk_chunk_sectors_left(target_offset, ti->max_io_len)));
11061101 }
11071102
11081103 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
@@ -1516,7 +1511,7 @@ static void __send_changing_extent_only(struct clone_info *ci, struct dm_target · 407 unmodified lines
15161511
15171512 static bool is_abnormal_io(struct bio *bio)
15181513 {
1519− unsigned int op = bio_op(bio);
1514+ enum req_op op = bio_op(bio);
15201515
15211516 if (op != REQ_OP_READ && op != REQ_OP_WRITE && op != REQ_OP_FLUSH) {
15221517 switch (op) {
@@ -1547,6 +1542,8 @@ static blk_status_t __process_abnormal_io(struct clone_info *ci, · 24 unmodified lines
15471542 case REQ_OP_WRITE_ZEROES:
15481543 num_bios = ti->num_write_zeroes_bios;
15491544 break;
1545+ default:
1546+ break;
15501547 }
15511548
15521549 /*
@@ -1628,7 +1625,7 @@ static blk_status_t __split_and_process_bio(struct clone_info *ci) · 75 unmodified lines
16281625 * Only support bio polling for normal IO, and the target io is
16291626 * exactly inside the dm_io instance (verified in dm_poll_dm_io)
16301627 */
1631− ci->submit_as_polled = ci->bio->bi_opf & REQ_POLLED;
1628+ ci->submit_as_polled = !!(ci->bio->bi_opf & REQ_POLLED);
16321629
16331630 len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
16341631 setup_split_accounting(ci, len);
@@ -1725,7 +1722,7 @@ static void dm_submit_bio(struct bio *bio) · 90 unmodified lines
17251722 struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
17261723 int srcu_idx;
17271724 struct dm_table *map;
1728− unsigned bio_opf = bio->bi_opf;
1725+ blk_opf_t bio_opf = bio->bi_opf;
17291726
17301727 map = dm_get_live_table_bio(md, &srcu_idx, bio_opf);
17311728
@@ -1899,7 +1896,7 @@ static void cleanup_mapped_device(struct mapped_device *md) · 167 unmodified lines
18991896 del_gendisk(md->disk);
19001897 }
19011898 dm_queue_destroy_crypto_profile(md->queue);
1902− blk_cleanup_disk(md->disk);
1899+ put_disk(md->disk);
19031900 }
19041901
19051902 if (md->pending_io) {
drivers/md/md-bitmap.c+3−3
@@ -165,7 +165,7 @@ static int read_sb_page(struct mddev *mddev, loff_t offset, · 164 unmodified lines
165165
166166 if (sync_page_io(rdev, target,
167167 roundup(size, bdev_logical_block_size(rdev->bdev)),
168− page, REQ_OP_READ, 0, true)) {
168+ page, REQ_OP_READ, true)) {
169169 page->index = index;
170170 return 0;
171171 }
@@ -302,7 +302,7 @@ static void write_page(struct bitmap *bitmap, struct page *page, int wait) · 130 unmodified lines
302302 atomic_inc(&bitmap->pending_writes);
303303 set_buffer_locked(bh);
304304 set_buffer_mapped(bh);
305− submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
305+ submit_bh(REQ_OP_WRITE | REQ_SYNC, bh);
306306 bh = bh->b_this_page;
307307 }
308308
@@ -394,7 +394,7 @@ static int read_page(struct file *file, unsigned long index, · 85 unmodified lines
394394 atomic_inc(&bitmap->pending_writes);
395395 set_buffer_locked(bh);
396396 set_buffer_mapped(bh);
397− submit_bh(REQ_OP_READ, 0, bh);
397+ submit_bh(REQ_OP_READ, bh);
398398 }
399399 blk_cur++;
400400 bh = bh->b_this_page;
drivers/md/md.c+8−8
@@ -993,15 +993,15 @@ int md_super_wait(struct mddev *mddev) · 992 unmodified lines
993993 }
994994
995995 int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
996− struct page *page, int op, int op_flags, bool metadata_op)
996+ struct page *page, blk_opf_t opf, bool metadata_op)
997997 {
998998 struct bio bio;
999999 struct bio_vec bvec;
10001000
10011001 if (metadata_op && rdev->meta_bdev)
1002− bio_init(&bio, rdev->meta_bdev, &bvec, 1, op | op_flags);
1002+ bio_init(&bio, rdev->meta_bdev, &bvec, 1, opf);
10031003 else
1004− bio_init(&bio, rdev->bdev, &bvec, 1, op | op_flags);
1004+ bio_init(&bio, rdev->bdev, &bvec, 1, opf);
10051005
10061006 if (metadata_op)
10071007 bio.bi_iter.bi_sector = sector + rdev->sb_start;
@@ -1024,7 +1024,7 @@ static int read_disk_sb(struct md_rdev *rdev, int size) · 16 unmodified lines
10241024 if (rdev->sb_loaded)
10251025 return 0;
10261026
1027− if (!sync_page_io(rdev, 0, size, rdev->sb_page, REQ_OP_READ, 0, true))
1027+ if (!sync_page_io(rdev, 0, size, rdev->sb_page, REQ_OP_READ, true))
10281028 goto fail;
10291029 rdev->sb_loaded = 1;
10301030 return 0;
@@ -1722,7 +1722,7 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ · 691 unmodified lines
17221722 return -EINVAL;
17231723 bb_sector = (long long)offset;
17241724 if (!sync_page_io(rdev, bb_sector, sectors << 9,
1725− rdev->bb_page, REQ_OP_READ, 0, true))
1725+ rdev->bb_page, REQ_OP_READ, true))
17261726 return -EIO;
17271727 bbp = (__le64 *)page_address(rdev->bb_page);
17281728 rdev->badblocks.shift = sb->bblog_shift;
@@ -2438,7 +2438,7 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev) · 709 unmodified lines
24382438 mdname(mddev), mddev->max_disks);
24392439 return -EBUSY;
24402440 }
2441− bdevname(rdev->bdev,b);
2441+ snprintf(b, sizeof(b), "%pg", rdev->bdev);
24422442 strreplace(b, '/', '!');
24432443
24442444 rdev->mddev = mddev;
@@ -5579,7 +5579,7 @@ static void md_free(struct kobject *ko) · 3134 unmodified lines
55795579
55805580 if (mddev->gendisk) {
55815581 del_gendisk(mddev->gendisk);
5582− blk_cleanup_disk(mddev->gendisk);
5582+ put_disk(mddev->gendisk);
55835583 }
55845584 percpu_ref_exit(&mddev->writes_pending);
55855585
@@ -5718,7 +5718,7 @@ static int md_alloc(dev_t dev, char *name) · 132 unmodified lines
57185718 out_del_gendisk:
57195719 del_gendisk(disk);
57205720 out_cleanup_disk:
5721− blk_cleanup_disk(disk);
5721+ put_disk(disk);
57225722 out_unlock_disks_mutex:
57235723 mutex_unlock(&disks_mutex);
57245724 mddev_put(mddev);
drivers/md/md.h+1−2
@@ -738,8 +738,7 @@ extern void md_super_write(struct mddev *mddev, struct md_rdev *rdev, · 737 unmodified lines
738738 sector_t sector, int size, struct page *page);
739739 extern int md_super_wait(struct mddev *mddev);
740740 extern int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
741− struct page *page, int op, int op_flags,
742− bool metadata_op);
741+ struct page *page, blk_opf_t opf, bool metadata_op);
743742 extern void md_do_sync(struct md_thread *thread);
744743 extern void md_new_event(void);
745744 extern void md_allow_write(struct mddev *mddev);
drivers/md/raid1.c+7−7
@@ -1220,8 +1220,8 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, · 1219 unmodified lines
12201220 struct raid1_info *mirror;
12211221 struct bio *read_bio;
12221222 struct bitmap *bitmap = mddev->bitmap;
1223− const int op = bio_op(bio);
1224− const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1223+ const enum req_op op = bio_op(bio);
1224+ const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
12251225 int max_sectors;
12261226 int rdisk;
12271227 bool r1bio_existed = !!r1_bio;
@@ -1240,7 +1240,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, · 12 unmodified lines
12401240 rcu_read_lock();
12411241 rdev = rcu_dereference(conf->mirrors[r1_bio->read_disk].rdev);
12421242 if (rdev)
1243− bdevname(rdev->bdev, b);
1243+ snprintf(b, sizeof(b), "%pg", rdev->bdev);
12441244 else
12451245 strcpy(b, "???");
12461246 rcu_read_unlock();
@@ -1988,9 +1988,9 @@ static void end_sync_write(struct bio *bio) · 741 unmodified lines
19881988 }
19891989
19901990 static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector,
1991− int sectors, struct page *page, int rw)
1991+ int sectors, struct page *page, int rw)
19921992 {
1993− if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false))
1993+ if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
19941994 /* success */
19951995 return 1;
19961996 if (rw == WRITE) {
@@ -2057,7 +2057,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio) · 60 unmodified lines
20572057 rdev = conf->mirrors[d].rdev;
20582058 if (sync_page_io(rdev, sect, s<<9,
20592059 pages[idx],
2060− REQ_OP_READ, 0, false)) {
2060+ REQ_OP_READ, false)) {
20612061 success = 1;
20622062 break;
20632063 }
@@ -2305,7 +2305,7 @@ static void fix_read_error(struct r1conf *conf, int read_disk, · 241 unmodified lines
23052305 atomic_inc(&rdev->nr_pending);
23062306 rcu_read_unlock();
23072307 if (sync_page_io(rdev, sect, s<<9,
2308− conf->tmppage, REQ_OP_READ, 0, false))
2308+ conf->tmppage, REQ_OP_READ, false))
23092309 success = 1;
23102310 rdev_dec_pending(rdev, mddev);
23112311 if (success)
drivers/md/raid10.c+11−11
@@ -1136,8 +1136,8 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, · 1135 unmodified lines
11361136 {
11371137 struct r10conf *conf = mddev->private;
11381138 struct bio *read_bio;
1139− const int op = bio_op(bio);
1140− const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1139+ const enum req_op op = bio_op(bio);
1140+ const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
11411141 int max_sectors;
11421142 struct md_rdev *rdev;
11431143 char b[BDEVNAME_SIZE];
@@ -1164,7 +1164,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, · 20 unmodified lines
11641164 disk = r10_bio->devs[slot].devnum;
11651165 err_rdev = rcu_dereference(conf->mirrors[disk].rdev);
11661166 if (err_rdev)
1167− bdevname(err_rdev->bdev, b);
1167+ snprintf(b, sizeof(b), "%pg", err_rdev->bdev);
11681168 else {
11691169 strcpy(b, "???");
11701170 /* This never gets dereferenced */
@@ -1230,9 +1230,9 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, · 59 unmodified lines
12301230 struct bio *bio, bool replacement,
12311231 int n_copy)
12321232 {
1233− const int op = bio_op(bio);
1234− const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1235− const unsigned long do_fua = (bio->bi_opf & REQ_FUA);
1233+ const enum req_op op = bio_op(bio);
1234+ const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
1235+ const blk_opf_t do_fua = bio->bi_opf & REQ_FUA;
12361236 unsigned long flags;
12371237 struct blk_plug_cb *cb;
12381238 struct raid1_plug_cb *plug = NULL;
@@ -2512,7 +2512,7 @@ static void fix_recovery_read_error(struct r10bio *r10_bio) · 1273 unmodified lines
25122512 addr,
25132513 s << 9,
25142514 pages[idx],
2515− REQ_OP_READ, 0, false);
2515+ REQ_OP_READ, false);
25162516 if (ok) {
25172517 rdev = conf->mirrors[dw].rdev;
25182518 addr = r10_bio->devs[1].addr + sect;
@@ -2520,7 +2520,7 @@ static void fix_recovery_read_error(struct r10bio *r10_bio) · 1 unmodified line
25202520 addr,
25212521 s << 9,
25222522 pages[idx],
2523− REQ_OP_WRITE, 0, false);
2523+ REQ_OP_WRITE, false);
25242524 if (!ok) {
25252525 set_bit(WriteErrorSeen, &rdev->flags);
25262526 if (!test_and_set_bit(WantReplacement,
@@ -2644,7 +2644,7 @@ static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, · 117 unmodified lines
26442644 if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
26452645 && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
26462646 return -1;
2647− if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false))
2647+ if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
26482648 /* success */
26492649 return 1;
26502650 if (rw == WRITE) {
@@ -2726,7 +2726,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10 · 75 unmodified lines
27262726 sect,
27272727 s<<9,
27282728 conf->tmppage,
2729− REQ_OP_READ, 0, false);
2729+ REQ_OP_READ, false);
27302730 rdev_dec_pending(rdev, mddev);
27312731 rcu_read_lock();
27322732 if (success)
@@ -5107,7 +5107,7 @@ static int handle_reshape_read_error(struct mddev *mddev, · 2374 unmodified lines
51075107 addr,
51085108 s << 9,
51095109 pages[idx],
5110− REQ_OP_READ, 0, false);
5110+ REQ_OP_READ, false);
51115111 rdev_dec_pending(rdev, mddev);
51125112 rcu_read_lock();
51135113 if (success)
drivers/md/raid5-cache.c+6−6
@@ -1788,7 +1788,7 @@ static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos, · 1787 unmodified lines
17881788 mb = page_address(page);
17891789 mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
17901790 mb, PAGE_SIZE));
1791− if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE,
1791+ if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE |
17921792 REQ_SYNC | REQ_FUA, false)) {
17931793 __free_page(page);
17941794 return -EIO;
@@ -1898,7 +1898,7 @@ r5l_recovery_replay_one_stripe(struct r5conf *conf, · 103 unmodified lines
18981898 atomic_inc(&rdev->nr_pending);
18991899 rcu_read_unlock();
19001900 sync_page_io(rdev, sh->sector, PAGE_SIZE,
1901− sh->dev[disk_index].page, REQ_OP_WRITE, 0,
1901+ sh->dev[disk_index].page, REQ_OP_WRITE,
19021902 false);
19031903 rdev_dec_pending(rdev, rdev->mddev);
19041904 rcu_read_lock();
@@ -1908,7 +1908,7 @@ r5l_recovery_replay_one_stripe(struct r5conf *conf, · 3 unmodified lines
19081908 atomic_inc(&rrdev->nr_pending);
19091909 rcu_read_unlock();
19101910 sync_page_io(rrdev, sh->sector, PAGE_SIZE,
1911− sh->dev[disk_index].page, REQ_OP_WRITE, 0,
1911+ sh->dev[disk_index].page, REQ_OP_WRITE,
19121912 false);
19131913 rdev_dec_pending(rrdev, rrdev->mddev);
19141914 rcu_read_lock();
@@ -2394,7 +2394,7 @@ r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log, · 479 unmodified lines
23942394 PAGE_SIZE));
23952395 kunmap_atomic(addr);
23962396 sync_page_io(log->rdev, write_pos, PAGE_SIZE,
2397− dev->page, REQ_OP_WRITE, 0, false);
2397+ dev->page, REQ_OP_WRITE, false);
23982398 write_pos = r5l_ring_add(log, write_pos,
23992399 BLOCK_SECTORS);
24002400 offset += sizeof(__le32) +
@@ -2406,7 +2406,7 @@ r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log, · 5 unmodified lines
24062406 mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
24072407 mb, PAGE_SIZE));
24082408 sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page,
2409− REQ_OP_WRITE, REQ_SYNC | REQ_FUA, false);
2409+ REQ_OP_WRITE | REQ_SYNC | REQ_FUA, false);
24102410 sh->log_start = ctx->pos;
24112411 list_add_tail(&sh->r5c, &log->stripe_in_journal_list);
24122412 atomic_inc(&log->stripe_in_journal_count);
@@ -2971,7 +2971,7 @@ static int r5l_load_log(struct r5l_log *log) · 558 unmodified lines
29712971 if (!page)
29722972 return -ENOMEM;
29732973
2974− if (!sync_page_io(rdev, cp, PAGE_SIZE, page, REQ_OP_READ, 0, false)) {
2974+ if (!sync_page_io(rdev, cp, PAGE_SIZE, page, REQ_OP_READ, false)) {
29752975 ret = -EIO;
29762976 goto ioerr;
29772977 }
drivers/md/raid5-ppl.c+6−6
@@ -897,7 +897,7 @@ static int ppl_recover_entry(struct ppl_log *log, struct ppl_header_entry *e, · 896 unmodified lines
897897 __func__, indent, "", rdev->bdev,
898898 (unsigned long long)sector);
899899 if (!sync_page_io(rdev, sector, block_size, page2,
900− REQ_OP_READ, 0, false)) {
900+ REQ_OP_READ, false)) {
901901 md_error(mddev, rdev);
902902 pr_debug("%s:%*s read failed!\n", __func__,
903903 indent, "");
@@ -919,7 +919,7 @@ static int ppl_recover_entry(struct ppl_log *log, struct ppl_header_entry *e, · 15 unmodified lines
919919 (unsigned long long)(ppl_sector + i));
920920 if (!sync_page_io(log->rdev,
921921 ppl_sector - log->rdev->data_offset + i,
922− block_size, page2, REQ_OP_READ, 0,
922+ block_size, page2, REQ_OP_READ,
923923 false)) {
924924 pr_debug("%s:%*s read failed!\n", __func__,
925925 indent, "");
@@ -946,7 +946,7 @@ static int ppl_recover_entry(struct ppl_log *log, struct ppl_header_entry *e, · 20 unmodified lines
946946 (unsigned long long)parity_sector,
947947 parity_rdev->bdev);
948948 if (!sync_page_io(parity_rdev, parity_sector, block_size,
949− page1, REQ_OP_WRITE, 0, false)) {
949+ page1, REQ_OP_WRITE, false)) {
950950 pr_debug("%s:%*s parity write error!\n", __func__,
951951 indent, "");
952952 md_error(mddev, parity_rdev);
@@ -998,7 +998,7 @@ static int ppl_recover(struct ppl_log *log, struct ppl_header *pplhdr, · 45 unmodified lines
998998 int s = pp_size > PAGE_SIZE ? PAGE_SIZE : pp_size;
999999
10001000 if (!sync_page_io(rdev, sector - rdev->data_offset,
1001− s, page, REQ_OP_READ, 0, false)) {
1001+ s, page, REQ_OP_READ, false)) {
10021002 md_error(mddev, rdev);
10031003 ret = -EIO;
10041004 goto out;
@@ -1062,7 +1062,7 @@ static int ppl_write_empty_header(struct ppl_log *log) · 57 unmodified lines
10621062
10631063 if (!sync_page_io(rdev, rdev->ppl.sector - rdev->data_offset,
10641064 PPL_HEADER_SIZE, page, REQ_OP_WRITE | REQ_SYNC |
1065− REQ_FUA, 0, false)) {
1065+ REQ_FUA, false)) {
10661066 md_error(rdev->mddev, rdev);
10671067 ret = -EIO;
10681068 }
@@ -1100,7 +1100,7 @@ static int ppl_load_distributed(struct ppl_log *log) · 31 unmodified lines
11001100 if (!sync_page_io(rdev,
11011101 rdev->ppl.sector - rdev->data_offset +
11021102 pplhdr_offset, PAGE_SIZE, page, REQ_OP_READ,
1103− 0, false)) {
1103+ false)) {
11041104 md_error(mddev, rdev);
11051105 ret = -EIO;
11061106 /* if not able to read - don't recover any PPL */
drivers/md/raid5.c+2−1
@@ -1082,7 +1082,8 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) · 1081 unmodified lines
10821082 should_defer = conf->batch_bio_dispatch && conf->group_cnt;
10831083
10841084 for (i = disks; i--; ) {
1085− int op, op_flags = 0;
1085+ enum req_op op;
1086+ blk_opf_t op_flags = 0;
10861087 int replace_only = 0;
10871088 struct bio *bi, *rbi;
10881089 struct md_rdev *rdev, *rrdev = NULL;
drivers/memstick/core/ms_block.c+1−2
@@ -2129,7 +2129,7 @@ static int msb_init_disk(struct memstick_dev *card) · 2128 unmodified lines
21292129 return 0;
21302130
21312131 out_cleanup_disk:
2132− blk_cleanup_disk(msb->disk);
2132+ put_disk(msb->disk);
21332133 out_free_tag_set:
21342134 blk_mq_free_tag_set(&msb->tag_set);
21352135 out_release_id:
@@ -2187,7 +2187,6 @@ static void msb_remove(struct memstick_dev *card) · 51 unmodified lines
21872187
21882188 /* Remove the disk */
21892189 del_gendisk(msb->disk);
2190− blk_cleanup_queue(msb->queue);
21912190 blk_mq_free_tag_set(&msb->tag_set);
21922191 msb->queue = NULL;
21932192
drivers/memstick/core/mspro_block.c+1−2
@@ -1209,7 +1209,7 @@ static int mspro_block_init_disk(struct memstick_dev *card) · 1208 unmodified lines
12091209 return 0;
12101210
12111211 out_cleanup_disk:
1212− blk_cleanup_disk(msb->disk);
1212+ put_disk(msb->disk);
12131213 out_free_tag_set:
12141214 blk_mq_free_tag_set(&msb->tag_set);
12151215 out_release_id:
@@ -1294,7 +1294,6 @@ static void mspro_block_remove(struct memstick_dev *card) · 78 unmodified lines
12941294 del_gendisk(msb->disk);
12951295 dev_dbg(&card->dev, "mspro block remove\n");
12961296
1297− blk_cleanup_queue(msb->queue);
12981297 blk_mq_free_tag_set(&msb->tag_set);
12991298 msb->queue = NULL;
13001299
drivers/mmc/core/block.c+3−3
@@ -2505,11 +2505,11 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, · 2504 unmodified lines
25052505 dev_set_drvdata(&card->dev, md);
25062506 ret = device_add_disk(md->parent, md->disk, mmc_disk_attr_groups);
25072507 if (ret)
2508− goto err_cleanup_queue;
2508+ goto err_put_disk;
25092509 return md;
25102510
2511− err_cleanup_queue:
2512− blk_cleanup_queue(md->disk->queue);
2511+ err_put_disk:
2512+ put_disk(md->disk);
25132513 blk_mq_free_tag_set(&md->queue.tag_set);
25142514 err_kfree:
25152515 kfree(md);
drivers/mmc/core/queue.c+1−3
@@ -116,8 +116,7 @@ static enum blk_eh_timer_return mmc_cqe_timed_out(struct request *req) · 115 unmodified lines
116116 }
117117 }
118118
119−static enum blk_eh_timer_return mmc_mq_timed_out(struct request *req,
120− bool reserved)
119+static enum blk_eh_timer_return mmc_mq_timed_out(struct request *req)
121120 {
122121 struct request_queue *q = req->q;
123122 struct mmc_queue *mq = q->queuedata;
@@ -494,7 +493,6 @@ void mmc_cleanup_queue(struct mmc_queue *mq) · 370 unmodified lines
494493 if (blk_queue_quiesced(q))
495494 blk_mq_unquiesce_queue(q);
496495
497− blk_cleanup_queue(q);
498496 blk_mq_free_tag_set(&mq->tag_set);
499497
500498 /*
drivers/mtd/mtd_blkdevs.c+2−2
@@ -29,7 +29,7 @@ static void blktrans_dev_release(struct kref *kref) · 28 unmodified lines
2929 struct mtd_blktrans_dev *dev =
3030 container_of(kref, struct mtd_blktrans_dev, ref);
3131
32− blk_cleanup_disk(dev->disk);
32+ put_disk(dev->disk);
3333 blk_mq_free_tag_set(dev->tag_set);
3434 kfree(dev->tag_set);
3535 list_del(&dev->list);
@@ -398,7 +398,7 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) · 362 unmodified lines
398398 return 0;
399399
400400 out_cleanup_disk:
401− blk_cleanup_disk(new->disk);
401+ put_disk(new->disk);
402402 out_free_tag_set:
403403 blk_mq_free_tag_set(new->tag_set);
404404 out_kfree_tag_set:
drivers/mtd/ubi/block.c+2−2
@@ -467,7 +467,7 @@ out_destroy_wq: · 466 unmodified lines
467467 out_remove_minor:
468468 idr_remove(&ubiblock_minor_idr, gd->first_minor);
469469 out_cleanup_disk:
470− blk_cleanup_disk(dev->gd);
470+ put_disk(dev->gd);
471471 out_free_tags:
472472 blk_mq_free_tag_set(&dev->tag_set);
473473 out_free_dev:
@@ -486,7 +486,7 @@ static void ubiblock_cleanup(struct ubiblock *dev) · 12 unmodified lines
486486 destroy_workqueue(dev->wq);
487487 /* Finally destroy the blk queue */
488488 dev_info(disk_to_dev(dev->gd), "released");
489− blk_cleanup_disk(dev->gd);
489+ put_disk(dev->gd);
490490 blk_mq_free_tag_set(&dev->tag_set);
491491 idr_remove(&ubiblock_minor_idr, dev->gd->first_minor);
492492 }
drivers/nvdimm/btt.c+4−4
@@ -1422,7 +1422,7 @@ static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip, · 1421 unmodified lines
14221422
14231423 static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip,
14241424 struct page *page, unsigned int len, unsigned int off,
1425− unsigned int op, sector_t sector)
1425+ enum req_op op, sector_t sector)
14261426 {
14271427 int ret;
14281428
@@ -1483,7 +1483,7 @@ static void btt_submit_bio(struct bio *bio) · 54 unmodified lines
14831483 }
14841484
14851485 static int btt_rw_page(struct block_device *bdev, sector_t sector,
1486− struct page *page, unsigned int op)
1486+ struct page *page, enum req_op op)
14871487 {
14881488 struct btt *btt = bdev->bd_disk->private_data;
14891489 int rc;
@@ -1548,14 +1548,14 @@ static int btt_blk_init(struct btt *btt) · 58 unmodified lines
15481548 return 0;
15491549
15501550 out_cleanup_disk:
1551− blk_cleanup_disk(btt->btt_disk);
1551+ put_disk(btt->btt_disk);
15521552 return rc;
15531553 }
15541554
15551555 static void btt_blk_cleanup(struct btt *btt)
15561556 {
15571557 del_gendisk(btt->btt_disk);
1558− blk_cleanup_disk(btt->btt_disk);
1558+ put_disk(btt->btt_disk);
15591559 }
15601560
15611561 /**
drivers/nvdimm/pmem.c+3−3
@@ -239,7 +239,7 @@ static void pmem_submit_bio(struct bio *bio) · 238 unmodified lines
239239 }
240240
241241 static int pmem_rw_page(struct block_device *bdev, sector_t sector,
242− struct page *page, unsigned int op)
242+ struct page *page, enum req_op op)
243243 {
244244 struct pmem_device *pmem = bdev->bd_disk->private_data;
245245 blk_status_t rc;
@@ -450,7 +450,7 @@ static void pmem_release_disk(void *__pmem) · 204 unmodified lines
450450 put_dax(pmem->dax_dev);
451451 del_gendisk(pmem->disk);
452452
453− blk_cleanup_disk(pmem->disk);
453+ put_disk(pmem->disk);
454454 }
455455
456456 static int pmem_attach_disk(struct device *dev,
@@ -596,7 +596,7 @@ out_cleanup_dax: · 139 unmodified lines
596596 kill_dax(pmem->dax_dev);
597597 put_dax(pmem->dax_dev);
598598 out:
599− blk_cleanup_disk(pmem->disk);
599+ put_disk(pmem->disk);
600600 return rc;
601601 }
602602
drivers/nvme/host/apple.c+2−3
@@ -862,8 +862,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) · 861 unmodified lines
862862 }
863863 }
864864
865−static enum blk_eh_timer_return apple_nvme_timeout(struct request *req,
866− bool reserved)
865+static enum blk_eh_timer_return apple_nvme_timeout(struct request *req)
867866 {
868867 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
869868 struct apple_nvme_queue *q = iod->q;
@@ -1502,7 +1501,7 @@ static int apple_nvme_probe(struct platform_device *pdev) · 632 unmodified lines
15021501
15031502 if (!blk_get_queue(anv->ctrl.admin_q)) {
15041503 nvme_start_admin_queue(&anv->ctrl);
1505− blk_cleanup_queue(anv->ctrl.admin_q);
1504+ blk_mq_destroy_queue(anv->ctrl.admin_q);
15061505 anv->ctrl.admin_q = NULL;
15071506 ret = -ENODEV;
15081507 goto put_dev;
drivers/nvme/host/core.c+2−3
@@ -418,7 +418,7 @@ blk_status_t nvme_host_path_error(struct request *req) · 417 unmodified lines
418418 }
419419 EXPORT_SYMBOL_GPL(nvme_host_path_error);
420420
421−bool nvme_cancel_request(struct request *req, void *data, bool reserved)
421+bool nvme_cancel_request(struct request *req, void *data)
422422 {
423423 dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
424424 "Cancelling I/O %d", req->tag);
@@ -4061,7 +4061,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid, · 3636 unmodified lines
40614061 mutex_unlock(&ctrl->subsys->lock);
40624062 nvme_put_ns_head(ns->head);
40634063 out_cleanup_disk:
4064− blk_cleanup_disk(disk);
4064+ put_disk(disk);
40654065 out_free_ns:
40664066 kfree(ns);
40674067 out_free_id:
@@ -4103,7 +4103,6 @@ static void nvme_ns_remove(struct nvme_ns *ns) · 35 unmodified lines
41034103 if (!nvme_ns_head_multipath(ns->head))
41044104 nvme_cdev_del(&ns->cdev, &ns->cdev_device);
41054105 del_gendisk(ns->disk);
4106− blk_cleanup_queue(ns->queue);
41074106
41084107 down_write(&ns->ctrl->namespaces_rwsem);
41094108 list_del_init(&ns->list);
drivers/nvme/host/fc.c+8−10
@@ -2392,7 +2392,7 @@ nvme_fc_ctrl_free(struct kref *ref) · 2391 unmodified lines
23922392 unsigned long flags;
23932393
23942394 if (ctrl->ctrl.tagset) {
2395− blk_cleanup_queue(ctrl->ctrl.connect_q);
2395+ blk_mq_destroy_queue(ctrl->ctrl.connect_q);
23962396 blk_mq_free_tag_set(&ctrl->tag_set);
23972397 }
23982398
@@ -2402,8 +2402,8 @@ nvme_fc_ctrl_free(struct kref *ref) · 3 unmodified lines
24022402 spin_unlock_irqrestore(&ctrl->rport->lock, flags);
24032403
24042404 nvme_start_admin_queue(&ctrl->ctrl);
2405− blk_cleanup_queue(ctrl->ctrl.admin_q);
2406− blk_cleanup_queue(ctrl->ctrl.fabrics_q);
2405+ blk_mq_destroy_queue(ctrl->ctrl.admin_q);
2406+ blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
24072407 blk_mq_free_tag_set(&ctrl->admin_tag_set);
24082408
24092409 kfree(ctrl->queues);
@@ -2456,8 +2456,7 @@ nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl) · 46 unmodified lines
24562456 * status. The done path will return the io request back to the block
24572457 * layer with an error status.
24582458 */
2459−static bool
2460−nvme_fc_terminate_exchange(struct request *req, void *data, bool reserved)
2459+static bool nvme_fc_terminate_exchange(struct request *req, void *data)
24612460 {
24622461 struct nvme_ctrl *nctrl = data;
24632462 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
@@ -2565,8 +2564,7 @@ nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg) · 101 unmodified lines
25652564 nvme_reset_ctrl(&ctrl->ctrl);
25662565 }
25672566
2568−static enum blk_eh_timer_return
2569−nvme_fc_timeout(struct request *rq, bool reserved)
2567+static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
25702568 {
25712569 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
25722570 struct nvme_fc_ctrl *ctrl = op->ctrl;
@@ -2953,7 +2951,7 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) · 380 unmodified lines
29532951 out_delete_hw_queues:
29542952 nvme_fc_delete_hw_io_queues(ctrl);
29552953 out_cleanup_blk_queue:
2956− blk_cleanup_queue(ctrl->ctrl.connect_q);
2954+ blk_mq_destroy_queue(ctrl->ctrl.connect_q);
29572955 out_free_tag_set:
29582956 blk_mq_free_tag_set(&ctrl->tag_set);
29592957 nvme_fc_free_io_queues(ctrl);
@@ -3642,9 +3640,9 @@ fail_ctrl: · 682 unmodified lines
36423640 return ERR_PTR(-EIO);
36433641
36443642 out_cleanup_admin_q:
3645− blk_cleanup_queue(ctrl->ctrl.admin_q);
3643+ blk_mq_destroy_queue(ctrl->ctrl.admin_q);
36463644 out_cleanup_fabrics_q:
3647− blk_cleanup_queue(ctrl->ctrl.fabrics_q);
3645+ blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
36483646 out_free_admin_tag_set:
36493647 blk_mq_free_tag_set(&ctrl->admin_tag_set);
36503648 out_free_queues:
drivers/nvme/host/ioctl.c+2−2
@@ -68,7 +68,7 @@ static struct request *nvme_alloc_user_request(struct request_queue *q, · 67 unmodified lines
6868 struct nvme_command *cmd, void __user *ubuffer,
6969 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
7070 u32 meta_seed, void **metap, unsigned timeout, bool vec,
71− unsigned int rq_flags, blk_mq_req_flags_t blk_flags)
71+ blk_opf_t rq_flags, blk_mq_req_flags_t blk_flags)
7272 {
7373 bool write = nvme_is_write(cmd);
7474 struct nvme_ns *ns = q->queuedata;
@@ -407,7 +407,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 332 unmodified lines
407407 struct nvme_uring_data d;
408408 struct nvme_command c;
409409 struct request *req;
410− unsigned int rq_flags = 0;
410+ blk_opf_t rq_flags = 0;
411411 blk_mq_req_flags_t blk_flags = 0;
412412 void *meta = NULL;
413413
drivers/nvme/host/multipath.c+2−2
@@ -830,7 +830,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) · 829 unmodified lines
830830 ns->head->disk->queue);
831831 #ifdef CONFIG_BLK_DEV_ZONED
832832 if (blk_queue_is_zoned(ns->queue) && ns->head->disk)
833− ns->head->disk->queue->nr_zones = ns->queue->nr_zones;
833+ ns->head->disk->nr_zones = ns->disk->nr_zones;
834834 #endif
835835 }
836836
@@ -853,7 +853,7 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head) · 16 unmodified lines
853853 /* make sure all pending bios are cleaned up */
854854 kblockd_schedule_work(&head->requeue_work);
855855 flush_work(&head->requeue_work);
856− blk_cleanup_disk(head->disk);
856+ put_disk(head->disk);
857857 }
858858
859859 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
drivers/nvme/host/nvme.h+2−2
@@ -698,7 +698,7 @@ static __always_inline void nvme_complete_batch(struct io_comp_batch *iob, · 697 unmodified lines
698698 }
699699
700700 blk_status_t nvme_host_path_error(struct request *req);
701−bool nvme_cancel_request(struct request *req, void *data, bool reserved);
701+bool nvme_cancel_request(struct request *req, void *data);
702702 void nvme_cancel_tagset(struct nvme_ctrl *ctrl);
703703 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl);
704704 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
@@ -734,7 +734,7 @@ void nvme_wait_freeze(struct nvme_ctrl *ctrl); · 29 unmodified lines
734734 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
735735 void nvme_start_freeze(struct nvme_ctrl *ctrl);
736736
737−static inline unsigned int nvme_req_op(struct nvme_command *cmd)
737+static inline enum req_op nvme_req_op(struct nvme_command *cmd)
738738 {
739739 return nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
740740 }
drivers/nvme/host/pci.c+2−2
@@ -1344,7 +1344,7 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) · 1343 unmodified lines
13441344 "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off\" and report a bug\n");
13451345 }
13461346
1347−static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
1347+static enum blk_eh_timer_return nvme_timeout(struct request *req)
13481348 {
13491349 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
13501350 struct nvme_queue *nvmeq = iod->nvmeq;
@@ -1760,7 +1760,7 @@ static void nvme_dev_remove_admin(struct nvme_dev *dev) · 409 unmodified lines
17601760 * queue to flush these to completion.
17611761 */
17621762 nvme_start_admin_queue(&dev->ctrl);
1763− blk_cleanup_queue(dev->ctrl.admin_q);
1763+ blk_mq_destroy_queue(dev->ctrl.admin_q);
17641764 blk_mq_free_tag_set(&dev->admin_tagset);
17651765 }
17661766 }
drivers/nvme/host/rdma.c+7−8
@@ -840,8 +840,8 @@ static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl, · 839 unmodified lines
840840 bool remove)
841841 {
842842 if (remove) {
843− blk_cleanup_queue(ctrl->ctrl.admin_q);
844− blk_cleanup_queue(ctrl->ctrl.fabrics_q);
843+ blk_mq_destroy_queue(ctrl->ctrl.admin_q);
844+ blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
845845 blk_mq_free_tag_set(ctrl->ctrl.admin_tagset);
846846 }
847847 if (ctrl->async_event_sqe.data) {
@@ -935,10 +935,10 @@ out_stop_queue: · 87 unmodified lines
935935 nvme_cancel_admin_tagset(&ctrl->ctrl);
936936 out_cleanup_queue:
937937 if (new)
938− blk_cleanup_queue(ctrl->ctrl.admin_q);
938+ blk_mq_destroy_queue(ctrl->ctrl.admin_q);
939939 out_cleanup_fabrics_q:
940940 if (new)
941− blk_cleanup_queue(ctrl->ctrl.fabrics_q);
941+ blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
942942 out_free_tagset:
943943 if (new)
944944 blk_mq_free_tag_set(ctrl->ctrl.admin_tagset);
@@ -957,7 +957,7 @@ static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl, · 12 unmodified lines
957957 bool remove)
958958 {
959959 if (remove) {
960− blk_cleanup_queue(ctrl->ctrl.connect_q);
960+ blk_mq_destroy_queue(ctrl->ctrl.connect_q);
961961 blk_mq_free_tag_set(ctrl->ctrl.tagset);
962962 }
963963 nvme_rdma_free_io_queues(ctrl);
@@ -1012,7 +1012,7 @@ out_wait_freeze_timed_out: · 48 unmodified lines
10121012 out_cleanup_connect_q:
10131013 nvme_cancel_tagset(&ctrl->ctrl);
10141014 if (new)
1015− blk_cleanup_queue(ctrl->ctrl.connect_q);
1015+ blk_mq_destroy_queue(ctrl->ctrl.connect_q);
10161016 out_free_tag_set:
10171017 if (new)
10181018 blk_mq_free_tag_set(ctrl->ctrl.tagset);
@@ -2021,8 +2021,7 @@ static void nvme_rdma_complete_timed_out(struct request *rq) · 1002 unmodified lines
20212021 nvmf_complete_timed_out_request(rq);
20222022 }
20232023
2024−static enum blk_eh_timer_return
2025−nvme_rdma_timeout(struct request *rq, bool reserved)
2024+static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq)
20262025 {
20272026 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
20282027 struct nvme_rdma_queue *queue = req->queue;
drivers/nvme/host/tcp.c+7−8
@@ -1884,7 +1884,7 @@ static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) · 1883 unmodified lines
18841884 {
18851885 nvme_tcp_stop_io_queues(ctrl);
18861886 if (remove) {
1887− blk_cleanup_queue(ctrl->connect_q);
1887+ blk_mq_destroy_queue(ctrl->connect_q);
18881888 blk_mq_free_tag_set(ctrl->tagset);
18891889 }
18901890 nvme_tcp_free_io_queues(ctrl);
@@ -1939,7 +1939,7 @@ out_wait_freeze_timed_out: · 48 unmodified lines
19391939 out_cleanup_connect_q:
19401940 nvme_cancel_tagset(ctrl);
19411941 if (new)
1942− blk_cleanup_queue(ctrl->connect_q);
1942+ blk_mq_destroy_queue(ctrl->connect_q);
19431943 out_free_tag_set:
19441944 if (new)
19451945 blk_mq_free_tag_set(ctrl->tagset);
@@ -1952,8 +1952,8 @@ static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) · 6 unmodified lines
19521952 {
19531953 nvme_tcp_stop_queue(ctrl, 0);
19541954 if (remove) {
1955− blk_cleanup_queue(ctrl->admin_q);
1956− blk_cleanup_queue(ctrl->fabrics_q);
1955+ blk_mq_destroy_queue(ctrl->admin_q);
1956+ blk_mq_destroy_queue(ctrl->fabrics_q);
19571957 blk_mq_free_tag_set(ctrl->admin_tagset);
19581958 }
19591959 nvme_tcp_free_admin_queue(ctrl);
@@ -2011,10 +2011,10 @@ out_stop_queue: · 51 unmodified lines
20112011 nvme_cancel_admin_tagset(ctrl);
20122012 out_cleanup_queue:
20132013 if (new)
2014− blk_cleanup_queue(ctrl->admin_q);
2014+ blk_mq_destroy_queue(ctrl->admin_q);
20152015 out_cleanup_fabrics_q:
20162016 if (new)
2017− blk_cleanup_queue(ctrl->fabrics_q);
2017+ blk_mq_destroy_queue(ctrl->fabrics_q);
20182018 out_free_tagset:
20192019 if (new)
20202020 blk_mq_free_tag_set(ctrl->admin_tagset);
@@ -2323,8 +2323,7 @@ static void nvme_tcp_complete_timed_out(struct request *rq) · 302 unmodified lines
23232323 nvmf_complete_timed_out_request(rq);
23242324 }
23252325
2326−static enum blk_eh_timer_return
2327−nvme_tcp_timeout(struct request *rq, bool reserved)
2326+static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq)
23282327 {
23292328 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
23302329 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
drivers/nvme/host/zns.c+3−3
@@ -109,10 +109,10 @@ int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf) · 108 unmodified lines
109109 goto free_data;
110110 }
111111
112− blk_queue_set_zoned(ns->disk, BLK_ZONED_HM);
112+ disk_set_zoned(ns->disk, BLK_ZONED_HM);
113113 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
114− blk_queue_max_open_zones(q, le32_to_cpu(id->mor) + 1);
115− blk_queue_max_active_zones(q, le32_to_cpu(id->mar) + 1);
114+ disk_set_max_open_zones(ns->disk, le32_to_cpu(id->mor) + 1);
115+ disk_set_max_active_zones(ns->disk, le32_to_cpu(id->mar) + 1);
116116 free_data:
117117 kfree(id);
118118 return status;
drivers/nvme/target/io-cmd-bdev.c+9−8
@@ -246,7 +246,8 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) · 245 unmodified lines
246246 struct scatterlist *sg;
247247 struct blk_plug plug;
248248 sector_t sector;
249− int op, i, rc;
249+ blk_opf_t opf;
250+ int i, rc;
250251 struct sg_mapping_iter prot_miter;
251252 unsigned int iter_flags;
252253 unsigned int total_len = nvmet_rw_data_len(req) + req->metadata_len;
@@ -260,26 +261,26 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) · 7 unmodified lines
260261 }
261262
262263 if (req->cmd->rw.opcode == nvme_cmd_write) {
263− op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
264+ opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
264265 if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
265− op |= REQ_FUA;
266+ opf |= REQ_FUA;
266267 iter_flags = SG_MITER_TO_SG;
267268 } else {
268− op = REQ_OP_READ;
269+ opf = REQ_OP_READ;
269270 iter_flags = SG_MITER_FROM_SG;
270271 }
271272
272273 if (is_pci_p2pdma_page(sg_page(req->sg)))
273− op |= REQ_NOMERGE;
274+ opf |= REQ_NOMERGE;
274275
275276 sector = nvmet_lba_to_sect(req->ns, req->cmd->rw.slba);
276277
277278 if (nvmet_use_inline_bvec(req)) {
278279 bio = &req->b.inline_bio;
279280 bio_init(bio, req->ns->bdev, req->inline_bvec,
280− ARRAY_SIZE(req->inline_bvec), op);
281+ ARRAY_SIZE(req->inline_bvec), opf);
281282 } else {
282− bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt), op,
283+ bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt), opf,
283284 GFP_KERNEL);
284285 }
285286 bio->bi_iter.bi_sector = sector;
@@ -306,7 +307,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) · 20 unmodified lines
306307 }
307308
308309 bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt),
309− op, GFP_KERNEL);
310+ opf, GFP_KERNEL);
310311 bio->bi_iter.bi_sector = sector;
311312
312313 bio_chain(bio, prev);
drivers/nvme/target/loop.c+6−6
@@ -266,8 +266,8 @@ static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) · 265 unmodified lines
266266 if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags))
267267 return;
268268 nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
269− blk_cleanup_queue(ctrl->ctrl.admin_q);
270− blk_cleanup_queue(ctrl->ctrl.fabrics_q);
269+ blk_mq_destroy_queue(ctrl->ctrl.admin_q);
270+ blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
271271 blk_mq_free_tag_set(&ctrl->admin_tag_set);
272272 }
273273
@@ -283,7 +283,7 @@ static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) · 9 unmodified lines
283283 mutex_unlock(&nvme_loop_ctrl_mutex);
284284
285285 if (nctrl->tagset) {
286− blk_cleanup_queue(ctrl->ctrl.connect_q);
286+ blk_mq_destroy_queue(ctrl->ctrl.connect_q);
287287 blk_mq_free_tag_set(&ctrl->tag_set);
288288 }
289289 kfree(ctrl->queues);
@@ -410,9 +410,9 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) · 120 unmodified lines
410410
411411 out_cleanup_queue:
412412 clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
413− blk_cleanup_queue(ctrl->ctrl.admin_q);
413+ blk_mq_destroy_queue(ctrl->ctrl.admin_q);
414414 out_cleanup_fabrics_q:
415− blk_cleanup_queue(ctrl->ctrl.fabrics_q);
415+ blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
416416 out_free_tagset:
417417 blk_mq_free_tag_set(&ctrl->admin_tag_set);
418418 out_free_sq:
@@ -554,7 +554,7 @@ static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl) · 135 unmodified lines
554554 return 0;
555555
556556 out_cleanup_connect_q:
557− blk_cleanup_queue(ctrl->ctrl.connect_q);
557+ blk_mq_destroy_queue(ctrl->ctrl.connect_q);
558558 out_free_tagset:
559559 blk_mq_free_tag_set(&ctrl->tag_set);
560560 out_destroy_queues:
drivers/nvme/target/zns.c+12−12
@@ -57,10 +57,10 @@ bool nvmet_bdev_zns_enable(struct nvmet_ns *ns) · 56 unmodified lines
5757 * zones, reject the device. Otherwise, use report zones to detect if
5858 * the device has conventional zones.
5959 */
60− if (ns->bdev->bd_disk->queue->conv_zones_bitmap)
60+ if (ns->bdev->bd_disk->conv_zones_bitmap)
6161 return false;
6262
63− ret = blkdev_report_zones(ns->bdev, 0, blkdev_nr_zones(bd_disk),
63+ ret = blkdev_report_zones(ns->bdev, 0, bdev_nr_zones(ns->bdev),
6464 validate_conv_zones_cb, NULL);
6565 if (ret < 0)
6666 return false;
@@ -241,7 +241,7 @@ static unsigned long nvmet_req_nr_zones_from_slba(struct nvmet_req *req) · 174 unmodified lines
241241 {
242242 unsigned int sect = nvmet_lba_to_sect(req->ns, req->cmd->zmr.slba);
243243
244− return blkdev_nr_zones(req->ns->bdev->bd_disk) -
244+ return bdev_nr_zones(req->ns->bdev) -
245245 (sect >> ilog2(bdev_zone_sectors(req->ns->bdev)));
246246 }
247247
@@ -308,7 +308,7 @@ void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req) · 60 unmodified lines
308308 queue_work(zbd_wq, &req->z.zmgmt_work);
309309 }
310310
311−static inline enum req_opf zsa_req_op(u8 zsa)
311+static inline enum req_op zsa_req_op(u8 zsa)
312312 {
313313 switch (zsa) {
314314 case NVME_ZONE_OPEN:
@@ -386,7 +386,7 @@ static int zmgmt_send_scan_cb(struct blk_zone *z, unsigned i, void *d) · 71 unmodified lines
386386 static u16 nvmet_bdev_zone_mgmt_emulate_all(struct nvmet_req *req)
387387 {
388388 struct block_device *bdev = req->ns->bdev;
389− unsigned int nr_zones = blkdev_nr_zones(bdev->bd_disk);
389+ unsigned int nr_zones = bdev_nr_zones(bdev);
390390 struct request_queue *q = bdev_get_queue(bdev);
391391 struct bio *bio = NULL;
392392 sector_t sector = 0;
@@ -413,8 +413,8 @@ static u16 nvmet_bdev_zone_mgmt_emulate_all(struct nvmet_req *req) · 20 unmodified lines
413413 ret = 0;
414414 }
415415
416− while (sector < get_capacity(bdev->bd_disk)) {
417− if (test_bit(blk_queue_zone_no(q, sector), d.zbitmap)) {
416+ while (sector < bdev_nr_sectors(bdev)) {
417+ if (test_bit(disk_zone_no(bdev->bd_disk, sector), d.zbitmap)) {
418418 bio = blk_next_bio(bio, bdev, 0,
419419 zsa_req_op(req->cmd->zms.zsa) | REQ_SYNC,
420420 GFP_KERNEL);
@@ -422,7 +422,7 @@ static u16 nvmet_bdev_zone_mgmt_emulate_all(struct nvmet_req *req) · 1 unmodified line
422422 /* This may take a while, so be nice to others */
423423 cond_resched();
424424 }
425− sector += blk_queue_zone_sectors(q);
425+ sector += bdev_zone_sectors(bdev);
426426 }
427427
428428 if (bio) {
@@ -465,7 +465,7 @@ static void nvmet_bdev_zmgmt_send_work(struct work_struct *w) · 36 unmodified lines
465465 {
466466 struct nvmet_req *req = container_of(w, struct nvmet_req, z.zmgmt_work);
467467 sector_t sect = nvmet_lba_to_sect(req->ns, req->cmd->zms.slba);
468− enum req_opf op = zsa_req_op(req->cmd->zms.zsa);
468+ enum req_op op = zsa_req_op(req->cmd->zms.zsa);
469469 struct block_device *bdev = req->ns->bdev;
470470 sector_t zone_sectors = bdev_zone_sectors(bdev);
471471 u16 status = NVME_SC_SUCCESS;
@@ -525,7 +525,7 @@ static void nvmet_bdev_zone_append_bio_done(struct bio *bio) · 53 unmodified lines
525525 void nvmet_bdev_execute_zone_append(struct nvmet_req *req)
526526 {
527527 sector_t sect = nvmet_lba_to_sect(req->ns, req->cmd->rw.slba);
528− const unsigned int op = REQ_OP_ZONE_APPEND | REQ_SYNC | REQ_IDLE;
528+ const blk_opf_t opf = REQ_OP_ZONE_APPEND | REQ_SYNC | REQ_IDLE;
529529 u16 status = NVME_SC_SUCCESS;
530530 unsigned int total_len = 0;
531531 struct scatterlist *sg;
@@ -556,9 +556,9 @@ void nvmet_bdev_execute_zone_append(struct nvmet_req *req) · 24 unmodified lines
556556 if (nvmet_use_inline_bvec(req)) {
557557 bio = &req->z.inline_bio;
558558 bio_init(bio, req->ns->bdev, req->inline_bvec,
559− ARRAY_SIZE(req->inline_bvec), op);
559+ ARRAY_SIZE(req->inline_bvec), opf);
560560 } else {
561− bio = bio_alloc(req->ns->bdev, req->sg_cnt, op, GFP_KERNEL);
561+ bio = bio_alloc(req->ns->bdev, req->sg_cnt, opf, GFP_KERNEL);
562562 }
563563
564564 bio->bi_end_io = nvmet_bdev_zone_append_bio_done;
drivers/s390/block/dasd.c+2−2
@@ -3145,7 +3145,7 @@ out: · 3144 unmodified lines
31453145 * BLK_EH_DONE if the request is handled or terminated
31463146 * by the driver.
31473147 */
3148−enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved)
3148+enum blk_eh_timer_return dasd_times_out(struct request *req)
31493149 {
31503150 struct dasd_block *block = req->q->queuedata;
31513151 struct dasd_device *device;
@@ -3280,7 +3280,7 @@ static int dasd_alloc_queue(struct dasd_block *block) · 128 unmodified lines
32803280 static void dasd_free_queue(struct dasd_block *block)
32813281 {
32823282 if (block->request_queue) {
3283− blk_cleanup_queue(block->request_queue);
3283+ blk_mq_destroy_queue(block->request_queue);
32843284 blk_mq_free_tag_set(&block->tag_set);
32853285 block->request_queue = NULL;
32863286 }
drivers/s390/block/dasd_genhd.c+2−2
@@ -41,8 +41,8 @@ int dasd_gendisk_alloc(struct dasd_block *block) · 40 unmodified lines
4141 if (base->devindex >= DASD_PER_MAJOR)
4242 return -EBUSY;
4343
44− gdp = __alloc_disk_node(block->request_queue, NUMA_NO_NODE,
45− &dasd_bio_compl_lkclass);
44+ gdp = blk_mq_alloc_disk_for_queue(block->request_queue,
45+ &dasd_bio_compl_lkclass);
4646 if (!gdp)
4747 return -ENOMEM;
4848
drivers/s390/block/dasd_int.h+1−1
@@ -795,7 +795,7 @@ void dasd_free_device(struct dasd_device *); · 794 unmodified lines
795795 struct dasd_block *dasd_alloc_block(void);
796796 void dasd_free_block(struct dasd_block *);
797797
798−enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved);
798+enum blk_eh_timer_return dasd_times_out(struct request *req);
799799
800800 void dasd_enable_device(struct dasd_device *);
801801 void dasd_set_target_state(struct dasd_device *, int);
drivers/s390/block/dcssblk.c+4−4
@@ -414,7 +414,7 @@ removeseg: · 413 unmodified lines
414414 kill_dax(dev_info->dax_dev);
415415 put_dax(dev_info->dax_dev);
416416 del_gendisk(dev_info->gd);
417− blk_cleanup_disk(dev_info->gd);
417+ put_disk(dev_info->gd);
418418 up_write(&dcssblk_devices_sem);
419419
420420 if (device_remove_file_self(dev, attr)) {
@@ -712,7 +712,7 @@ out_dax: · 291 unmodified lines
712712 put_dax(dev_info->dax_dev);
713713 put_dev:
714714 list_del(&dev_info->lh);
715− blk_cleanup_disk(dev_info->gd);
715+ put_disk(dev_info->gd);
716716 list_for_each_entry(seg_info, &dev_info->seg_list, lh) {
717717 segment_unload(seg_info->segment_name);
718718 }
@@ -722,7 +722,7 @@ put_dev: · 3 unmodified lines
722722 dev_list_del:
723723 list_del(&dev_info->lh);
724724 release_gd:
725− blk_cleanup_disk(dev_info->gd);
725+ put_disk(dev_info->gd);
726726 up_write(&dcssblk_devices_sem);
727727 seg_list_del:
728728 if (dev_info == NULL)
@@ -790,7 +790,7 @@ dcssblk_remove_store(struct device *dev, struct device_attribute *attr, const ch · 61 unmodified lines
790790 kill_dax(dev_info->dax_dev);
791791 put_dax(dev_info->dax_dev);
792792 del_gendisk(dev_info->gd);
793− blk_cleanup_disk(dev_info->gd);
793+ put_disk(dev_info->gd);
794794
795795 /* unload all related segments */
796796 list_for_each_entry(entry, &dev_info->seg_list, lh)
drivers/s390/block/scm_blk.c+2−2
@@ -501,7 +501,7 @@ int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev) · 500 unmodified lines
501501 return 0;
502502
503503 out_cleanup_disk:
504− blk_cleanup_disk(bdev->gendisk);
504+ put_disk(bdev->gendisk);
505505 out_tag:
506506 blk_mq_free_tag_set(&bdev->tag_set);
507507 out:
@@ -512,7 +512,7 @@ out: · 4 unmodified lines
512512 void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
513513 {
514514 del_gendisk(bdev->gendisk);
515− blk_cleanup_disk(bdev->gendisk);
515+ put_disk(bdev->gendisk);
516516 blk_mq_free_tag_set(&bdev->tag_set);
517517 }
518518
drivers/scsi/aacraid/comminit.c+1−1
@@ -272,7 +272,7 @@ static void aac_queue_init(struct aac_dev * dev, struct aac_queue * q, u32 *mem, · 271 unmodified lines
272272 q->entries = qsize;
273273 }
274274
275−static bool wait_for_io_iter(struct scsi_cmnd *cmd, void *data, bool rsvd)
275+static bool wait_for_io_iter(struct scsi_cmnd *cmd, void *data)
276276 {
277277 int *active = data;
278278
drivers/scsi/aacraid/linit.c+1−1
@@ -633,7 +633,7 @@ struct fib_count_data { · 632 unmodified lines
633633 int krlcnt;
634634 };
635635
636−static bool fib_count_iter(struct scsi_cmnd *scmnd, void *data, bool reserved)
636+static bool fib_count_iter(struct scsi_cmnd *scmnd, void *data)
637637 {
638638 struct fib_count_data *fib_count = data;
639639
drivers/scsi/device_handler/scsi_dh_alua.c+2−2
@@ -127,7 +127,7 @@ static int submit_rtpg(struct scsi_device *sdev, unsigned char *buff, · 126 unmodified lines
127127 int bufflen, struct scsi_sense_hdr *sshdr, int flags)
128128 {
129129 u8 cdb[MAX_COMMAND_SIZE];
130− int req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
130+ blk_opf_t req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
131131 REQ_FAILFAST_DRIVER;
132132
133133 /* Prepare the command. */
@@ -157,7 +157,7 @@ static int submit_stpg(struct scsi_device *sdev, int group_id, · 23 unmodified lines
157157 u8 cdb[MAX_COMMAND_SIZE];
158158 unsigned char stpg_data[8];
159159 int stpg_len = 8;
160− int req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
160+ blk_opf_t req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
161161 REQ_FAILFAST_DRIVER;
162162
163163 /* Prepare the data buffer */
drivers/scsi/device_handler/scsi_dh_emc.c+1−1
@@ -239,7 +239,7 @@ static int send_trespass_cmd(struct scsi_device *sdev, · 238 unmodified lines
239239 unsigned char cdb[MAX_COMMAND_SIZE];
240240 int err, res = SCSI_DH_OK, len;
241241 struct scsi_sense_hdr sshdr;
242− u64 req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
242+ blk_opf_t req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
243243 REQ_FAILFAST_DRIVER;
244244
245245 if (csdev->flags & CLARIION_SHORT_TRESPASS) {
drivers/scsi/device_handler/scsi_dh_hp_sw.c+2−2
@@ -83,7 +83,7 @@ static int hp_sw_tur(struct scsi_device *sdev, struct hp_sw_dh_data *h) · 82 unmodified lines
8383 unsigned char cmd[6] = { TEST_UNIT_READY };
8484 struct scsi_sense_hdr sshdr;
8585 int ret = SCSI_DH_OK, res;
86− u64 req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
86+ blk_opf_t req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
8787 REQ_FAILFAST_DRIVER;
8888
8989 retry:
@@ -121,7 +121,7 @@ static int hp_sw_start_stop(struct hp_sw_dh_data *h) · 31 unmodified lines
121121 struct scsi_device *sdev = h->sdev;
122122 int res, rc = SCSI_DH_OK;
123123 int retry_cnt = HP_SW_RETRIES;
124− u64 req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
124+ blk_opf_t req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
125125 REQ_FAILFAST_DRIVER;
126126
127127 retry:
drivers/scsi/device_handler/scsi_dh_rdac.c+1−1
@@ -536,7 +536,7 @@ static void send_mode_select(struct work_struct *work) · 535 unmodified lines
536536 unsigned char cdb[MAX_COMMAND_SIZE];
537537 struct scsi_sense_hdr sshdr;
538538 unsigned int data_size;
539− u64 req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
539+ blk_opf_t req_flags = REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
540540 REQ_FAILFAST_DRIVER;
541541
542542 spin_lock(&ctlr->ms_lock);
drivers/scsi/fnic/fnic_scsi.c+4−10
@@ -1350,8 +1350,7 @@ int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do) · 1349 unmodified lines
13501350 return wq_work_done;
13511351 }
13521352
1353−static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data,
1354− bool reserved)
1353+static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data)
13551354 {
13561355 const int tag = scsi_cmd_to_rq(sc)->tag;
13571356 struct fnic *fnic = data;
@@ -1548,8 +1547,7 @@ struct fnic_rport_abort_io_iter_data { · 190 unmodified lines
15481547 int term_cnt;
15491548 };
15501549
1551−static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data,
1552− bool reserved)
1550+static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data)
15531551 {
15541552 struct fnic_rport_abort_io_iter_data *iter_data = data;
15551553 struct fnic *fnic = iter_data->fnic;
@@ -2003,8 +2001,7 @@ struct fnic_pending_aborts_iter_data { · 447 unmodified lines
20032001 int ret;
20042002 };
20052003
2006−static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc,
2007− void *data, bool reserved)
2004+static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data)
20082005 {
20092006 struct fnic_pending_aborts_iter_data *iter_data = data;
20102007 struct fnic *fnic = iter_data->fnic;
@@ -2019,8 +2016,6 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, · 8 unmodified lines
20192016
20202017 if (sc == iter_data->lr_sc || sc->device != lun_dev)
20212018 return true;
2022− if (reserved)
2023− return true;
20242019
20252020 io_lock = fnic_io_lock_tag(fnic, abt_tag);
20262021 spin_lock_irqsave(io_lock, flags);
@@ -2670,8 +2665,7 @@ call_fc_exch_mgr_reset: · 643 unmodified lines
26702665
26712666 }
26722667
2673−static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data,
2674− bool reserved)
2668+static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data)
26752669 {
26762670 struct fnic_pending_aborts_iter_data *iter_data = data;
26772671 struct fnic *fnic = iter_data->fnic;
drivers/scsi/hosts.c+6−8
@@ -566,8 +566,7 @@ struct Scsi_Host *scsi_host_get(struct Scsi_Host *shost) · 565 unmodified lines
566566 }
567567 EXPORT_SYMBOL(scsi_host_get);
568568
569−static bool scsi_host_check_in_flight(struct request *rq, void *data,
570− bool reserved)
569+static bool scsi_host_check_in_flight(struct request *rq, void *data)
571570 {
572571 int *count = data;
573572 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
@@ -662,7 +661,7 @@ void scsi_flush_work(struct Scsi_Host *shost) · 88 unmodified lines
662661 }
663662 EXPORT_SYMBOL_GPL(scsi_flush_work);
664663
665−static bool complete_all_cmds_iter(struct request *rq, void *data, bool rsvd)
664+static bool complete_all_cmds_iter(struct request *rq, void *data)
666665 {
667666 struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
668667 enum scsi_host_status status = *(enum scsi_host_status *)data;
@@ -693,17 +692,16 @@ void scsi_host_complete_all_commands(struct Scsi_Host *shost, · 24 unmodified lines
693692 EXPORT_SYMBOL_GPL(scsi_host_complete_all_commands);
694693
695694 struct scsi_host_busy_iter_data {
696− bool (*fn)(struct scsi_cmnd *, void *, bool);
695+ bool (*fn)(struct scsi_cmnd *, void *);
697696 void *priv;
698697 };
699698
700−static bool __scsi_host_busy_iter_fn(struct request *req, void *priv,
701− bool reserved)
699+static bool __scsi_host_busy_iter_fn(struct request *req, void *priv)
702700 {
703701 struct scsi_host_busy_iter_data *iter_data = priv;
704702 struct scsi_cmnd *sc = blk_mq_rq_to_pdu(req);
705703
706− return iter_data->fn(sc, iter_data->priv, reserved);
704+ return iter_data->fn(sc, iter_data->priv);
707705 }
708706
709707 /**
@@ -716,7 +714,7 @@ static bool __scsi_host_busy_iter_fn(struct request *req, void *priv, · 6 unmodified lines
716714 * ithas to be provided by the caller
717715 **/
718716 void scsi_host_busy_iter(struct Scsi_Host *shost,
719− bool (*fn)(struct scsi_cmnd *, void *, bool),
717+ bool (*fn)(struct scsi_cmnd *, void *),
720718 void *priv)
721719 {
722720 struct scsi_host_busy_iter_data iter_data = {
drivers/scsi/mpi3mr/mpi3mr_os.c+4−12
@@ -381,14 +381,12 @@ void mpi3mr_invalidate_devhandles(struct mpi3mr_ioc *mrioc) · 380 unmodified lines
381381 * mpi3mr_print_scmd - print individual SCSI command
382382 * @rq: Block request
383383 * @data: Adapter instance reference
384− * @reserved: N/A. Currently not used
385384 *
386385 * Print the SCSI command details if it is in LLD scope.
387386 *
388387 * Return: true always.
389388 */
390−static bool mpi3mr_print_scmd(struct request *rq,
391− void *data, bool reserved)
389+static bool mpi3mr_print_scmd(struct request *rq, void *data)
392390 {
393391 struct mpi3mr_ioc *mrioc = (struct mpi3mr_ioc *)data;
394392 struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
@@ -412,7 +410,6 @@ out: · 17 unmodified lines
412410 * mpi3mr_flush_scmd - Flush individual SCSI command
413411 * @rq: Block request
414412 * @data: Adapter instance reference
415− * @reserved: N/A. Currently not used
416413 *
417414 * Return the SCSI command to the upper layers if it is in LLD
418415 * scope.
@@ -420,8 +417,7 @@ out: · 1 unmodified line
420417 * Return: true always.
421418 */
422419
423−static bool mpi3mr_flush_scmd(struct request *rq,
424− void *data, bool reserved)
420+static bool mpi3mr_flush_scmd(struct request *rq, void *data)
425421 {
426422 struct mpi3mr_ioc *mrioc = (struct mpi3mr_ioc *)data;
427423 struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
@@ -451,7 +447,6 @@ out: · 23 unmodified lines
451447 * mpi3mr_count_dev_pending - Count commands pending for a lun
452448 * @rq: Block request
453449 * @data: SCSI device reference
454− * @reserved: Unused
455450 *
456451 * This is an iterator function called for each SCSI command in
457452 * a host and if the command is pending in the LLD for the
@@ -461,8 +456,7 @@ out: · 3 unmodified lines
461456 * Return: true always.
462457 */
463458
464−static bool mpi3mr_count_dev_pending(struct request *rq,
465− void *data, bool reserved)
459+static bool mpi3mr_count_dev_pending(struct request *rq, void *data)
466460 {
467461 struct scsi_device *sdev = (struct scsi_device *)data;
468462 struct mpi3mr_sdev_priv_data *sdev_priv_data = sdev->hostdata;
@@ -485,7 +479,6 @@ out: · 16 unmodified lines
485479 * mpi3mr_count_tgt_pending - Count commands pending for target
486480 * @rq: Block request
487481 * @data: SCSI target reference
488− * @reserved: Unused
489482 *
490483 * This is an iterator function called for each SCSI command in
491484 * a host and if the command is pending in the LLD for the
@@ -495,8 +488,7 @@ out: · 3 unmodified lines
495488 * Return: true always.
496489 */
497490
498−static bool mpi3mr_count_tgt_pending(struct request *rq,
499− void *data, bool reserved)
491+static bool mpi3mr_count_tgt_pending(struct request *rq, void *data)
500492 {
501493 struct scsi_target *starget = (struct scsi_target *)data;
502494 struct mpi3mr_stgt_priv_data *stgt_priv_data = starget->hostdata;
drivers/scsi/scsi_error.c+11−11
@@ -139,7 +139,7 @@ static bool scsi_eh_should_retry_cmd(struct scsi_cmnd *cmd) · 138 unmodified lines
139139 *
140140 * Note: this function must be called only for a command that has timed out.
141141 * Because the block layer marks a request as complete before it calls
142− * scsi_times_out(), a .scsi_done() call from the LLD for a command that has
142+ * scsi_timeout(), a .scsi_done() call from the LLD for a command that has
143143 * timed out do not have any effect. Hence it is safe to call
144144 * scsi_finish_command() from this function.
145145 */
@@ -316,7 +316,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd) · 170 unmodified lines
316316 }
317317
318318 /**
319− * scsi_times_out - Timeout function for normal scsi commands.
319+ * scsi_timeout - Timeout function for normal scsi commands.
320320 * @req: request that is timing out.
321321 *
322322 * Notes:
@@ -325,7 +325,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd) · 2 unmodified lines
325325 * normal completion function determines that the timer has already
326326 * fired, then it mustn't do anything.
327327 */
328−enum blk_eh_timer_return scsi_times_out(struct request *req)
328+enum blk_eh_timer_return scsi_timeout(struct request *req)
329329 {
330330 struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req);
331331 enum blk_eh_timer_return rtn = BLK_EH_DONE;
@@ -1779,7 +1779,7 @@ static void scsi_eh_offline_sdevs(struct list_head *work_q, · 1447 unmodified lines
17791779 * scsi_noretry_cmd - determine if command should be failed fast
17801780 * @scmd: SCSI cmd to examine.
17811781 */
1782−int scsi_noretry_cmd(struct scsi_cmnd *scmd)
1782+bool scsi_noretry_cmd(struct scsi_cmnd *scmd)
17831783 {
17841784 struct request *req = scsi_cmd_to_rq(scmd);
17851785
@@ -1789,19 +1789,19 @@ int scsi_noretry_cmd(struct scsi_cmnd *scmd) · 3 unmodified lines
17891789 case DID_TIME_OUT:
17901790 goto check_type;
17911791 case DID_BUS_BUSY:
1792− return req->cmd_flags & REQ_FAILFAST_TRANSPORT;
1792+ return !!(req->cmd_flags & REQ_FAILFAST_TRANSPORT);
17931793 case DID_PARITY:
1794− return req->cmd_flags & REQ_FAILFAST_DEV;
1794+ return !!(req->cmd_flags & REQ_FAILFAST_DEV);
17951795 case DID_ERROR:
17961796 if (get_status_byte(scmd) == SAM_STAT_RESERVATION_CONFLICT)
1797− return 0;
1797+ return false;
17981798 fallthrough;
17991799 case DID_SOFT_ERROR:
1800− return req->cmd_flags & REQ_FAILFAST_DRIVER;
1800+ return !!(req->cmd_flags & REQ_FAILFAST_DRIVER);
18011801 }
18021802
18031803 if (!scsi_status_is_check_condition(scmd->result))
1804− return 0;
1804+ return false;
18051805
18061806 check_type:
18071807 /*
@@ -1809,9 +1809,9 @@ check_type: · 1 unmodified line
18091809 * the check condition was retryable.
18101810 */
18111811 if (req->cmd_flags & REQ_FAILFAST_DEV || blk_rq_is_passthrough(req))
1812− return 1;
1812+ return true;
18131813
1814− return 0;
1814+ return false;
18151815 }
18161816
18171817 /**
drivers/scsi/scsi_lib.c+9−17
@@ -163,7 +163,7 @@ static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy) · 162 unmodified lines
163163 * Requeue this command. It will go before all other commands
164164 * that are already in the queue. Schedule requeue work under
165165 * lock such that the kblockd_schedule_work() call happens
166− * before blk_cleanup_queue() finishes.
166+ * before blk_mq_destroy_queue() finishes.
167167 */
168168 cmd->result = 0;
169169
@@ -209,8 +209,8 @@ void scsi_queue_insert(struct scsi_cmnd *cmd, int reason) · 39 unmodified lines
209209 int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
210210 int data_direction, void *buffer, unsigned bufflen,
211211 unsigned char *sense, struct scsi_sense_hdr *sshdr,
212− int timeout, int retries, u64 flags, req_flags_t rq_flags,
213− int *resid)
212+ int timeout, int retries, blk_opf_t flags,
213+ req_flags_t rq_flags, int *resid)
214214 {
215215 struct request *req;
216216 struct scsi_cmnd *scmd;
@@ -424,9 +424,9 @@ static void scsi_starved_list_run(struct Scsi_Host *shost) · 207 unmodified lines
424424 * it and the queue. Mitigate by taking a reference to the
425425 * queue and never touching the sdev again after we drop the
426426 * host lock. Note: if __scsi_remove_device() invokes
427− * blk_cleanup_queue() before the queue is run from this
427+ * blk_mq_destroy_queue() before the queue is run from this
428428 * function then blk_run_queue() will return immediately since
429− * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING.
429+ * blk_mq_destroy_queue() marks the queue with QUEUE_FLAG_DYING.
430430 */
431431 slq = sdev->request_queue;
432432 if (!blk_get_queue(slq))
@@ -633,7 +633,7 @@ static blk_status_t scsi_result_to_blk_status(struct scsi_cmnd *cmd, int result) · 200 unmodified lines
633633 */
634634 static unsigned int scsi_rq_err_bytes(const struct request *rq)
635635 {
636− unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
636+ blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
637637 unsigned int bytes = 0;
638638 struct bio *bio;
639639
@@ -1125,12 +1125,12 @@ static void scsi_initialize_rq(struct request *rq) · 485 unmodified lines
11251125 cmd->retries = 0;
11261126 }
11271127
1128−struct request *scsi_alloc_request(struct request_queue *q,
1129− unsigned int op, blk_mq_req_flags_t flags)
1128+struct request *scsi_alloc_request(struct request_queue *q, blk_opf_t opf,
1129+ blk_mq_req_flags_t flags)
11301130 {
11311131 struct request *rq;
11321132
1133− rq = blk_mq_alloc_request(q, op, flags);
1133+ rq = blk_mq_alloc_request(q, opf, flags);
11341134 if (!IS_ERR(rq))
11351135 scsi_initialize_rq(rq);
11361136 return rq;
@@ -1790,14 +1790,6 @@ out_put_budget: · 653 unmodified lines
17901790 return ret;
17911791 }
17921792
1793−static enum blk_eh_timer_return scsi_timeout(struct request *req,
1794− bool reserved)
1795−{
1796− if (reserved)
1797− return BLK_EH_RESET_TIMER;
1798− return scsi_times_out(req);
1799−}
1800−
18011793 static int scsi_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
18021794 unsigned int hctx_idx, unsigned int numa_node)
18031795 {
drivers/scsi/scsi_priv.h+2−2
@@ -72,7 +72,7 @@ extern void scsi_exit_devinfo(void); · 71 unmodified lines
7272
7373 /* scsi_error.c */
7474 extern void scmd_eh_abort_handler(struct work_struct *work);
75−extern enum blk_eh_timer_return scsi_times_out(struct request *req);
75+extern enum blk_eh_timer_return scsi_timeout(struct request *req);
7676 extern int scsi_error_handler(void *host);
7777 extern enum scsi_disposition scsi_decide_disposition(struct scsi_cmnd *cmd);
7878 extern void scsi_eh_wakeup(struct Scsi_Host *shost);
@@ -82,7 +82,7 @@ void scsi_eh_ready_devs(struct Scsi_Host *shost, · 3 unmodified lines
8282 struct list_head *done_q);
8383 int scsi_eh_get_sense(struct list_head *work_q,
8484 struct list_head *done_q);
85−int scsi_noretry_cmd(struct scsi_cmnd *scmd);
85+bool scsi_noretry_cmd(struct scsi_cmnd *scmd);
8686 void scsi_eh_done(struct scsi_cmnd *scmd);
8787
8888 /* scsi_lib.c */
drivers/scsi/scsi_sysfs.c+1−1
@@ -1475,7 +1475,7 @@ void __scsi_remove_device(struct scsi_device *sdev) · 1474 unmodified lines
14751475 scsi_device_set_state(sdev, SDEV_DEL);
14761476 mutex_unlock(&sdev->state_mutex);
14771477
1478− blk_cleanup_queue(sdev->request_queue);
1478+ blk_mq_destroy_queue(sdev->request_queue);
14791479 cancel_work_sync(&sdev->requeue_work);
14801480
14811481 if (sdev->host->hostt->slave_destroy)
drivers/scsi/sd.c+5−5
@@ -2934,15 +2934,15 @@ static void sd_read_block_characteristics(struct scsi_disk *sdkp) · 2933 unmodified lines
29342934
29352935 if (sdkp->device->type == TYPE_ZBC) {
29362936 /* Host-managed */
2937− blk_queue_set_zoned(sdkp->disk, BLK_ZONED_HM);
2937+ disk_set_zoned(sdkp->disk, BLK_ZONED_HM);
29382938 } else {
29392939 sdkp->zoned = zoned;
29402940 if (sdkp->zoned == 1) {
29412941 /* Host-aware */
2942− blk_queue_set_zoned(sdkp->disk, BLK_ZONED_HA);
2942+ disk_set_zoned(sdkp->disk, BLK_ZONED_HA);
29432943 } else {
29442944 /* Regular disk or drive managed disk */
2945− blk_queue_set_zoned(sdkp->disk, BLK_ZONED_NONE);
2945+ disk_set_zoned(sdkp->disk, BLK_ZONED_NONE);
29462946 }
29472947 }
29482948
@@ -3440,8 +3440,8 @@ static int sd_probe(struct device *dev) · 491 unmodified lines
34403440 if (!sdkp)
34413441 goto out;
34423442
3443− gd = __alloc_disk_node(sdp->request_queue, NUMA_NO_NODE,
3444− &sd_bio_compl_lkclass);
3443+ gd = blk_mq_alloc_disk_for_queue(sdp->request_queue,
3444+ &sd_bio_compl_lkclass);
34453445 if (!gd)
34463446 goto out_free;
34473447
drivers/scsi/sd_zbc.c+6−6
@@ -529,7 +529,7 @@ static unsigned int sd_zbc_zone_wp_update(struct scsi_cmnd *cmd, · 528 unmodified lines
529529 struct request *rq = scsi_cmd_to_rq(cmd);
530530 struct scsi_disk *sdkp = scsi_disk(rq->q->disk);
531531 unsigned int zno = blk_rq_zone_no(rq);
532− enum req_opf op = req_op(rq);
532+ enum req_op op = req_op(rq);
533533 unsigned long flags;
534534
535535 /*
@@ -855,7 +855,7 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp) · 319 unmodified lines
855855
856856 if (sdkp->zone_info.zone_blocks == zone_blocks &&
857857 sdkp->zone_info.nr_zones == nr_zones &&
858− disk->queue->nr_zones == nr_zones)
858+ disk->nr_zones == nr_zones)
859859 goto unlock;
860860
861861 flags = memalloc_noio_save();
@@ -929,7 +929,7 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, u8 buf[SD_BUF_SIZE]) · 67 unmodified lines
929929 /*
930930 * This can happen for a host aware disk with partitions.
931931 * The block device zone model was already cleared by
932− * blk_queue_set_zoned(). Only free the scsi disk zone
932+ * disk_set_zoned(). Only free the scsi disk zone
933933 * information and exit early.
934934 */
935935 sd_zbc_free_zone_info(sdkp);
@@ -950,10 +950,10 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, u8 buf[SD_BUF_SIZE]) · 14 unmodified lines
950950 blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
951951 blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE);
952952 if (sdkp->zones_max_open == U32_MAX)
953− blk_queue_max_open_zones(q, 0);
953+ disk_set_max_open_zones(disk, 0);
954954 else
955− blk_queue_max_open_zones(q, sdkp->zones_max_open);
956− blk_queue_max_active_zones(q, 0);
955+ disk_set_max_open_zones(disk, sdkp->zones_max_open);
956+ disk_set_max_active_zones(disk, 0);
957957 nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks);
958958
959959 /*
drivers/scsi/sr.c+2−2
@@ -624,8 +624,8 @@ static int sr_probe(struct device *dev) · 623 unmodified lines
624624 if (!cd)
625625 goto fail;
626626
627− disk = __alloc_disk_node(sdev->request_queue, NUMA_NO_NODE,
628− &sr_bio_compl_lkclass);
627+ disk = blk_mq_alloc_disk_for_queue(sdev->request_queue,
628+ &sr_bio_compl_lkclass);
629629 if (!disk)
630630 goto fail_free;
631631 mutex_init(&cd->lock);
drivers/target/target_core_iblock.c+2−2
@@ -343,7 +343,7 @@ static void iblock_bio_done(struct bio *bio) · 342 unmodified lines
343343 }
344344
345345 static struct bio *iblock_get_bio(struct se_cmd *cmd, sector_t lba, u32 sg_num,
346− unsigned int opf)
346+ blk_opf_t opf)
347347 {
348348 struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
349349 struct bio *bio;
@@ -723,7 +723,7 @@ iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, · 373 unmodified lines
723723 struct bio_list list;
724724 struct scatterlist *sg;
725725 u32 sg_num = sgl_nents;
726− unsigned int opf;
726+ blk_opf_t opf;
727727 unsigned bio_cnt;
728728 int i, rc;
729729 struct sg_mapping_iter prot_miter;
drivers/ufs/core/ufshcd.c+2−2
@@ -9509,7 +9509,7 @@ void ufshcd_remove(struct ufs_hba *hba) · 9508 unmodified lines
95099509 ufs_bsg_remove(hba);
95109510 ufshpb_remove(hba);
95119511 ufs_sysfs_remove_nodes(hba->dev);
9512− blk_cleanup_queue(hba->tmf_queue);
9512+ blk_mq_destroy_queue(hba->tmf_queue);
95139513 blk_mq_free_tag_set(&hba->tmf_tag_set);
95149514 scsi_remove_host(hba->host);
95159515 /* disable interrupts */
@@ -9805,7 +9805,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) · 289 unmodified lines
98059805 return 0;
98069806
98079807 free_tmf_queue:
9808− blk_cleanup_queue(hba->tmf_queue);
9808+ blk_mq_destroy_queue(hba->tmf_queue);
98099809 free_tmf_tag_set:
98109810 blk_mq_free_tag_set(&hba->tmf_tag_set);
98119811 out_remove_scsi_host:
drivers/ufs/core/ufshpb.c+3−4
@@ -433,9 +433,8 @@ int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) · 432 unmodified lines
433433 return 0;
434434 }
435435
436−static struct ufshpb_req *ufshpb_get_req(struct ufshpb_lu *hpb,
437− int rgn_idx, enum req_opf dir,
438− bool atomic)
436+static struct ufshpb_req *ufshpb_get_req(struct ufshpb_lu *hpb, int rgn_idx,
437+ enum req_op op, bool atomic)
439438 {
440439 struct ufshpb_req *rq;
441440 struct request *req;
@@ -446,7 +445,7 @@ static struct ufshpb_req *ufshpb_get_req(struct ufshpb_lu *hpb, · 4 unmodified lines
446445 return NULL;
447446
448447 retry:
449− req = blk_mq_alloc_request(hpb->sdev_ufs_lu->request_queue, dir,
448+ req = blk_mq_alloc_request(hpb->sdev_ufs_lu->request_queue, op,
450449 BLK_MQ_REQ_NOWAIT);
451450
452451 if (!atomic && (PTR_ERR(req) == -EWOULDBLOCK) && (--retries > 0)) {
fs/btrfs/check-integrity.c+2−2
@@ -152,7 +152,7 @@ struct btrfsic_block { · 151 unmodified lines
152152 struct btrfsic_block *next_in_same_bio;
153153 void *orig_bio_private;
154154 bio_end_io_t *orig_bio_end_io;
155− int submit_bio_bh_rw;
155+ blk_opf_t submit_bio_bh_rw;
156156 u64 flush_gen; /* only valid if !never_written */
157157 };
158158
@@ -1681,7 +1681,7 @@ static void btrfsic_process_written_block(struct btrfsic_dev_state *dev_state, · 1522 unmodified lines
16811681 u64 dev_bytenr, char **mapped_datav,
16821682 unsigned int num_pages,
16831683 struct bio *bio, int *bio_is_patched,
1684− int submit_bio_bh_rw)
1684+ blk_opf_t submit_bio_bh_rw)
16851685 {
16861686 int is_metadata;
16871687 struct btrfsic_block *block;
fs/btrfs/compression.c+3−3
@@ -455,7 +455,7 @@ static blk_status_t submit_compressed_bio(struct btrfs_fs_info *fs_info, · 454 unmodified lines
455455
456456
457457 static struct bio *alloc_compressed_bio(struct compressed_bio *cb, u64 disk_bytenr,
458− unsigned int opf, bio_end_io_t endio_func,
458+ blk_opf_t opf, bio_end_io_t endio_func,
459459 u64 *next_stripe_start)
460460 {
461461 struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb);
@@ -505,7 +505,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, · 43 unmodified lines
505505 unsigned int compressed_len,
506506 struct page **compressed_pages,
507507 unsigned int nr_pages,
508− unsigned int write_flags,
508+ blk_opf_t write_flags,
509509 struct cgroup_subsys_state *blkcg_css,
510510 bool writeback)
511511 {
@@ -517,7 +517,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, · 5 unmodified lines
517517 blk_status_t ret;
518518 int skip_sum = inode->flags & BTRFS_INODE_NODATASUM;
519519 const bool use_append = btrfs_use_zone_append(inode, disk_start);
520− const unsigned int bio_op = use_append ? REQ_OP_ZONE_APPEND : REQ_OP_WRITE;
520+ const enum req_op bio_op = use_append ? REQ_OP_ZONE_APPEND : REQ_OP_WRITE;
521521
522522 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
523523 IS_ALIGNED(len, fs_info->sectorsize));
fs/btrfs/compression.h+1−1
@@ -99,7 +99,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, · 98 unmodified lines
9999 unsigned int compressed_len,
100100 struct page **compressed_pages,
101101 unsigned int nr_pages,
102− unsigned int write_flags,
102+ blk_opf_t write_flags,
103103 struct cgroup_subsys_state *blkcg_css,
104104 bool writeback);
105105 void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
fs/btrfs/extent_io.c+9−9
@@ -3359,7 +3359,7 @@ static int calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, · 3358 unmodified lines
33593359 static int alloc_new_bio(struct btrfs_inode *inode,
33603360 struct btrfs_bio_ctrl *bio_ctrl,
33613361 struct writeback_control *wbc,
3362− unsigned int opf,
3362+ blk_opf_t opf,
33633363 bio_end_io_t end_io_func,
33643364 u64 disk_bytenr, u32 offset, u64 file_offset,
33653365 enum btrfs_compression_type compress_type)
@@ -3439,7 +3439,7 @@ error: · 73 unmodified lines
34393439 * @prev_bio_flags: flags of previous bio to see if we can merge the current one
34403440 * @compress_type: compress type for current bio
34413441 */
3442−static int submit_extent_page(unsigned int opf,
3442+static int submit_extent_page(blk_opf_t opf,
34433443 struct writeback_control *wbc,
34443444 struct btrfs_bio_ctrl *bio_ctrl,
34453445 struct page *page, u64 disk_bytenr,
@@ -3617,7 +3617,7 @@ __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset, · 171 unmodified lines
36173617 */
36183618 static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
36193619 struct btrfs_bio_ctrl *bio_ctrl,
3620− unsigned int read_flags, u64 *prev_em_start)
3620+ blk_opf_t read_flags, u64 *prev_em_start)
36213621 {
36223622 struct inode *inode = page->mapping->host;
36233623 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@@ -3985,8 +3985,8 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, · 361 unmodified lines
39853985 int saved_ret = 0;
39863986 int ret = 0;
39873987 int nr = 0;
3988− u32 opf = REQ_OP_WRITE;
3989− const unsigned int write_flags = wbc_to_write_flags(wbc);
3988+ enum req_op op = REQ_OP_WRITE;
3989+ const blk_opf_t write_flags = wbc_to_write_flags(wbc);
39903990 bool has_error = false;
39913991 bool compressed;
39923992
@@ -4060,7 +4060,7 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, · 67 unmodified lines
40604060 iosize = min(min(em_end, end + 1), dirty_range_end) - cur;
40614061
40624062 if (btrfs_use_zone_append(inode, em->block_start))
4063− opf = REQ_OP_ZONE_APPEND;
4063+ op = REQ_OP_ZONE_APPEND;
40644064
40654065 free_extent_map(em);
40664066 em = NULL;
@@ -4096,7 +4096,7 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, · 29 unmodified lines
40964096 */
40974097 btrfs_page_clear_dirty(fs_info, page, cur, iosize);
40984098
4099− ret = submit_extent_page(opf | write_flags, wbc,
4099+ ret = submit_extent_page(op | write_flags, wbc,
41004100 &epd->bio_ctrl, page,
41014101 disk_bytenr, iosize,
41024102 cur - page_offset(page),
@@ -4577,7 +4577,7 @@ static int write_one_subpage_eb(struct extent_buffer *eb, · 474 unmodified lines
45774577 {
45784578 struct btrfs_fs_info *fs_info = eb->fs_info;
45794579 struct page *page = eb->pages[0];
4580− unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META;
4580+ blk_opf_t write_flags = wbc_to_write_flags(wbc) | REQ_META;
45814581 bool no_dirty_ebs = false;
45824582 int ret;
45834583
@@ -4622,7 +4622,7 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, · 38 unmodified lines
46224622 {
46234623 u64 disk_bytenr = eb->start;
46244624 int i, num_pages;
4625− unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META;
4625+ blk_opf_t write_flags = wbc_to_write_flags(wbc) | REQ_META;
46264626 int ret = 0;
46274627
46284628 prepare_eb_write(eb);
fs/btrfs/inode.c+2−2
@@ -485,7 +485,7 @@ struct async_chunk { · 484 unmodified lines
485485 struct page *locked_page;
486486 u64 start;
487487 u64 end;
488− unsigned int write_flags;
488+ blk_opf_t write_flags;
489489 struct list_head extents;
490490 struct cgroup_subsys_state *blkcg_css;
491491 struct btrfs_work work;
@@ -1435,7 +1435,7 @@ static int cow_file_range_async(struct btrfs_inode *inode, · 943 unmodified lines
14351435 int i;
14361436 bool should_compress;
14371437 unsigned nofs_flag;
1438− const unsigned int write_flags = wbc_to_write_flags(wbc);
1438+ const blk_opf_t write_flags = wbc_to_write_flags(wbc);
14391439
14401440 unlock_extent(&inode->io_tree, start, end);
14411441
fs/btrfs/raid56.c+2−2
@@ -1136,7 +1136,7 @@ static int rbio_add_io_sector(struct btrfs_raid_bio *rbio, · 1135 unmodified lines
11361136 unsigned int stripe_nr,
11371137 unsigned int sector_nr,
11381138 unsigned long bio_max_len,
1139− unsigned int opf)
1139+ enum req_op op)
11401140 {
11411141 const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
11421142 struct bio *last = bio_list->tail;
@@ -1181,7 +1181,7 @@ static int rbio_add_io_sector(struct btrfs_raid_bio *rbio, · 38 unmodified lines
11811181
11821182 /* put a new bio on the list */
11831183 bio = bio_alloc(stripe->dev->bdev, max(bio_max_len >> PAGE_SHIFT, 1UL),
1184− opf, GFP_NOFS);
1184+ op, GFP_NOFS);
11851185 bio->bi_iter.bi_sector = disk_start >> 9;
11861186 bio->bi_private = rbio;
11871187
fs/buffer.c+31−30
@@ -52,7 +52,7 @@ · 51 unmodified lines
5252 #include "internal.h"
5353
5454 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
55−static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
55+static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
5656 struct writeback_control *wbc);
5757
5858 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
@@ -562,7 +562,7 @@ void write_boundary_block(struct block_device *bdev, · 503 unmodified lines
562562 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
563563 if (bh) {
564564 if (buffer_dirty(bh))
565− ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
565+ ll_rw_block(REQ_OP_WRITE, 1, &bh);
566566 put_bh(bh);
567567 }
568568 }
@@ -1174,7 +1174,7 @@ static struct buffer_head *__bread_slow(struct buffer_head *bh) · 605 unmodified lines
11741174 } else {
11751175 get_bh(bh);
11761176 bh->b_end_io = end_buffer_read_sync;
1177− submit_bh(REQ_OP_READ, 0, bh);
1177+ submit_bh(REQ_OP_READ, bh);
11781178 wait_on_buffer(bh);
11791179 if (buffer_uptodate(bh))
11801180 return bh;
@@ -1342,7 +1342,7 @@ void __breadahead(struct block_device *bdev, sector_t block, unsigned size) · 161 unmodified lines
13421342 {
13431343 struct buffer_head *bh = __getblk(bdev, block, size);
13441344 if (likely(bh)) {
1345− ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1345+ ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh);
13461346 brelse(bh);
13471347 }
13481348 }
@@ -1353,7 +1353,7 @@ void __breadahead_gfp(struct block_device *bdev, sector_t block, unsigned size, · 4 unmodified lines
13531353 {
13541354 struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
13551355 if (likely(bh)) {
1356− ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1356+ ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh);
13571357 brelse(bh);
13581358 }
13591359 }
@@ -1716,7 +1716,7 @@ int __block_write_full_page(struct inode *inode, struct page *page, · 356 unmodified lines
17161716 struct buffer_head *bh, *head;
17171717 unsigned int blocksize, bbits;
17181718 int nr_underway = 0;
1719− int write_flags = wbc_to_write_flags(wbc);
1719+ blk_opf_t write_flags = wbc_to_write_flags(wbc);
17201720
17211721 head = create_page_buffers(page, inode,
17221722 (1 << BH_Dirty)|(1 << BH_Uptodate));
@@ -1804,7 +1804,7 @@ int __block_write_full_page(struct inode *inode, struct page *page, · 81 unmodified lines
18041804 do {
18051805 struct buffer_head *next = bh->b_this_page;
18061806 if (buffer_async_write(bh)) {
1807− submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, wbc);
1807+ submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc);
18081808 nr_underway++;
18091809 }
18101810 bh = next;
@@ -1858,7 +1858,7 @@ recover: · 47 unmodified lines
18581858 struct buffer_head *next = bh->b_this_page;
18591859 if (buffer_async_write(bh)) {
18601860 clear_buffer_dirty(bh);
1861− submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, wbc);
1861+ submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc);
18621862 nr_underway++;
18631863 }
18641864 bh = next;
@@ -2033,7 +2033,7 @@ int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len, · 168 unmodified lines
20332033 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
20342034 !buffer_unwritten(bh) &&
20352035 (block_start < from || block_end > to)) {
2036− ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2036+ ll_rw_block(REQ_OP_READ, 1, &bh);
20372037 *wait_bh++=bh;
20382038 }
20392039 }
@@ -2334,7 +2334,7 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block) · 294 unmodified lines
23342334 if (buffer_uptodate(bh))
23352335 end_buffer_async_read(bh, 1);
23362336 else
2337− submit_bh(REQ_OP_READ, 0, bh);
2337+ submit_bh(REQ_OP_READ, bh);
23382338 }
23392339 return 0;
23402340 }
@@ -2665,7 +2665,7 @@ int nobh_write_begin(struct address_space *mapping, loff_t pos, unsigned len, · 324 unmodified lines
26652665 if (block_start < from || block_end > to) {
26662666 lock_buffer(bh);
26672667 bh->b_end_io = end_buffer_read_nobh;
2668− submit_bh(REQ_OP_READ, 0, bh);
2668+ submit_bh(REQ_OP_READ, bh);
26692669 nr_reads++;
26702670 }
26712671 }
@@ -2915,7 +2915,7 @@ int block_truncate_page(struct address_space *mapping, · 243 unmodified lines
29152915
29162916 if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
29172917 err = -EIO;
2918− ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2918+ ll_rw_block(REQ_OP_READ, 1, &bh);
29192919 wait_on_buffer(bh);
29202920 /* Uhhuh. Read error. Complain and punt. */
29212921 if (!buffer_uptodate(bh))
@@ -2994,9 +2994,10 @@ static void end_bio_bh_io_sync(struct bio *bio) · 72 unmodified lines
29942994 bio_put(bio);
29952995 }
29962996
2997−static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
2997+static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
29982998 struct writeback_control *wbc)
29992999 {
3000+ const enum req_op op = opf & REQ_OP_MASK;
30003001 struct bio *bio;
30013002
30023003 BUG_ON(!buffer_locked(bh));
@@ -3012,11 +3013,11 @@ static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, · 9 unmodified lines
30123013 clear_buffer_write_io_error(bh);
30133014
30143015 if (buffer_meta(bh))
3015− op_flags |= REQ_META;
3016+ opf |= REQ_META;
30163017 if (buffer_prio(bh))
3017− op_flags |= REQ_PRIO;
3018+ opf |= REQ_PRIO;
30183019
3019− bio = bio_alloc(bh->b_bdev, 1, op | op_flags, GFP_NOIO);
3020+ bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
30203021
30213022 fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
30223023
@@ -3040,22 +3041,21 @@ static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, · 17 unmodified lines
30403041 return 0;
30413042 }
30423043
3043−int submit_bh(int op, int op_flags, struct buffer_head *bh)
3044+int submit_bh(blk_opf_t opf, struct buffer_head *bh)
30443045 {
3045− return submit_bh_wbc(op, op_flags, bh, NULL);
3046+ return submit_bh_wbc(opf, bh, NULL);
30463047 }
30473048 EXPORT_SYMBOL(submit_bh);
30483049
30493050 /**
30503051 * ll_rw_block: low-level access to block devices (DEPRECATED)
3051− * @op: whether to %READ or %WRITE
3052− * @op_flags: req_flag_bits
3052+ * @opf: block layer request operation and flags.
30533053 * @nr: number of &struct buffer_heads in the array
30543054 * @bhs: array of pointers to &struct buffer_head
30553055 *
30563056 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
30573057 * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
3058− * @op_flags contains flags modifying the detailed I/O behavior, most notably
3058+ * @opf contains flags modifying the detailed I/O behavior, most notably
30593059 * %REQ_RAHEAD.
30603060 *
30613061 * This function drops any buffer that it cannot get a lock on (with the
@@ -3072,8 +3072,9 @@ EXPORT_SYMBOL(submit_bh); · 10 unmodified lines
30723072 * All of the buffers must be for the same device, and must also be a
30733073 * multiple of the current approved size for the device.
30743074 */
3075−void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[])
3075+void ll_rw_block(const blk_opf_t opf, int nr, struct buffer_head *bhs[])
30763076 {
3077+ const enum req_op op = opf & REQ_OP_MASK;
30773078 int i;
30783079
30793080 for (i = 0; i < nr; i++) {
@@ -3081,18 +3082,18 @@ void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[]) · 1 unmodified line
30813082
30823083 if (!trylock_buffer(bh))
30833084 continue;
3084− if (op == WRITE) {
3085+ if (op == REQ_OP_WRITE) {
30853086 if (test_clear_buffer_dirty(bh)) {
30863087 bh->b_end_io = end_buffer_write_sync;
30873088 get_bh(bh);
3088− submit_bh(op, op_flags, bh);
3089+ submit_bh(opf, bh);
30893090 continue;
30903091 }
30913092 } else {
30923093 if (!buffer_uptodate(bh)) {
30933094 bh->b_end_io = end_buffer_read_sync;
30943095 get_bh(bh);
3095− submit_bh(op, op_flags, bh);
3096+ submit_bh(opf, bh);
30963097 continue;
30973098 }
30983099 }
@@ -3101,7 +3102,7 @@ void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[]) · 2 unmodified lines
31013102 }
31023103 EXPORT_SYMBOL(ll_rw_block);
31033104
3104−void write_dirty_buffer(struct buffer_head *bh, int op_flags)
3105+void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
31053106 {
31063107 lock_buffer(bh);
31073108 if (!test_clear_buffer_dirty(bh)) {
@@ -3110,7 +3111,7 @@ void write_dirty_buffer(struct buffer_head *bh, int op_flags) · 2 unmodified lines
31103111 }
31113112 bh->b_end_io = end_buffer_write_sync;
31123113 get_bh(bh);
3113− submit_bh(REQ_OP_WRITE, op_flags, bh);
3114+ submit_bh(REQ_OP_WRITE | op_flags, bh);
31143115 }
31153116 EXPORT_SYMBOL(write_dirty_buffer);
31163117
@@ -3119,7 +3120,7 @@ EXPORT_SYMBOL(write_dirty_buffer); · 2 unmodified lines
31193120 * and then start new I/O and then wait upon it. The caller must have a ref on
31203121 * the buffer_head.
31213122 */
3122−int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
3123+int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
31233124 {
31243125 int ret = 0;
31253126
@@ -3137,7 +3138,7 @@ int __sync_dirty_buffer(struct buffer_head *bh, int op_flags) · 11 unmodified lines
31373138
31383139 get_bh(bh);
31393140 bh->b_end_io = end_buffer_write_sync;
3140− ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
3141+ ret = submit_bh(REQ_OP_WRITE | op_flags, bh);
31413142 wait_on_buffer(bh);
31423143 if (!ret && !buffer_uptodate(bh))
31433144 ret = -EIO;
@@ -3365,7 +3366,7 @@ int bh_submit_read(struct buffer_head *bh) · 221 unmodified lines
33653366
33663367 get_bh(bh);
33673368 bh->b_end_io = end_buffer_read_sync;
3368− submit_bh(REQ_OP_READ, 0, bh);
3369+ submit_bh(REQ_OP_READ, bh);
33693370 wait_on_buffer(bh);
33703371 if (buffer_uptodate(bh))
33713372 return 0;
fs/direct-io.c+23−17
@@ -117,8 +117,7 @@ struct dio_submit { · 116 unmodified lines
117117 /* dio_state communicated between submission path and end_io */
118118 struct dio {
119119 int flags; /* doesn't change */
120− int op;
121− int op_flags;
120+ blk_opf_t opf; /* request operation type and flags */
122121 struct gendisk *bio_disk;
123122 struct inode *inode;
124123 loff_t i_size; /* i_size when submitted */
@@ -167,12 +166,13 @@ static inline unsigned dio_pages_present(struct dio_submit *sdio) · 42 unmodified lines
167166 */
168167 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)
169168 {
169+ const enum req_op dio_op = dio->opf & REQ_OP_MASK;
170170 ssize_t ret;
171171
172172 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES,
173173 &sdio->from);
174174
175− if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) {
175+ if (ret < 0 && sdio->blocks_available && dio_op == REQ_OP_WRITE) {
176176 struct page *page = ZERO_PAGE(0);
177177 /*
178178 * A memory fault, but the filesystem has some outstanding
@@ -234,6 +234,7 @@ static inline struct page *dio_get_page(struct dio *dio, · 55 unmodified lines
234234 */
235235 static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags)
236236 {
237+ const enum req_op dio_op = dio->opf & REQ_OP_MASK;
237238 loff_t offset = dio->iocb->ki_pos;
238239 ssize_t transferred = 0;
239240 int err;
@@ -251,7 +252,7 @@ static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) · 11 unmodified lines
251252 transferred = dio->result;
252253
253254 /* Check for short read case */
254− if ((dio->op == REQ_OP_READ) &&
255+ if (dio_op == REQ_OP_READ &&
255256 ((offset + transferred) > dio->i_size))
256257 transferred = dio->i_size - offset;
257258 /* ignore EFAULT if some IO has been done */
@@ -286,7 +287,7 @@ static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) · 28 unmodified lines
286287 * zeros from unwritten extents.
287288 */
288289 if (flags & DIO_COMPLETE_INVALIDATE &&
289− ret > 0 && dio->op == REQ_OP_WRITE &&
290+ ret > 0 && dio_op == REQ_OP_WRITE &&
290291 dio->inode->i_mapping->nrpages) {
291292 err = invalidate_inode_pages2_range(dio->inode->i_mapping,
292293 offset >> PAGE_SHIFT,
@@ -305,7 +306,7 @@ static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) · 12 unmodified lines
305306 */
306307 dio->iocb->ki_pos += transferred;
307308
308− if (ret > 0 && dio->op == REQ_OP_WRITE)
309+ if (ret > 0 && dio_op == REQ_OP_WRITE)
309310 ret = generic_write_sync(dio->iocb, ret);
310311 dio->iocb->ki_complete(dio->iocb, ret);
311312 }
@@ -329,6 +330,7 @@ static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio); · 17 unmodified lines
329330 static void dio_bio_end_aio(struct bio *bio)
330331 {
331332 struct dio *dio = bio->bi_private;
333+ const enum req_op dio_op = dio->opf & REQ_OP_MASK;
332334 unsigned long remaining;
333335 unsigned long flags;
334336 bool defer_completion = false;
@@ -353,7 +355,7 @@ static void dio_bio_end_aio(struct bio *bio) · 18 unmodified lines
353355 */
354356 if (dio->result)
355357 defer_completion = dio->defer_completion ||
356− (dio->op == REQ_OP_WRITE &&
358+ (dio_op == REQ_OP_WRITE &&
357359 dio->inode->i_mapping->nrpages);
358360 if (defer_completion) {
359361 INIT_WORK(&dio->complete_work, dio_aio_complete_work);
@@ -396,7 +398,7 @@ dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, · 36 unmodified lines
396398 * bio_alloc() is guaranteed to return a bio when allowed to sleep and
397399 * we request a valid number of vectors.
398400 */
399− bio = bio_alloc(bdev, nr_vecs, dio->op | dio->op_flags, GFP_KERNEL);
401+ bio = bio_alloc(bdev, nr_vecs, dio->opf, GFP_KERNEL);
400402 bio->bi_iter.bi_sector = first_sector;
401403 if (dio->is_async)
402404 bio->bi_end_io = dio_bio_end_aio;
@@ -415,6 +417,7 @@ dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, · 12 unmodified lines
415417 */
416418 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
417419 {
420+ const enum req_op dio_op = dio->opf & REQ_OP_MASK;
418421 struct bio *bio = sdio->bio;
419422 unsigned long flags;
420423
@@ -426,7 +429,7 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) · 5 unmodified lines
426429 dio->refcount++;
427430 spin_unlock_irqrestore(&dio->bio_lock, flags);
428431
429− if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty)
432+ if (dio->is_async && dio_op == REQ_OP_READ && dio->should_dirty)
430433 bio_set_pages_dirty(bio);
431434
432435 dio->bio_disk = bio->bi_bdev->bd_disk;
@@ -492,7 +495,8 @@ static struct bio *dio_await_one(struct dio *dio) · 59 unmodified lines
492495 static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)
493496 {
494497 blk_status_t err = bio->bi_status;
495− bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty;
498+ const enum req_op dio_op = dio->opf & REQ_OP_MASK;
499+ bool should_dirty = dio_op == REQ_OP_READ && dio->should_dirty;
496500
497501 if (err) {
498502 if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT))
@@ -619,6 +623,7 @@ static int dio_set_defer_completion(struct dio *dio) · 120 unmodified lines
619623 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,
620624 struct buffer_head *map_bh)
621625 {
626+ const enum req_op dio_op = dio->opf & REQ_OP_MASK;
622627 int ret;
623628 sector_t fs_startblk; /* Into file, in filesystem-sized blocks */
624629 sector_t fs_endblk; /* Into file, in filesystem-sized blocks */
@@ -653,7 +658,7 @@ static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, · 28 unmodified lines
653658 * which may decide to handle it or also return an unmapped
654659 * buffer head.
655660 */
656− create = dio->op == REQ_OP_WRITE;
661+ create = dio_op == REQ_OP_WRITE;
657662 if (dio->flags & DIO_SKIP_HOLES) {
658663 i_size = i_size_read(dio->inode);
659664 if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits)
@@ -801,10 +806,11 @@ submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, · 141 unmodified lines
801806 unsigned offset, unsigned len, sector_t blocknr,
802807 struct buffer_head *map_bh)
803808 {
809+ const enum req_op dio_op = dio->opf & REQ_OP_MASK;
804810 int ret = 0;
805811 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */
806812
807− if (dio->op == REQ_OP_WRITE) {
813+ if (dio_op == REQ_OP_WRITE) {
808814 /*
809815 * Read accounting is performed in submit_bio()
810816 */
@@ -917,6 +923,7 @@ static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, · 106 unmodified lines
917923 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,
918924 struct buffer_head *map_bh)
919925 {
926+ const enum req_op dio_op = dio->opf & REQ_OP_MASK;
920927 const unsigned blkbits = sdio->blkbits;
921928 const unsigned i_blkbits = blkbits + sdio->blkfactor;
922929 int ret = 0;
@@ -992,7 +999,7 @@ do_holes: · 69 unmodified lines
992999 loff_t i_size_aligned;
9931000
9941001 /* AKPM: eargh, -ENOTBLK is a hack */
995− if (dio->op == REQ_OP_WRITE) {
1002+ if (dio_op == REQ_OP_WRITE) {
9961003 put_page(page);
9971004 return -ENOTBLK;
9981005 }
@@ -1196,12 +1203,11 @@ ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode, · 197 unmodified lines
11961203
11971204 dio->inode = inode;
11981205 if (iov_iter_rw(iter) == WRITE) {
1199− dio->op = REQ_OP_WRITE;
1200− dio->op_flags = REQ_SYNC | REQ_IDLE;
1206+ dio->opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
12011207 if (iocb->ki_flags & IOCB_NOWAIT)
1202− dio->op_flags |= REQ_NOWAIT;
1208+ dio->opf |= REQ_NOWAIT;
12031209 } else {
1204− dio->op = REQ_OP_READ;
1210+ dio->opf = REQ_OP_READ;
12051211 }
12061212
12071213 /*
fs/ext4/ext4.h+4−4
@@ -3058,14 +3058,14 @@ extern unsigned int ext4_list_backups(struct super_block *sb, · 3057 unmodified lines
30583058
30593059 /* super.c */
30603060 extern struct buffer_head *ext4_sb_bread(struct super_block *sb,
3061− sector_t block, int op_flags);
3061+ sector_t block, blk_opf_t op_flags);
30623062 extern struct buffer_head *ext4_sb_bread_unmovable(struct super_block *sb,
30633063 sector_t block);
3064−extern void ext4_read_bh_nowait(struct buffer_head *bh, int op_flags,
3064+extern void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags,
30653065 bh_end_io_t *end_io);
3066−extern int ext4_read_bh(struct buffer_head *bh, int op_flags,
3066+extern int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags,
30673067 bh_end_io_t *end_io);
3068−extern int ext4_read_bh_lock(struct buffer_head *bh, int op_flags, bool wait);
3068+extern int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait);
30693069 extern void ext4_sb_breadahead_unmovable(struct super_block *sb, sector_t block);
30703070 extern int ext4_seq_options_show(struct seq_file *seq, void *offset);
30713071 extern int ext4_calculate_overhead(struct super_block *sb);
fs/ext4/fast_commit.c+2−2
@@ -658,7 +658,7 @@ void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t star · 657 unmodified lines
658658
659659 static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)
660660 {
661− int write_flags = REQ_SYNC;
661+ blk_opf_t write_flags = REQ_SYNC;
662662 struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
663663
664664 /* Add REQ_FUA | REQ_PREFLUSH only its tail */
@@ -668,7 +668,7 @@ static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) · 3 unmodified lines
668668 set_buffer_dirty(bh);
669669 set_buffer_uptodate(bh);
670670 bh->b_end_io = ext4_end_buffer_io_sync;
671− submit_bh(REQ_OP_WRITE, write_flags, bh);
671+ submit_bh(REQ_OP_WRITE | write_flags, bh);
672672 EXT4_SB(sb)->s_fc_bh = NULL;
673673 }
674674
fs/ext4/mmp.c+6−5
@@ -52,7 +52,7 @@ static int write_mmp_block(struct super_block *sb, struct buffer_head *bh) · 51 unmodified lines
5252 lock_buffer(bh);
5353 bh->b_end_io = end_buffer_write_sync;
5454 get_bh(bh);
55− submit_bh(REQ_OP_WRITE, REQ_SYNC | REQ_META | REQ_PRIO, bh);
55+ submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, bh);
5656 wait_on_buffer(bh);
5757 sb_end_write(sb);
5858 if (unlikely(!buffer_uptodate(bh)))
@@ -150,8 +150,6 @@ static int kmmpd(void *data) · 91 unmodified lines
150150 mmp_check_interval = max(EXT4_MMP_CHECK_MULT * mmp_update_interval,
151151 EXT4_MMP_MIN_CHECK_INTERVAL);
152152 mmp->mmp_check_interval = cpu_to_le16(mmp_check_interval);
153− BUILD_BUG_ON(sizeof(mmp->mmp_bdevname) < BDEVNAME_SIZE);
154− bdevname(bh->b_bdev, mmp->mmp_bdevname);
155153
156154 memcpy(mmp->mmp_nodename, init_utsname()->nodename,
157155 sizeof(mmp->mmp_nodename));
@@ -372,13 +370,16 @@ skip: · 214 unmodified lines
372370
373371 EXT4_SB(sb)->s_mmp_bh = bh;
374372
373+ BUILD_BUG_ON(sizeof(mmp->mmp_bdevname) < BDEVNAME_SIZE);
374+ snprintf(mmp->mmp_bdevname, sizeof(mmp->mmp_bdevname),
375+ "%pg", bh->b_bdev);
376+
375377 /*
376378 * Start a kernel thread to update the MMP block periodically.
377379 */
378380 EXT4_SB(sb)->s_mmp_tsk = kthread_run(kmmpd, sb, "kmmpd-%.*s",
379381 (int)sizeof(mmp->mmp_bdevname),
380− bdevname(bh->b_bdev,
381− mmp->mmp_bdevname));
382+ mmp->mmp_bdevname);
382383 if (IS_ERR(EXT4_SB(sb)->s_mmp_tsk)) {
383384 EXT4_SB(sb)->s_mmp_tsk = NULL;
384385 ext4_warning(sb, "Unable to create kmmpd thread for %s.",
fs/ext4/super.c+10−10
@@ -159,7 +159,7 @@ MODULE_ALIAS("ext3"); · 158 unmodified lines
159159 #define IS_EXT3_SB(sb) ((sb)->s_bdev->bd_holder == &ext3_fs_type)
160160
161161
162−static inline void __ext4_read_bh(struct buffer_head *bh, int op_flags,
162+static inline void __ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags,
163163 bh_end_io_t *end_io)
164164 {
165165 /*
@@ -171,10 +171,10 @@ static inline void __ext4_read_bh(struct buffer_head *bh, int op_flags, · 5 unmodified lines
171171
172172 bh->b_end_io = end_io ? end_io : end_buffer_read_sync;
173173 get_bh(bh);
174− submit_bh(REQ_OP_READ, op_flags, bh);
174+ submit_bh(REQ_OP_READ | op_flags, bh);
175175 }
176176
177−void ext4_read_bh_nowait(struct buffer_head *bh, int op_flags,
177+void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags,
178178 bh_end_io_t *end_io)
179179 {
180180 BUG_ON(!buffer_locked(bh));
@@ -186,7 +186,7 @@ void ext4_read_bh_nowait(struct buffer_head *bh, int op_flags, · 5 unmodified lines
186186 __ext4_read_bh(bh, op_flags, end_io);
187187 }
188188
189−int ext4_read_bh(struct buffer_head *bh, int op_flags, bh_end_io_t *end_io)
189+int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, bh_end_io_t *end_io)
190190 {
191191 BUG_ON(!buffer_locked(bh));
192192
@@ -203,7 +203,7 @@ int ext4_read_bh(struct buffer_head *bh, int op_flags, bh_end_io_t *end_io) · 10 unmodified lines
203203 return -EIO;
204204 }
205205
206−int ext4_read_bh_lock(struct buffer_head *bh, int op_flags, bool wait)
206+int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
207207 {
208208 if (trylock_buffer(bh)) {
209209 if (wait)
@@ -227,8 +227,8 @@ int ext4_read_bh_lock(struct buffer_head *bh, int op_flags, bool wait) · 17 unmodified lines
227227 * return.
228228 */
229229 static struct buffer_head *__ext4_sb_bread_gfp(struct super_block *sb,
230− sector_t block, int op_flags,
231− gfp_t gfp)
230+ sector_t block,
231+ blk_opf_t op_flags, gfp_t gfp)
232232 {
233233 struct buffer_head *bh;
234234 int ret;
@@ -248,7 +248,7 @@ static struct buffer_head *__ext4_sb_bread_gfp(struct super_block *sb, · 13 unmodified lines
248248 }
249249
250250 struct buffer_head *ext4_sb_bread(struct super_block *sb, sector_t block,
251− int op_flags)
251+ blk_opf_t op_flags)
252252 {
253253 return __ext4_sb_bread_gfp(sb, block, op_flags, __GFP_MOVABLE);
254254 }
@@ -5939,8 +5939,8 @@ static int ext4_commit_super(struct super_block *sb) · 5684 unmodified lines
59395939 /* Clear potential dirty bit if it was journalled update */
59405940 clear_buffer_dirty(sbh);
59415941 sbh->b_end_io = end_buffer_write_sync;
5942− submit_bh(REQ_OP_WRITE,
5943− REQ_SYNC | (test_opt(sb, BARRIER) ? REQ_FUA : 0), sbh);
5942+ submit_bh(REQ_OP_WRITE | REQ_SYNC |
5943+ (test_opt(sb, BARRIER) ? REQ_FUA : 0), sbh);
59445944 wait_on_buffer(sbh);
59455945 if (buffer_write_io_error(sbh)) {
59465946 ext4_msg(sb, KERN_ERR, "I/O error while writing "
fs/f2fs/data.c+6−5
@@ -387,11 +387,11 @@ int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) · 386 unmodified lines
387387 return 0;
388388 }
389389
390−static unsigned int f2fs_io_flags(struct f2fs_io_info *fio)
390+static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio)
391391 {
392392 unsigned int temp_mask = (1 << NR_TEMP_TYPE) - 1;
393393 unsigned int fua_flag, meta_flag, io_flag;
394− unsigned int op_flags = 0;
394+ blk_opf_t op_flags = 0;
395395
396396 if (fio->op != REQ_OP_WRITE)
397397 return 0;
@@ -999,7 +999,7 @@ out: · 601 unmodified lines
999999 }
10001000
10011001 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
1002− unsigned nr_pages, unsigned op_flag,
1002+ unsigned nr_pages, blk_opf_t op_flag,
10031003 pgoff_t first_idx, bool for_write)
10041004 {
10051005 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
@@ -1047,7 +1047,8 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, · 41 unmodified lines
10471047
10481048 /* This can handle encryption stuffs */
10491049 static int f2fs_submit_page_read(struct inode *inode, struct page *page,
1050− block_t blkaddr, int op_flags, bool for_write)
1050+ block_t blkaddr, blk_opf_t op_flags,
1051+ bool for_write)
10511052 {
10521053 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
10531054 struct bio *bio;
@@ -1181,7 +1182,7 @@ int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) · 127 unmodified lines
11811182 }
11821183
11831184 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
1184− int op_flags, bool for_write)
1185+ blk_opf_t op_flags, bool for_write)
11851186 {
11861187 struct address_space *mapping = inode->i_mapping;
11871188 struct dnode_of_data dn;
fs/f2fs/f2fs.h+3−3
@@ -1183,8 +1183,8 @@ struct f2fs_io_info { · 1182 unmodified lines
11831183 nid_t ino; /* inode number */
11841184 enum page_type type; /* contains DATA/NODE/META/META_FLUSH */
11851185 enum temp_type temp; /* contains HOT/WARM/COLD */
1186− int op; /* contains REQ_OP_ */
1187− int op_flags; /* req_flag_bits */
1186+ enum req_op op; /* contains REQ_OP_ */
1187+ blk_opf_t op_flags; /* req_flag_bits */
11881188 block_t new_blkaddr; /* new block address to be written */
11891189 block_t old_blkaddr; /* old block address before Cow */
11901190 struct page *page; /* page to be written */
@@ -3741,7 +3741,7 @@ int f2fs_reserve_new_block(struct dnode_of_data *dn); · 2550 unmodified lines
37413741 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index);
37423742 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
37433743 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
3744− int op_flags, bool for_write);
3744+ blk_opf_t op_flags, bool for_write);
37453745 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index);
37463746 struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
37473747 bool for_write);
fs/f2fs/node.c+1−1
@@ -1327,7 +1327,7 @@ fail: · 1326 unmodified lines
13271327 * 0: f2fs_put_page(page, 0)
13281328 * LOCKED_PAGE or error: f2fs_put_page(page, 1)
13291329 */
1330−static int read_node_page(struct page *page, int op_flags)
1330+static int read_node_page(struct page *page, blk_opf_t op_flags)
13311331 {
13321332 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
13331333 struct node_info ni;
fs/f2fs/segment.c+1−1
@@ -1082,7 +1082,7 @@ static int __submit_discard_cmd(struct f2fs_sb_info *sbi, · 1081 unmodified lines
10821082 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
10831083 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
10841084 &(dcc->fstrim_list) : &(dcc->wait_list);
1085− int flag = dpolicy->sync ? REQ_SYNC : 0;
1085+ blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0;
10861086 block_t lstart, start, len, total_len;
10871087 int err = 0;
10881088
fs/gfs2/bmap.c+2−3
@@ -310,9 +310,8 @@ static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end) · 309 unmodified lines
310310 if (trylock_buffer(rabh)) {
311311 if (!buffer_uptodate(rabh)) {
312312 rabh->b_end_io = end_buffer_read_sync;
313− submit_bh(REQ_OP_READ,
314− REQ_RAHEAD | REQ_META | REQ_PRIO,
315− rabh);
313+ submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META |
314+ REQ_PRIO, rabh);
316315 continue;
317316 }
318317 unlock_buffer(rabh);
fs/gfs2/dir.c+2−3
@@ -1508,9 +1508,8 @@ static void gfs2_dir_readahead(struct inode *inode, unsigned hsize, u32 index, · 1507 unmodified lines
15081508 continue;
15091509 }
15101510 bh->b_end_io = end_buffer_read_sync;
1511− submit_bh(REQ_OP_READ,
1512− REQ_RAHEAD | REQ_META | REQ_PRIO,
1513− bh);
1511+ submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META |
1512+ REQ_PRIO, bh);
15141513 continue;
15151514 }
15161515 brelse(bh);
fs/gfs2/log.c+2−2
@@ -823,7 +823,7 @@ void gfs2_flush_revokes(struct gfs2_sbd *sdp) · 822 unmodified lines
823823
824824 void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
825825 u64 seq, u32 tail, u32 lblock, u32 flags,
826− int op_flags)
826+ blk_opf_t op_flags)
827827 {
828828 struct gfs2_log_header *lh;
829829 u32 hash, crc;
@@ -905,7 +905,7 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, · 75 unmodified lines
905905
906906 static void log_write_header(struct gfs2_sbd *sdp, u32 flags)
907907 {
908− int op_flags = REQ_PREFLUSH | REQ_FUA | REQ_META | REQ_SYNC;
908+ blk_opf_t op_flags = REQ_PREFLUSH | REQ_FUA | REQ_META | REQ_SYNC;
909909 enum gfs2_freeze_state state = atomic_read(&sdp->sd_freeze_state);
910910
911911 gfs2_assert_withdraw(sdp, (state != SFS_FROZEN));
fs/gfs2/log.h+1−1
@@ -82,7 +82,7 @@ extern void gfs2_log_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, · 81 unmodified lines
8282 unsigned int *extra_revokes);
8383 extern void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
8484 u64 seq, u32 tail, u32 lblock, u32 flags,
85− int op_flags);
85+ blk_opf_t op_flags);
8686 extern void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl,
8787 u32 type);
8888 extern void gfs2_log_commit(struct gfs2_sbd *sdp, struct gfs2_trans *trans);
fs/gfs2/lops.c+2−2
@@ -238,7 +238,7 @@ static void gfs2_end_log_write(struct bio *bio) · 237 unmodified lines
238238 * there is no pending bio, then this is a no-op.
239239 */
240240
241−void gfs2_log_submit_bio(struct bio **biop, int opf)
241+void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf)
242242 {
243243 struct bio *bio = *biop;
244244 if (bio) {
@@ -292,7 +292,7 @@ static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno, · 47 unmodified lines
292292 */
293293
294294 static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno,
295− struct bio **biop, int op,
295+ struct bio **biop, enum req_op op,
296296 bio_end_io_t *end_io, bool flush)
297297 {
298298 struct bio *bio = *biop;
fs/gfs2/lops.h+1−1
@@ -16,7 +16,7 @@ extern u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn); · 15 unmodified lines
1616 extern void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
1717 struct page *page, unsigned size, unsigned offset,
1818 u64 blkno);
19−extern void gfs2_log_submit_bio(struct bio **biop, int opf);
19+extern void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf);
2020 extern void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh);
2121 extern int gfs2_find_jhead(struct gfs2_jdesc *jd,
2222 struct gfs2_log_header_host *head, bool keep_cache);
fs/gfs2/meta_io.c+8−10
@@ -34,7 +34,7 @@ static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wb · 33 unmodified lines
3434 {
3535 struct buffer_head *bh, *head;
3636 int nr_underway = 0;
37− int write_flags = REQ_META | REQ_PRIO | wbc_to_write_flags(wbc);
37+ blk_opf_t write_flags = REQ_META | REQ_PRIO | wbc_to_write_flags(wbc);
3838
3939 BUG_ON(!PageLocked(page));
4040 BUG_ON(!page_has_buffers(page));
@@ -75,7 +75,7 @@ static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wb · 34 unmodified lines
7575 do {
7676 struct buffer_head *next = bh->b_this_page;
7777 if (buffer_async_write(bh)) {
78− submit_bh(REQ_OP_WRITE, write_flags, bh);
78+ submit_bh(REQ_OP_WRITE | write_flags, bh);
7979 nr_underway++;
8080 }
8181 bh = next;
@@ -217,14 +217,13 @@ static void gfs2_meta_read_endio(struct bio *bio) · 135 unmodified lines
217217 * Submit several consecutive buffer head I/O requests as a single bio I/O
218218 * request. (See submit_bh_wbc.)
219219 */
220−static void gfs2_submit_bhs(int op, int op_flags, struct buffer_head *bhs[],
221− int num)
220+static void gfs2_submit_bhs(blk_opf_t opf, struct buffer_head *bhs[], int num)
222221 {
223222 while (num > 0) {
224223 struct buffer_head *bh = *bhs;
225224 struct bio *bio;
226225
227− bio = bio_alloc(bh->b_bdev, num, op | op_flags, GFP_NOIO);
226+ bio = bio_alloc(bh->b_bdev, num, opf, GFP_NOIO);
228227 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
229228 while (num > 0) {
230229 bh = *bhs;
@@ -288,7 +287,7 @@ int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags, · 57 unmodified lines
288287 }
289288 }
290289
291− gfs2_submit_bhs(REQ_OP_READ, REQ_META | REQ_PRIO, bhs, num);
290+ gfs2_submit_bhs(REQ_OP_READ | REQ_META | REQ_PRIO, bhs, num);
292291 if (!(flags & DIO_WAIT))
293292 return 0;
294293
@@ -527,7 +526,7 @@ struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen) · 232 unmodified lines
527526 if (buffer_uptodate(first_bh))
528527 goto out;
529528 if (!buffer_locked(first_bh))
530− ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &first_bh);
529+ ll_rw_block(REQ_OP_READ | REQ_META | REQ_PRIO, 1, &first_bh);
531530
532531 dblock++;
533532 extlen--;
@@ -536,9 +535,8 @@ struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen) · 2 unmodified lines
536535 bh = gfs2_getbuf(gl, dblock, CREATE);
537536
538537 if (!buffer_uptodate(bh) && !buffer_locked(bh))
539− ll_rw_block(REQ_OP_READ,
540− REQ_RAHEAD | REQ_META | REQ_PRIO,
541− 1, &bh);
538+ ll_rw_block(REQ_OP_READ | REQ_RAHEAD | REQ_META |
539+ REQ_PRIO, 1, &bh);
542540 brelse(bh);
543541 dblock++;
544542 extlen--;
fs/gfs2/quota.c+1−1
@@ -746,7 +746,7 @@ static int gfs2_write_buf_to_page(struct gfs2_inode *ip, unsigned long index, · 745 unmodified lines
746746 if (PageUptodate(page))
747747 set_buffer_uptodate(bh);
748748 if (!buffer_uptodate(bh)) {
749− ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &bh);
749+ ll_rw_block(REQ_OP_READ | REQ_META | REQ_PRIO, 1, &bh);
750750 wait_on_buffer(bh);
751751 if (!buffer_uptodate(bh))
752752 goto unlock_out;
fs/hfsplus/hfsplus_fs.h+1−1
@@ -525,7 +525,7 @@ int hfsplus_compare_dentry(const struct dentry *dentry, unsigned int len, · 524 unmodified lines
525525
526526 /* wrapper.c */
527527 int hfsplus_submit_bio(struct super_block *sb, sector_t sector, void *buf,
528− void **data, int op, int op_flags);
528+ void **data, blk_opf_t opf);
529529 int hfsplus_read_wrapper(struct super_block *sb);
530530
531531 /*
fs/hfsplus/part_tbl.c+2−3
@@ -112,8 +112,7 @@ static int hfs_parse_new_pmap(struct super_block *sb, void *buf, · 111 unmodified lines
112112 if ((u8 *)pm - (u8 *)buf >= buf_size) {
113113 res = hfsplus_submit_bio(sb,
114114 *part_start + HFS_PMAP_BLK + i,
115− buf, (void **)&pm, REQ_OP_READ,
116− 0);
115+ buf, (void **)&pm, REQ_OP_READ);
117116 if (res)
118117 return res;
119118 }
@@ -137,7 +136,7 @@ int hfs_part_find(struct super_block *sb, · 17 unmodified lines
137136 return -ENOMEM;
138137
139138 res = hfsplus_submit_bio(sb, *part_start + HFS_PMAP_BLK,
140− buf, &data, REQ_OP_READ, 0);
139+ buf, &data, REQ_OP_READ);
141140 if (res)
142141 goto out;
143142
fs/hfsplus/super.c+2−2
@@ -221,7 +221,7 @@ static int hfsplus_sync_fs(struct super_block *sb, int wait) · 220 unmodified lines
221221
222222 error2 = hfsplus_submit_bio(sb,
223223 sbi->part_start + HFSPLUS_VOLHEAD_SECTOR,
224− sbi->s_vhdr_buf, NULL, REQ_OP_WRITE,
224+ sbi->s_vhdr_buf, NULL, REQ_OP_WRITE |
225225 REQ_SYNC);
226226 if (!error)
227227 error = error2;
@@ -230,7 +230,7 @@ static int hfsplus_sync_fs(struct super_block *sb, int wait) · 2 unmodified lines
230230
231231 error2 = hfsplus_submit_bio(sb,
232232 sbi->part_start + sbi->sect_count - 2,
233− sbi->s_backup_vhdr_buf, NULL, REQ_OP_WRITE,
233+ sbi->s_backup_vhdr_buf, NULL, REQ_OP_WRITE |
234234 REQ_SYNC);
235235 if (!error)
236236 error2 = error;
fs/hfsplus/wrapper.c+6−6
@@ -45,8 +45,9 @@ struct hfsplus_wd { · 44 unmodified lines
4545 * will work correctly.
4646 */
4747 int hfsplus_submit_bio(struct super_block *sb, sector_t sector,
48− void *buf, void **data, int op, int op_flags)
48+ void *buf, void **data, blk_opf_t opf)
4949 {
50+ const enum req_op op = opf & REQ_OP_MASK;
5051 struct bio *bio;
5152 int ret = 0;
5253 u64 io_size;
@@ -63,10 +64,10 @@ int hfsplus_submit_bio(struct super_block *sb, sector_t sector, · 10 unmodified lines
6364 offset = start & (io_size - 1);
6465 sector &= ~((io_size >> HFSPLUS_SECTOR_SHIFT) - 1);
6566
66− bio = bio_alloc(sb->s_bdev, 1, op | op_flags, GFP_NOIO);
67+ bio = bio_alloc(sb->s_bdev, 1, opf, GFP_NOIO);
6768 bio->bi_iter.bi_sector = sector;
6869
69− if (op != WRITE && data)
70+ if (op != REQ_OP_WRITE && data)
7071 *data = (u8 *)buf + offset;
7172
7273 while (io_size > 0) {
@@ -184,7 +185,7 @@ int hfsplus_read_wrapper(struct super_block *sb) · 111 unmodified lines
184185 reread:
185186 error = hfsplus_submit_bio(sb, part_start + HFSPLUS_VOLHEAD_SECTOR,
186187 sbi->s_vhdr_buf, (void **)&sbi->s_vhdr,
187− REQ_OP_READ, 0);
188+ REQ_OP_READ);
188189 if (error)
189190 goto out_free_backup_vhdr;
190191
@@ -216,8 +217,7 @@ reread: · 25 unmodified lines
216217
217218 error = hfsplus_submit_bio(sb, part_start + part_size - 2,
218219 sbi->s_backup_vhdr_buf,
219− (void **)&sbi->s_backup_vhdr, REQ_OP_READ,
220− 0);
220+ (void **)&sbi->s_backup_vhdr, REQ_OP_READ);
221221 if (error)
222222 goto out_free_backup_vhdr;
223223
fs/iomap/direct-io.c+6−6
@@ -52,7 +52,7 @@ struct iomap_dio { · 51 unmodified lines
5252 };
5353
5454 static struct bio *iomap_dio_alloc_bio(const struct iomap_iter *iter,
55− struct iomap_dio *dio, unsigned short nr_vecs, unsigned int opf)
55+ struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf)
5656 {
5757 if (dio->dops && dio->dops->bio_set)
5858 return bio_alloc_bioset(iter->iomap.bdev, nr_vecs, opf,
@@ -212,10 +212,10 @@ static void iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio, · 153 unmodified lines
212212 * mapping, and whether or not we want FUA. Note that we can end up
213213 * clearing the WRITE_FUA flag in the dio request.
214214 */
215−static inline unsigned int iomap_dio_bio_opflags(struct iomap_dio *dio,
215+static inline blk_opf_t iomap_dio_bio_opflags(struct iomap_dio *dio,
216216 const struct iomap *iomap, bool use_fua)
217217 {
218− unsigned int opflags = REQ_SYNC | REQ_IDLE;
218+ blk_opf_t opflags = REQ_SYNC | REQ_IDLE;
219219
220220 if (!(dio->flags & IOMAP_DIO_WRITE)) {
221221 WARN_ON_ONCE(iomap->flags & IOMAP_F_ZONE_APPEND);
@@ -242,10 +242,9 @@ static loff_t iomap_dio_bio_iter(const struct iomap_iter *iter, · 20 unmodified lines
242242 struct inode *inode = iter->inode;
243243 unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
244244 unsigned int fs_block_size = i_blocksize(inode), pad;
245− unsigned int align = iov_iter_alignment(dio->submit.iter);
246245 loff_t length = iomap_length(iter);
247246 loff_t pos = iter->pos;
248− unsigned int bio_opf;
247+ blk_opf_t bio_opf;
249248 struct bio *bio;
250249 bool need_zeroout = false;
251250 bool use_fua = false;
@@ -253,7 +252,8 @@ static loff_t iomap_dio_bio_iter(const struct iomap_iter *iter, · 1 unmodified line
253252 size_t copied = 0;
254253 size_t orig_count;
255254
256− if ((pos | length | align) & ((1 << blkbits) - 1))
255+ if ((pos | length) & ((1 << blkbits) - 1) ||
256+ !bdev_iter_is_aligned(iomap->bdev, dio->submit.iter))
257257 return -EINVAL;
258258
259259 if (iomap->type == IOMAP_UNWRITTEN) {
fs/isofs/compress.c+1−1
@@ -82,7 +82,7 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, · 81 unmodified lines
8282 return 0;
8383 }
8484 haveblocks = isofs_get_blocks(inode, blocknum, bhs, needblocks);
85− ll_rw_block(REQ_OP_READ, 0, haveblocks, bhs);
85+ ll_rw_block(REQ_OP_READ, haveblocks, bhs);
8686
8787 curbh = 0;
8888 curpage = 0;
fs/jbd2/commit.c+4−4
@@ -155,10 +155,10 @@ static int journal_submit_commit_record(journal_t *journal, · 154 unmodified lines
155155
156156 if (journal->j_flags & JBD2_BARRIER &&
157157 !jbd2_has_feature_async_commit(journal))
158− ret = submit_bh(REQ_OP_WRITE,
159− REQ_SYNC | REQ_PREFLUSH | REQ_FUA, bh);
158+ ret = submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH |
159+ REQ_FUA, bh);
160160 else
161− ret = submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
161+ ret = submit_bh(REQ_OP_WRITE | REQ_SYNC, bh);
162162
163163 *cbh = bh;
164164 return ret;
@@ -763,7 +763,7 @@ start_journal_io: · 598 unmodified lines
763763 clear_buffer_dirty(bh);
764764 set_buffer_uptodate(bh);
765765 bh->b_end_io = journal_end_buffer_io_sync;
766− submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
766+ submit_bh(REQ_OP_WRITE | REQ_SYNC, bh);
767767 }
768768 cond_resched();
769769
fs/jbd2/journal.c+14−11
@@ -1465,7 +1465,8 @@ journal_t *jbd2_journal_init_dev(struct block_device *bdev, · 1464 unmodified lines
14651465 if (!journal)
14661466 return NULL;
14671467
1468− bdevname(journal->j_dev, journal->j_devname);
1468+ snprintf(journal->j_devname, sizeof(journal->j_devname),
1469+ "%pg", journal->j_dev);
14691470 strreplace(journal->j_devname, '/', '!');
14701471 jbd2_stats_proc_init(journal);
14711472
@@ -1507,7 +1508,8 @@ journal_t *jbd2_journal_init_inode(struct inode *inode) · 35 unmodified lines
15071508 return NULL;
15081509
15091510 journal->j_inode = inode;
1510− bdevname(journal->j_dev, journal->j_devname);
1511+ snprintf(journal->j_devname, sizeof(journal->j_devname),
1512+ "%pg", journal->j_dev);
15111513 p = strreplace(journal->j_devname, '/', '!');
15121514 sprintf(p, "-%lu", journal->j_inode->i_ino);
15131515 jbd2_stats_proc_init(journal);
@@ -1602,7 +1604,7 @@ static int journal_reset(journal_t *journal) · 88 unmodified lines
16021604 * This function expects that the caller will have locked the journal
16031605 * buffer head, and will return with it unlocked
16041606 */
1605−static int jbd2_write_superblock(journal_t *journal, int write_flags)
1607+static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags)
16061608 {
16071609 struct buffer_head *bh = journal->j_sb_buffer;
16081610 journal_superblock_t *sb = journal->j_superblock;
@@ -1636,7 +1638,7 @@ static int jbd2_write_superblock(journal_t *journal, int write_flags) · 27 unmodified lines
16361638 sb->s_checksum = jbd2_superblock_csum(journal, sb);
16371639 get_bh(bh);
16381640 bh->b_end_io = end_buffer_write_sync;
1639− ret = submit_bh(REQ_OP_WRITE, write_flags, bh);
1641+ ret = submit_bh(REQ_OP_WRITE | write_flags, bh);
16401642 wait_on_buffer(bh);
16411643 if (buffer_write_io_error(bh)) {
16421644 clear_buffer_write_io_error(bh);
@@ -1659,13 +1661,14 @@ static int jbd2_write_superblock(journal_t *journal, int write_flags) · 16 unmodified lines
16591661 * @journal: The journal to update.
16601662 * @tail_tid: TID of the new transaction at the tail of the log
16611663 * @tail_block: The first block of the transaction at the tail of the log
1662− * @write_op: With which operation should we write the journal sb
1664+ * @write_flags: Flags for the journal sb write operation
16631665 *
16641666 * Update a journal's superblock information about log tail and write it to
16651667 * disk, waiting for the IO to complete.
16661668 */
16671669 int jbd2_journal_update_sb_log_tail(journal_t *journal, tid_t tail_tid,
1668− unsigned long tail_block, int write_op)
1670+ unsigned long tail_block,
1671+ blk_opf_t write_flags)
16691672 {
16701673 journal_superblock_t *sb = journal->j_superblock;
16711674 int ret;
@@ -1685,7 +1688,7 @@ int jbd2_journal_update_sb_log_tail(journal_t *journal, tid_t tail_tid, · 13 unmodified lines
16851688 sb->s_sequence = cpu_to_be32(tail_tid);
16861689 sb->s_start = cpu_to_be32(tail_block);
16871690
1688− ret = jbd2_write_superblock(journal, write_op);
1691+ ret = jbd2_write_superblock(journal, write_flags);
16891692 if (ret)
16901693 goto out;
16911694
@@ -1702,12 +1705,12 @@ out: · 10 unmodified lines
17021705 /**
17031706 * jbd2_mark_journal_empty() - Mark on disk journal as empty.
17041707 * @journal: The journal to update.
1705− * @write_op: With which operation should we write the journal sb
1708+ * @write_flags: Flags for the journal sb write operation
17061709 *
17071710 * Update a journal's dynamic superblock fields to show that journal is empty.
17081711 * Write updated superblock to disk waiting for IO to complete.
17091712 */
1710−static void jbd2_mark_journal_empty(journal_t *journal, int write_op)
1713+static void jbd2_mark_journal_empty(journal_t *journal, blk_opf_t write_flags)
17111714 {
17121715 journal_superblock_t *sb = journal->j_superblock;
17131716 bool had_fast_commit = false;
@@ -1733,7 +1736,7 @@ static void jbd2_mark_journal_empty(journal_t *journal, int write_op) · 19 unmodified lines
17331736 had_fast_commit = true;
17341737 }
17351738
1736− jbd2_write_superblock(journal, write_op);
1739+ jbd2_write_superblock(journal, write_flags);
17371740
17381741 if (had_fast_commit)
17391742 jbd2_set_feature_fast_commit(journal);
@@ -1898,7 +1901,7 @@ static int journal_get_superblock(journal_t *journal) · 158 unmodified lines
18981901
18991902 J_ASSERT(bh != NULL);
19001903 if (!buffer_uptodate(bh)) {
1901− ll_rw_block(REQ_OP_READ, 0, 1, &bh);
1904+ ll_rw_block(REQ_OP_READ, 1, &bh);
19021905 wait_on_buffer(bh);
19031906 if (!buffer_uptodate(bh)) {
19041907 printk(KERN_ERR
fs/jbd2/recovery.c+2−2
@@ -100,7 +100,7 @@ static int do_readahead(journal_t *journal, unsigned int start) · 99 unmodified lines
100100 if (!buffer_uptodate(bh) && !buffer_locked(bh)) {
101101 bufs[nbufs++] = bh;
102102 if (nbufs == MAXBUF) {
103− ll_rw_block(REQ_OP_READ, 0, nbufs, bufs);
103+ ll_rw_block(REQ_OP_READ, nbufs, bufs);
104104 journal_brelse_array(bufs, nbufs);
105105 nbufs = 0;
106106 }
@@ -109,7 +109,7 @@ static int do_readahead(journal_t *journal, unsigned int start) · 2 unmodified lines
109109 }
110110
111111 if (nbufs)
112− ll_rw_block(REQ_OP_READ, 0, nbufs, bufs);
112+ ll_rw_block(REQ_OP_READ, nbufs, bufs);
113113 err = 0;
114114
115115 failed:
fs/mpage.c+3−3
@@ -145,13 +145,13 @@ static struct bio *do_mpage_readpage(struct mpage_readpage_args *args) · 144 unmodified lines
145145 struct block_device *bdev = NULL;
146146 int length;
147147 int fully_mapped = 1;
148− int op = REQ_OP_READ;
148+ blk_opf_t opf = REQ_OP_READ;
149149 unsigned nblocks;
150150 unsigned relative_block;
151151 gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
152152
153153 if (args->is_readahead) {
154− op |= REQ_RAHEAD;
154+ opf |= REQ_RAHEAD;
155155 gfp |= __GFP_NORETRY | __GFP_NOWARN;
156156 }
157157
@@ -269,7 +269,7 @@ alloc_new: · 111 unmodified lines
269269 page))
270270 goto out;
271271 }
272− args->bio = bio_alloc(bdev, bio_max_segs(args->nr_pages), op,
272+ args->bio = bio_alloc(bdev, bio_max_segs(args->nr_pages), opf,
273273 gfp);
274274 if (args->bio == NULL)
275275 goto confused;
fs/nfs/blocklayout/blocklayout.c+6−7
@@ -121,7 +121,7 @@ static bool offset_in_map(u64 offset, struct pnfs_block_dev_map *map) · 120 unmodified lines
121121 }
122122
123123 static struct bio *
124−do_add_page_to_bio(struct bio *bio, int npg, int rw, sector_t isect,
124+do_add_page_to_bio(struct bio *bio, int npg, enum req_op op, sector_t isect,
125125 struct page *page, struct pnfs_block_dev_map *map,
126126 struct pnfs_block_extent *be, bio_end_io_t end_io,
127127 struct parallel_io *par, unsigned int offset, int *len)
@@ -131,7 +131,7 @@ do_add_page_to_bio(struct bio *bio, int npg, int rw, sector_t isect, · 3 unmodified lines
131131 u64 disk_addr, end;
132132
133133 dprintk("%s: npg %d rw %d isect %llu offset %u len %d\n", __func__,
134− npg, rw, (unsigned long long)isect, offset, *len);
134+ npg, (__force u32)op, (unsigned long long)isect, offset, *len);
135135
136136 /* translate to device offset */
137137 isect += be->be_v_offset;
@@ -154,7 +154,7 @@ do_add_page_to_bio(struct bio *bio, int npg, int rw, sector_t isect, · 16 unmodified lines
154154
155155 retry:
156156 if (!bio) {
157− bio = bio_alloc(map->bdev, bio_max_segs(npg), rw, GFP_NOIO);
157+ bio = bio_alloc(map->bdev, bio_max_segs(npg), op, GFP_NOIO);
158158 bio->bi_iter.bi_sector = disk_addr >> SECTOR_SHIFT;
159159 bio->bi_end_io = end_io;
160160 bio->bi_private = par;
@@ -291,7 +291,7 @@ bl_read_pagelist(struct nfs_pgio_header *header) · 130 unmodified lines
291291 } else {
292292 bio = do_add_page_to_bio(bio,
293293 header->page_array.npages - i,
294− READ,
294+ REQ_OP_READ,
295295 isect, pages[i], &map, &be,
296296 bl_end_io_read, par,
297297 pg_offset, &pg_len);
@@ -420,9 +420,8 @@ bl_write_pagelist(struct nfs_pgio_header *header, int sync) · 122 unmodified lines
420420
421421 pg_len = PAGE_SIZE;
422422 bio = do_add_page_to_bio(bio, header->page_array.npages - i,
423− WRITE, isect, pages[i], &map, &be,
424− bl_end_io_write, par,
425− 0, &pg_len);
423+ REQ_OP_WRITE, isect, pages[i], &map,
424+ &be, bl_end_io_write, par, 0, &pg_len);
426425 if (IS_ERR(bio)) {
427426 header->pnfs_error = PTR_ERR(bio);
428427 bio = NULL;
fs/nilfs2/btnode.c+4−4
@@ -70,7 +70,7 @@ nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr) · 69 unmodified lines
7070 }
7171
7272 int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr,
73− sector_t pblocknr, int mode, int mode_flags,
73+ sector_t pblocknr, blk_opf_t opf,
7474 struct buffer_head **pbh, sector_t *submit_ptr)
7575 {
7676 struct buffer_head *bh;
@@ -103,13 +103,13 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr, · 26 unmodified lines
103103 }
104104 }
105105
106− if (mode_flags & REQ_RAHEAD) {
106+ if (opf & REQ_RAHEAD) {
107107 if (pblocknr != *submit_ptr + 1 || !trylock_buffer(bh)) {
108108 err = -EBUSY; /* internal code */
109109 brelse(bh);
110110 goto out_locked;
111111 }
112− } else { /* mode == READ */
112+ } else { /* opf == REQ_OP_READ */
113113 lock_buffer(bh);
114114 }
115115 if (buffer_uptodate(bh)) {
@@ -122,7 +122,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr, · 6 unmodified lines
122122 bh->b_blocknr = pblocknr; /* set block address for read */
123123 bh->b_end_io = end_buffer_read_sync;
124124 get_bh(bh);
125− submit_bh(mode, mode_flags, bh);
125+ submit_bh(opf, bh);
126126 bh->b_blocknr = blocknr; /* set back to the given block address */
127127 *submit_ptr = pblocknr;
128128 err = 0;
fs/nilfs2/btnode.h+2−2
@@ -34,8 +34,8 @@ void nilfs_init_btnc_inode(struct inode *btnc_inode); · 33 unmodified lines
3434 void nilfs_btnode_cache_clear(struct address_space *);
3535 struct buffer_head *nilfs_btnode_create_block(struct address_space *btnc,
3636 __u64 blocknr);
37−int nilfs_btnode_submit_block(struct address_space *, __u64, sector_t, int,
38− int, struct buffer_head **, sector_t *);
37+int nilfs_btnode_submit_block(struct address_space *, __u64, sector_t,
38+ blk_opf_t, struct buffer_head **, sector_t *);
3939 void nilfs_btnode_delete(struct buffer_head *);
4040 int nilfs_btnode_prepare_change_key(struct address_space *,
4141 struct nilfs_btnode_chkey_ctxt *);
fs/nilfs2/btree.c+3−3
@@ -477,7 +477,7 @@ static int __nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr, · 476 unmodified lines
477477 sector_t submit_ptr = 0;
478478 int ret;
479479
480− ret = nilfs_btnode_submit_block(btnc, ptr, 0, REQ_OP_READ, 0, &bh,
480+ ret = nilfs_btnode_submit_block(btnc, ptr, 0, REQ_OP_READ, &bh,
481481 &submit_ptr);
482482 if (ret) {
483483 if (ret != -EEXIST)
@@ -495,8 +495,8 @@ static int __nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr, · 11 unmodified lines
495495 ptr2 = nilfs_btree_node_get_ptr(ra->node, i, ra->ncmax);
496496
497497 ret = nilfs_btnode_submit_block(btnc, ptr2, 0,
498− REQ_OP_READ, REQ_RAHEAD,
499− &ra_bh, &submit_ptr);
498+ REQ_OP_READ | REQ_RAHEAD,
499+ &ra_bh, &submit_ptr);
500500 if (likely(!ret || ret == -EEXIST))
501501 brelse(ra_bh);
502502 else if (ret != -EBUSY)
fs/nilfs2/gcinode.c+3−4
@@ -92,7 +92,7 @@ int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff, · 91 unmodified lines
9292 bh->b_blocknr = pbn;
9393 bh->b_end_io = end_buffer_read_sync;
9494 get_bh(bh);
95− submit_bh(REQ_OP_READ, 0, bh);
95+ submit_bh(REQ_OP_READ, bh);
9696 if (vbn)
9797 bh->b_blocknr = vbn;
9898 out:
@@ -129,9 +129,8 @@ int nilfs_gccache_submit_read_node(struct inode *inode, sector_t pbn, · 30 unmodified lines
129129 struct inode *btnc_inode = NILFS_I(inode)->i_assoc_inode;
130130 int ret;
131131
132− ret = nilfs_btnode_submit_block(btnc_inode->i_mapping,
133− vbn ? : pbn, pbn, REQ_OP_READ, 0,
134− out_bh, &pbn);
132+ ret = nilfs_btnode_submit_block(btnc_inode->i_mapping, vbn ? : pbn, pbn,
133+ REQ_OP_READ, out_bh, &pbn);
135134 if (ret == -EEXIST) /* internal code (cache hit) */
136135 ret = 0;
137136 return ret;
fs/nilfs2/mdt.c+10−9
@@ -111,8 +111,8 @@ static int nilfs_mdt_create_block(struct inode *inode, unsigned long block, · 110 unmodified lines
111111 }
112112
113113 static int
114−nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff,
115− int mode, int mode_flags, struct buffer_head **out_bh)
114+nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff, blk_opf_t opf,
115+ struct buffer_head **out_bh)
116116 {
117117 struct buffer_head *bh;
118118 __u64 blknum = 0;
@@ -126,12 +126,12 @@ nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff, · 7 unmodified lines
126126 if (buffer_uptodate(bh))
127127 goto out;
128128
129− if (mode_flags & REQ_RAHEAD) {
129+ if (opf & REQ_RAHEAD) {
130130 if (!trylock_buffer(bh)) {
131131 ret = -EBUSY;
132132 goto failed_bh;
133133 }
134− } else /* mode == READ */
134+ } else /* opf == REQ_OP_READ */
135135 lock_buffer(bh);
136136
137137 if (buffer_uptodate(bh)) {
@@ -148,10 +148,11 @@ nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff, · 10 unmodified lines
148148
149149 bh->b_end_io = end_buffer_read_sync;
150150 get_bh(bh);
151− submit_bh(mode, mode_flags, bh);
151+ submit_bh(opf, bh);
152152 ret = 0;
153153
154− trace_nilfs2_mdt_submit_block(inode, inode->i_ino, blkoff, mode);
154+ trace_nilfs2_mdt_submit_block(inode, inode->i_ino, blkoff,
155+ opf & REQ_OP_MASK);
155156 out:
156157 get_bh(bh);
157158 *out_bh = bh;
@@ -172,7 +173,7 @@ static int nilfs_mdt_read_block(struct inode *inode, unsigned long block, · 14 unmodified lines
172173 int i, nr_ra_blocks = NILFS_MDT_MAX_RA_BLOCKS;
173174 int err;
174175
175− err = nilfs_mdt_submit_block(inode, block, REQ_OP_READ, 0, &first_bh);
176+ err = nilfs_mdt_submit_block(inode, block, REQ_OP_READ, &first_bh);
176177 if (err == -EEXIST) /* internal code */
177178 goto out;
178179
@@ -182,8 +183,8 @@ static int nilfs_mdt_read_block(struct inode *inode, unsigned long block, · 3 unmodified lines
182183 if (readahead) {
183184 blkoff = block + 1;
184185 for (i = 0; i < nr_ra_blocks; i++, blkoff++) {
185− err = nilfs_mdt_submit_block(inode, blkoff, REQ_OP_READ,
186− REQ_RAHEAD, &bh);
186+ err = nilfs_mdt_submit_block(inode, blkoff,
187+ REQ_OP_READ | REQ_RAHEAD, &bh);
187188 if (likely(!err || err == -EEXIST))
188189 brelse(bh);
189190 else if (err != -EBUSY)
fs/ntfs/aops.c+3−3
@@ -342,7 +342,7 @@ handle_zblock: · 341 unmodified lines
342342 for (i = 0; i < nr; i++) {
343343 tbh = arr[i];
344344 if (likely(!buffer_uptodate(tbh)))
345− submit_bh(REQ_OP_READ, 0, tbh);
345+ submit_bh(REQ_OP_READ, tbh);
346346 else
347347 ntfs_end_buffer_async_read(tbh, 1);
348348 }
@@ -859,7 +859,7 @@ lock_retry_remap: · 510 unmodified lines
859859 do {
860860 struct buffer_head *next = bh->b_this_page;
861861 if (buffer_async_write(bh)) {
862− submit_bh(REQ_OP_WRITE, 0, bh);
862+ submit_bh(REQ_OP_WRITE, bh);
863863 need_end_writeback = false;
864864 }
865865 bh = next;
@@ -1187,7 +1187,7 @@ lock_retry_remap: · 321 unmodified lines
11871187 BUG_ON(!buffer_mapped(tbh));
11881188 get_bh(tbh);
11891189 tbh->b_end_io = end_buffer_write_sync;
1190− submit_bh(REQ_OP_WRITE, 0, tbh);
1190+ submit_bh(REQ_OP_WRITE, tbh);
11911191 }
11921192 /* Synchronize the mft mirror now if not @sync. */
11931193 if (is_mft && !sync)
fs/ntfs/compress.c+1−1
@@ -658,7 +658,7 @@ lock_retry_remap: · 657 unmodified lines
658658 }
659659 get_bh(tbh);
660660 tbh->b_end_io = end_buffer_read_sync;
661− submit_bh(REQ_OP_READ, 0, tbh);
661+ submit_bh(REQ_OP_READ, tbh);
662662 }
663663
664664 /* Wait for io completion on all buffer heads. */
fs/ntfs/file.c+1−1
@@ -537,7 +537,7 @@ static inline int ntfs_submit_bh_for_read(struct buffer_head *bh) · 536 unmodified lines
537537 lock_buffer(bh);
538538 get_bh(bh);
539539 bh->b_end_io = end_buffer_read_sync;
540− return submit_bh(REQ_OP_READ, 0, bh);
540+ return submit_bh(REQ_OP_READ, bh);
541541 }
542542
543543 /**
fs/ntfs/logfile.c+1−1
@@ -807,7 +807,7 @@ map_vcn: · 806 unmodified lines
807807 * completed ignore errors afterwards as we can assume
808808 * that if one buffer worked all of them will work.
809809 */
810− submit_bh(REQ_OP_WRITE, 0, bh);
810+ submit_bh(REQ_OP_WRITE, bh);
811811 if (should_wait) {
812812 should_wait = false;
813813 wait_on_buffer(bh);
fs/ntfs/mft.c+2−2
@@ -583,7 +583,7 @@ int ntfs_sync_mft_mirror(ntfs_volume *vol, const unsigned long mft_no, · 582 unmodified lines
583583 clear_buffer_dirty(tbh);
584584 get_bh(tbh);
585585 tbh->b_end_io = end_buffer_write_sync;
586− submit_bh(REQ_OP_WRITE, 0, tbh);
586+ submit_bh(REQ_OP_WRITE, tbh);
587587 }
588588 /* Wait on i/o completion of buffers. */
589589 for (i_bhs = 0; i_bhs < nr_bhs; i_bhs++) {
@@ -780,7 +780,7 @@ int write_mft_record_nolock(ntfs_inode *ni, MFT_RECORD *m, int sync) · 190 unmodified lines
780780 clear_buffer_dirty(tbh);
781781 get_bh(tbh);
782782 tbh->b_end_io = end_buffer_write_sync;
783− submit_bh(REQ_OP_WRITE, 0, tbh);
783+ submit_bh(REQ_OP_WRITE, tbh);
784784 }
785785 /* Synchronize the mft mirror now if not @sync. */
786786 if (!sync && ni->mft_no < vol->mftmirr_size)
fs/ntfs3/file.c+1−1
@@ -242,7 +242,7 @@ static int ntfs_zero_range(struct inode *inode, u64 vbo, u64 vbo_to) · 241 unmodified lines
242242 lock_buffer(bh);
243243 bh->b_end_io = end_buffer_read_sync;
244244 get_bh(bh);
245− submit_bh(REQ_OP_READ, 0, bh);
245+ submit_bh(REQ_OP_READ, bh);
246246
247247 wait_on_buffer(bh);
248248 if (!buffer_uptodate(bh)) {
fs/ntfs3/fsntfs.c+1−1
@@ -1448,7 +1448,7 @@ int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, · 1447 unmodified lines
14481448 */
14491449 int ntfs_bio_pages(struct ntfs_sb_info *sbi, const struct runs_tree *run,
14501450 struct page **pages, u32 nr_pages, u64 vbo, u32 bytes,
1451− u32 op)
1451+ enum req_op op)
14521452 {
14531453 int err = 0;
14541454 struct bio *new, *bio = NULL;
fs/ntfs3/inode.c+1−1
@@ -629,7 +629,7 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, · 628 unmodified lines
629629 bh->b_size = block_size;
630630 off = vbo & (PAGE_SIZE - 1);
631631 set_bh_page(bh, page, off);
632− ll_rw_block(REQ_OP_READ, 0, 1, &bh);
632+ ll_rw_block(REQ_OP_READ, 1, &bh);
633633 wait_on_buffer(bh);
634634 if (!buffer_uptodate(bh)) {
635635 err = -EIO;
fs/ntfs3/ntfs_fs.h+1−1
@@ -617,7 +617,7 @@ int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, · 616 unmodified lines
617617 struct ntfs_buffers *nb, int sync);
618618 int ntfs_bio_pages(struct ntfs_sb_info *sbi, const struct runs_tree *run,
619619 struct page **pages, u32 nr_pages, u64 vbo, u32 bytes,
620− u32 op);
620+ enum req_op op);
621621 int ntfs_bio_fill_1(struct ntfs_sb_info *sbi, const struct runs_tree *run);
622622 int ntfs_vbo_to_lbo(struct ntfs_sb_info *sbi, const struct runs_tree *run,
623623 u64 vbo, u64 *lbo, u64 *bytes);
fs/ocfs2/aops.c+1−1
@@ -638,7 +638,7 @@ int ocfs2_map_page_blocks(struct page *page, u64 *p_blkno, · 637 unmodified lines
638638 !buffer_new(bh) &&
639639 ocfs2_should_read_blk(inode, page, block_start) &&
640640 (block_start < from || block_end > to)) {
641− ll_rw_block(REQ_OP_READ, 0, 1, &bh);
641+ ll_rw_block(REQ_OP_READ, 1, &bh);
642642 *wait_bh++=bh;
643643 }
644644
fs/ocfs2/buffer_head_io.c+4−4
@@ -64,7 +64,7 @@ int ocfs2_write_block(struct ocfs2_super *osb, struct buffer_head *bh, · 63 unmodified lines
6464
6565 get_bh(bh); /* for end_buffer_write_sync() */
6666 bh->b_end_io = end_buffer_write_sync;
67− submit_bh(REQ_OP_WRITE, 0, bh);
67+ submit_bh(REQ_OP_WRITE, bh);
6868
6969 wait_on_buffer(bh);
7070
@@ -147,7 +147,7 @@ int ocfs2_read_blocks_sync(struct ocfs2_super *osb, u64 block, · 76 unmodified lines
147147
148148 get_bh(bh); /* for end_buffer_read_sync() */
149149 bh->b_end_io = end_buffer_read_sync;
150− submit_bh(REQ_OP_READ, 0, bh);
150+ submit_bh(REQ_OP_READ, bh);
151151 }
152152
153153 read_failure:
@@ -328,7 +328,7 @@ int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr, · 174 unmodified lines
328328 if (validate)
329329 set_buffer_needs_validate(bh);
330330 bh->b_end_io = end_buffer_read_sync;
331− submit_bh(REQ_OP_READ, 0, bh);
331+ submit_bh(REQ_OP_READ, bh);
332332 continue;
333333 }
334334 }
@@ -449,7 +449,7 @@ int ocfs2_write_super_or_backup(struct ocfs2_super *osb, · 114 unmodified lines
449449 get_bh(bh); /* for end_buffer_write_sync() */
450450 bh->b_end_io = end_buffer_write_sync;
451451 ocfs2_compute_meta_ecc(osb->sb, bh->b_data, &di->i_check);
452− submit_bh(REQ_OP_WRITE, 0, bh);
452+ submit_bh(REQ_OP_WRITE, bh);
453453
454454 wait_on_buffer(bh);
455455
fs/ocfs2/cluster/heartbeat.c+35−40
@@ -235,8 +235,6 @@ struct o2hb_region { · 234 unmodified lines
235235 * (hr_steady_iterations == 0) within hr_unsteady_iterations */
236236 atomic_t hr_unsteady_iterations;
237237
238− char hr_dev_name[BDEVNAME_SIZE];
239−
240238 unsigned int hr_timeout_ms;
241239
242240 /* randomized as the region goes up and down so that a node
@@ -287,8 +285,8 @@ static void o2hb_write_timeout(struct work_struct *work) · 44 unmodified lines
287285 container_of(work, struct o2hb_region,
288286 hr_write_timeout_work.work);
289287
290− mlog(ML_ERROR, "Heartbeat write timeout to device %s after %u "
291− "milliseconds\n", reg->hr_dev_name,
288+ mlog(ML_ERROR, "Heartbeat write timeout to device %pg after %u "
289+ "milliseconds\n", reg->hr_bdev,
292290 jiffies_to_msecs(jiffies - reg->hr_last_timeout_start));
293291
294292 if (o2hb_global_heartbeat_active()) {
@@ -383,9 +381,9 @@ static void o2hb_nego_timeout(struct work_struct *work) · 88 unmodified lines
383381
384382 if (master_node == o2nm_this_node()) {
385383 if (!test_bit(master_node, reg->hr_nego_node_bitmap)) {
386− printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%s).\n",
384+ printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg).\n",
387385 o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000,
388− config_item_name(&reg->hr_item), reg->hr_dev_name);
386+ config_item_name(&reg->hr_item), reg->hr_bdev);
389387 set_bit(master_node, reg->hr_nego_node_bitmap);
390388 }
391389 if (memcmp(reg->hr_nego_node_bitmap, live_node_bitmap,
@@ -399,8 +397,8 @@ static void o2hb_nego_timeout(struct work_struct *work) · 7 unmodified lines
399397 return;
400398 }
401399
402− printk(KERN_NOTICE "o2hb: all nodes hb write hung, maybe region %s (%s) is down.\n",
403− config_item_name(&reg->hr_item), reg->hr_dev_name);
400+ printk(KERN_NOTICE "o2hb: all nodes hb write hung, maybe region %s (%pg) is down.\n",
401+ config_item_name(&reg->hr_item), reg->hr_bdev);
404402 /* approve negotiate timeout request. */
405403 o2hb_arm_timeout(reg);
406404
@@ -419,9 +417,9 @@ static void o2hb_nego_timeout(struct work_struct *work) · 12 unmodified lines
419417 }
420418 } else {
421419 /* negotiate timeout with master node. */
422− printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%s), negotiate timeout with node %d.\n",
420+ printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg), negotiate timeout with node %d.\n",
423421 o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, config_item_name(&reg->hr_item),
424− reg->hr_dev_name, master_node);
422+ reg->hr_bdev, master_node);
425423 ret = o2hb_send_nego_msg(reg->hr_key, O2HB_NEGO_TIMEOUT_MSG,
426424 master_node);
427425 if (ret)
@@ -437,8 +435,8 @@ static int o2hb_nego_timeout_handler(struct o2net_msg *msg, u32 len, void *data, · 9 unmodified lines
437435 struct o2hb_nego_msg *nego_msg;
438436
439437 nego_msg = (struct o2hb_nego_msg *)msg->buf;
440− printk(KERN_NOTICE "o2hb: receive negotiate timeout message from node %d on region %s (%s).\n",
441− nego_msg->node_num, config_item_name(&reg->hr_item), reg->hr_dev_name);
438+ printk(KERN_NOTICE "o2hb: receive negotiate timeout message from node %d on region %s (%pg).\n",
439+ nego_msg->node_num, config_item_name(&reg->hr_item), reg->hr_bdev);
442440 if (nego_msg->node_num < O2NM_MAX_NODES)
443441 set_bit(nego_msg->node_num, reg->hr_nego_node_bitmap);
444442 else
@@ -452,8 +450,8 @@ static int o2hb_nego_approve_handler(struct o2net_msg *msg, u32 len, void *data, · 7 unmodified lines
452450 {
453451 struct o2hb_region *reg = data;
454452
455− printk(KERN_NOTICE "o2hb: negotiate timeout approved by master node on region %s (%s).\n",
456− config_item_name(&reg->hr_item), reg->hr_dev_name);
453+ printk(KERN_NOTICE "o2hb: negotiate timeout approved by master node on region %s (%pg).\n",
454+ config_item_name(&reg->hr_item), reg->hr_bdev);
457455 o2hb_arm_timeout(reg);
458456 return 0;
459457 }
@@ -503,8 +501,7 @@ static void o2hb_bio_end_io(struct bio *bio) · 43 unmodified lines
503501 static struct bio *o2hb_setup_one_bio(struct o2hb_region *reg,
504502 struct o2hb_bio_wait_ctxt *wc,
505503 unsigned int *current_slot,
506− unsigned int max_slots, int op,
507− int op_flags)
504+ unsigned int max_slots, blk_opf_t opf)
508505 {
509506 int len, current_page;
510507 unsigned int vec_len, vec_start;
@@ -518,7 +515,7 @@ static struct bio *o2hb_setup_one_bio(struct o2hb_region *reg, · 7 unmodified lines
518515 * GFP_KERNEL that the local node can get fenced. It would be
519516 * nicest if we could pre-allocate these bios and avoid this
520517 * all together. */
521− bio = bio_alloc(reg->hr_bdev, 16, op | op_flags, GFP_ATOMIC);
518+ bio = bio_alloc(reg->hr_bdev, 16, opf, GFP_ATOMIC);
522519 if (!bio) {
523520 mlog(ML_ERROR, "Could not alloc slots BIO!\n");
524521 bio = ERR_PTR(-ENOMEM);
@@ -566,7 +563,7 @@ static int o2hb_read_slots(struct o2hb_region *reg, · 41 unmodified lines
566563
567564 while(current_slot < max_slots) {
568565 bio = o2hb_setup_one_bio(reg, &wc, &current_slot, max_slots,
569− REQ_OP_READ, 0);
566+ REQ_OP_READ);
570567 if (IS_ERR(bio)) {
571568 status = PTR_ERR(bio);
572569 mlog_errno(status);
@@ -598,8 +595,8 @@ static int o2hb_issue_node_write(struct o2hb_region *reg, · 25 unmodified lines
598595
599596 slot = o2nm_this_node();
600597
601− bio = o2hb_setup_one_bio(reg, write_wc, &slot, slot+1, REQ_OP_WRITE,
602− REQ_SYNC);
598+ bio = o2hb_setup_one_bio(reg, write_wc, &slot, slot+1,
599+ REQ_OP_WRITE | REQ_SYNC);
603600 if (IS_ERR(bio)) {
604601 status = PTR_ERR(bio);
605602 mlog_errno(status);
@@ -689,8 +686,8 @@ static int o2hb_check_own_slot(struct o2hb_region *reg) · 83 unmodified lines
689686 else
690687 errstr = ERRSTR3;
691688
692− mlog(ML_ERROR, "%s (%s): expected(%u:0x%llx, 0x%llx), "
693− "ondisk(%u:0x%llx, 0x%llx)\n", errstr, reg->hr_dev_name,
689+ mlog(ML_ERROR, "%s (%pg): expected(%u:0x%llx, 0x%llx), "
690+ "ondisk(%u:0x%llx, 0x%llx)\n", errstr, reg->hr_bdev,
694691 slot->ds_node_num, (unsigned long long)slot->ds_last_generation,
695692 (unsigned long long)slot->ds_last_time, hb_block->hb_node,
696693 (unsigned long long)le64_to_cpu(hb_block->hb_generation),
@@ -863,8 +860,8 @@ static void o2hb_set_quorum_device(struct o2hb_region *reg) · 166 unmodified lines
863860 sizeof(o2hb_live_node_bitmap)))
864861 goto unlock;
865862
866− printk(KERN_NOTICE "o2hb: Region %s (%s) is now a quorum device\n",
867− config_item_name(&reg->hr_item), reg->hr_dev_name);
863+ printk(KERN_NOTICE "o2hb: Region %s (%pg) is now a quorum device\n",
864+ config_item_name(&reg->hr_item), reg->hr_bdev);
868865
869866 set_bit(reg->hr_region_num, o2hb_quorum_region_bitmap);
870867
@@ -922,8 +919,8 @@ static int o2hb_check_slot(struct o2hb_region *reg, · 51 unmodified lines
922919 /* The node is live but pushed out a bad crc. We
923920 * consider it a transient miss but don't populate any
924921 * other values as they may be junk. */
925− mlog(ML_ERROR, "Node %d has written a bad crc to %s\n",
926− slot->ds_node_num, reg->hr_dev_name);
922+ mlog(ML_ERROR, "Node %d has written a bad crc to %pg\n",
923+ slot->ds_node_num, reg->hr_bdev);
927924 o2hb_dump_slot(hb_block);
928925
929926 slot->ds_equal_samples++;
@@ -1002,11 +999,11 @@ fire_callbacks: · 72 unmodified lines
1002999 slot_dead_ms = le32_to_cpu(hb_block->hb_dead_ms);
10031000 if (slot_dead_ms && slot_dead_ms != dead_ms) {
10041001 /* TODO: Perhaps we can fail the region here. */
1005− mlog(ML_ERROR, "Node %d on device %s has a dead count "
1002+ mlog(ML_ERROR, "Node %d on device %pg has a dead count "
10061003 "of %u ms, but our count is %u ms.\n"
10071004 "Please double check your configuration values "
10081005 "for 'O2CB_HEARTBEAT_THRESHOLD'\n",
1009− slot->ds_node_num, reg->hr_dev_name, slot_dead_ms,
1006+ slot->ds_node_num, reg->hr_bdev, slot_dead_ms,
10101007 dead_ms);
10111008 }
10121009 goto out;
@@ -1145,8 +1142,8 @@ static int o2hb_do_disk_heartbeat(struct o2hb_region *reg) · 132 unmodified lines
11451142 /* Do not re-arm the write timeout on I/O error - we
11461143 * can't be sure that the new block ever made it to
11471144 * disk */
1148− mlog(ML_ERROR, "Write error %d on device \"%s\"\n",
1149− write_wc.wc_error, reg->hr_dev_name);
1145+ mlog(ML_ERROR, "Write error %d on device \"%pg\"\n",
1146+ write_wc.wc_error, reg->hr_bdev);
11501147 ret = write_wc.wc_error;
11511148 goto bail;
11521149 }
@@ -1170,9 +1167,9 @@ bail: · 17 unmodified lines
11701167 if (atomic_read(&reg->hr_steady_iterations) != 0) {
11711168 if (atomic_dec_and_test(&reg->hr_unsteady_iterations)) {
11721169 printk(KERN_NOTICE "o2hb: Unable to stabilize "
1173− "heartbeat on region %s (%s)\n",
1170+ "heartbeat on region %s (%pg)\n",
11741171 config_item_name(&reg->hr_item),
1175− reg->hr_dev_name);
1172+ reg->hr_bdev);
11761173 atomic_set(&reg->hr_steady_iterations, 0);
11771174 reg->hr_aborted_start = 1;
11781175 wake_up(&o2hb_steady_queue);
@@ -1494,7 +1491,7 @@ static void o2hb_region_release(struct config_item *item) · 315 unmodified lines
14941491 struct page *page;
14951492 struct o2hb_region *reg = to_o2hb_region(item);
14961493
1497− mlog(ML_HEARTBEAT, "hb region release (%s)\n", reg->hr_dev_name);
1494+ mlog(ML_HEARTBEAT, "hb region release (%pg)\n", reg->hr_bdev);
14981495
14991496 kfree(reg->hr_tmp_block);
15001497
@@ -1641,7 +1638,7 @@ static ssize_t o2hb_region_dev_show(struct config_item *item, char *page) · 140 unmodified lines
16411638 unsigned int ret = 0;
16421639
16431640 if (to_o2hb_region(item)->hr_bdev)
1644− ret = sprintf(page, "%s\n", to_o2hb_region(item)->hr_dev_name);
1641+ ret = sprintf(page, "%pg\n", to_o2hb_region(item)->hr_bdev);
16451642
16461643 return ret;
16471644 }
@@ -1798,8 +1795,6 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, · 150 unmodified lines
17981795 goto out2;
17991796 }
18001797
1801− bdevname(reg->hr_bdev, reg->hr_dev_name);
1802−
18031798 sectsize = bdev_logical_block_size(reg->hr_bdev);
18041799 if (sectsize != reg->hr_block_bytes) {
18051800 mlog(ML_ERROR,
@@ -1895,8 +1890,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, · 89 unmodified lines
18951890 ret = -EIO;
18961891
18971892 if (hb_task && o2hb_global_heartbeat_active())
1898− printk(KERN_NOTICE "o2hb: Heartbeat started on region %s (%s)\n",
1899− config_item_name(&reg->hr_item), reg->hr_dev_name);
1893+ printk(KERN_NOTICE "o2hb: Heartbeat started on region %s (%pg)\n",
1894+ config_item_name(&reg->hr_item), reg->hr_bdev);
19001895
19011896 out3:
19021897 if (ret < 0) {
@@ -2088,10 +2083,10 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group, · 185 unmodified lines
20882083 quorum_region = 1;
20892084 clear_bit(reg->hr_region_num, o2hb_quorum_region_bitmap);
20902085 spin_unlock(&o2hb_live_lock);
2091− printk(KERN_NOTICE "o2hb: Heartbeat %s on region %s (%s)\n",
2086+ printk(KERN_NOTICE "o2hb: Heartbeat %s on region %s (%pg)\n",
20922087 ((atomic_read(&reg->hr_steady_iterations) == 0) ?
20932088 "stopped" : "start aborted"), config_item_name(item),
2094− reg->hr_dev_name);
2089+ reg->hr_bdev);
20952090 }
20962091
20972092 /*
fs/ocfs2/super.c+1−1
@@ -1764,7 +1764,7 @@ static int ocfs2_get_sector(struct super_block *sb, · 1763 unmodified lines
17641764 if (!buffer_dirty(*bh))
17651765 clear_buffer_uptodate(*bh);
17661766 unlock_buffer(*bh);
1767− ll_rw_block(REQ_OP_READ, 0, 1, bh);
1767+ ll_rw_block(REQ_OP_READ, 1, bh);
17681768 wait_on_buffer(*bh);
17691769 if (!buffer_uptodate(*bh)) {
17701770 mlog_errno(-EIO);
fs/reiserfs/inode.c+2−2
@@ -2656,7 +2656,7 @@ static int reiserfs_write_full_page(struct page *page, · 2655 unmodified lines
26562656 do {
26572657 struct buffer_head *next = bh->b_this_page;
26582658 if (buffer_async_write(bh)) {
2659− submit_bh(REQ_OP_WRITE, 0, bh);
2659+ submit_bh(REQ_OP_WRITE, bh);
26602660 nr++;
26612661 }
26622662 put_bh(bh);
@@ -2716,7 +2716,7 @@ fail: · 53 unmodified lines
27162716 struct buffer_head *next = bh->b_this_page;
27172717 if (buffer_async_write(bh)) {
27182718 clear_buffer_dirty(bh);
2719− submit_bh(REQ_OP_WRITE, 0, bh);
2719+ submit_bh(REQ_OP_WRITE, bh);
27202720 nr++;
27212721 }
27222722 put_bh(bh);
fs/reiserfs/journal.c+6−6
@@ -650,7 +650,7 @@ static void submit_logged_buffer(struct buffer_head *bh) · 649 unmodified lines
650650 BUG();
651651 if (!buffer_uptodate(bh))
652652 BUG();
653− submit_bh(REQ_OP_WRITE, 0, bh);
653+ submit_bh(REQ_OP_WRITE, bh);
654654 }
655655
656656 static void submit_ordered_buffer(struct buffer_head *bh)
@@ -660,7 +660,7 @@ static void submit_ordered_buffer(struct buffer_head *bh) · 3 unmodified lines
660660 clear_buffer_dirty(bh);
661661 if (!buffer_uptodate(bh))
662662 BUG();
663− submit_bh(REQ_OP_WRITE, 0, bh);
663+ submit_bh(REQ_OP_WRITE, bh);
664664 }
665665
666666 #define CHUNK_SIZE 32
@@ -868,7 +868,7 @@ loop_next: · 201 unmodified lines
868868 */
869869 if (buffer_dirty(bh) && unlikely(bh->b_page->mapping == NULL)) {
870870 spin_unlock(lock);
871− ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
871+ ll_rw_block(REQ_OP_WRITE, 1, &bh);
872872 spin_lock(lock);
873873 }
874874 put_bh(bh);
@@ -1054,7 +1054,7 @@ static int flush_commit_list(struct super_block *s, · 179 unmodified lines
10541054 if (tbh) {
10551055 if (buffer_dirty(tbh)) {
10561056 depth = reiserfs_write_unlock_nested(s);
1057− ll_rw_block(REQ_OP_WRITE, 0, 1, &tbh);
1057+ ll_rw_block(REQ_OP_WRITE, 1, &tbh);
10581058 reiserfs_write_lock_nested(s, depth);
10591059 }
10601060 put_bh(tbh) ;
@@ -2240,7 +2240,7 @@ abort_replay: · 1179 unmodified lines
22402240 }
22412241 }
22422242 /* read in the log blocks, memcpy to the corresponding real block */
2243− ll_rw_block(REQ_OP_READ, 0, get_desc_trans_len(desc), log_blocks);
2243+ ll_rw_block(REQ_OP_READ, get_desc_trans_len(desc), log_blocks);
22442244 for (i = 0; i < get_desc_trans_len(desc); i++) {
22452245
22462246 wait_on_buffer(log_blocks[i]);
@@ -2342,7 +2342,7 @@ static struct buffer_head *reiserfs_breada(struct block_device *dev, · 95 unmodified lines
23422342 } else
23432343 bhlist[j++] = bh;
23442344 }
2345− ll_rw_block(REQ_OP_READ, 0, j, bhlist);
2345+ ll_rw_block(REQ_OP_READ, j, bhlist);
23462346 for (i = 1; i < j; i++)
23472347 brelse(bhlist[i]);
23482348 bh = bhlist[0];
fs/reiserfs/stree.c+2−2
@@ -579,7 +579,7 @@ static int search_by_key_reada(struct super_block *s, · 578 unmodified lines
579579 if (!buffer_uptodate(bh[j])) {
580580 if (depth == -1)
581581 depth = reiserfs_write_unlock_nested(s);
582− ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, bh + j);
582+ ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, bh + j);
583583 }
584584 brelse(bh[j]);
585585 }
@@ -685,7 +685,7 @@ int search_by_key(struct super_block *sb, const struct cpu_key *key, · 99 unmodified lines
685685 if (!buffer_uptodate(bh) && depth == -1)
686686 depth = reiserfs_write_unlock_nested(sb);
687687
688− ll_rw_block(REQ_OP_READ, 0, 1, &bh);
688+ ll_rw_block(REQ_OP_READ, 1, &bh);
689689 wait_on_buffer(bh);
690690
691691 if (depth != -1)
fs/reiserfs/super.c+1−1
@@ -1702,7 +1702,7 @@ static int read_super_block(struct super_block *s, int offset) · 1701 unmodified lines
17021702 /* after journal replay, reread all bitmap and super blocks */
17031703 static int reread_meta_blocks(struct super_block *s)
17041704 {
1705− ll_rw_block(REQ_OP_READ, 0, 1, &SB_BUFFER_WITH_SB(s));
1705+ ll_rw_block(REQ_OP_READ, 1, &SB_BUFFER_WITH_SB(s));
17061706 wait_on_buffer(SB_BUFFER_WITH_SB(s));
17071707 if (!buffer_uptodate(SB_BUFFER_WITH_SB(s))) {
17081708 reiserfs_warning(s, "reiserfs-2504", "error reading the super");
fs/udf/dir.c+1−1
@@ -130,7 +130,7 @@ static int udf_readdir(struct file *file, struct dir_context *ctx) · 129 unmodified lines
130130 brelse(tmp);
131131 }
132132 if (num) {
133− ll_rw_block(REQ_OP_READ, REQ_RAHEAD, num, bha);
133+ ll_rw_block(REQ_OP_READ | REQ_RAHEAD, num, bha);
134134 for (i = 0; i < num; i++)
135135 brelse(bha[i]);
136136 }
fs/udf/directory.c+1−1
@@ -89,7 +89,7 @@ struct fileIdentDesc *udf_fileident_read(struct inode *dir, loff_t *nf_pos, · 88 unmodified lines
8989 brelse(tmp);
9090 }
9191 if (num) {
92− ll_rw_block(REQ_OP_READ, REQ_RAHEAD, num, bha);
92+ ll_rw_block(REQ_OP_READ | REQ_RAHEAD, num, bha);
9393 for (i = 0; i < num; i++)
9494 brelse(bha[i]);
9595 }
fs/udf/inode.c+1−1
@@ -1214,7 +1214,7 @@ struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, · 1213 unmodified lines
12141214 if (buffer_uptodate(bh))
12151215 return bh;
12161216
1217− ll_rw_block(REQ_OP_READ, 0, 1, &bh);
1217+ ll_rw_block(REQ_OP_READ, 1, &bh);
12181218
12191219 wait_on_buffer(bh);
12201220 if (buffer_uptodate(bh))
fs/ufs/balloc.c+1−1
@@ -296,7 +296,7 @@ static void ufs_change_blocknr(struct inode *inode, sector_t beg, · 295 unmodified lines
296296 if (!buffer_mapped(bh))
297297 map_bh(bh, inode->i_sb, oldb + pos);
298298 if (!buffer_uptodate(bh)) {
299− ll_rw_block(REQ_OP_READ, 0, 1, &bh);
299+ ll_rw_block(REQ_OP_READ, 1, &bh);
300300 wait_on_buffer(bh);
301301 if (!buffer_uptodate(bh)) {
302302 ufs_error(inode->i_sb, __func__,
fs/xfs/xfs_bio_io.c+1−1
@@ -15,7 +15,7 @@ xfs_rw_bdev( · 14 unmodified lines
1515 sector_t sector,
1616 unsigned int count,
1717 char *data,
18− unsigned int op)
18+ enum req_op op)
1919
2020 {
2121 unsigned int is_vmalloc = is_vmalloc_addr(data);
fs/xfs/xfs_buf.c+2−2
@@ -1416,7 +1416,7 @@ xfs_buf_ioapply_map( · 1415 unmodified lines
14161416 int map,
14171417 int *buf_offset,
14181418 int *count,
1419− int op)
1419+ blk_opf_t op)
14201420 {
14211421 int page_index;
14221422 unsigned int total_nr_pages = bp->b_page_count;
@@ -1493,7 +1493,7 @@ _xfs_buf_ioapply( · 70 unmodified lines
14931493 struct xfs_buf *bp)
14941494 {
14951495 struct blk_plug plug;
1496− int op;
1496+ blk_opf_t op;
14971497 int offset;
14981498 int size;
14991499 int i;
fs/xfs/xfs_linux.h+1−1
@@ -196,7 +196,7 @@ static inline uint64_t howmany_64(uint64_t x, uint32_t y) · 195 unmodified lines
196196 }
197197
198198 int xfs_rw_bdev(struct block_device *bdev, sector_t sector, unsigned int count,
199− char *data, unsigned int op);
199+ char *data, enum req_op op);
200200
201201 #define ASSERT_ALWAYS(expr) \
202202 (likely(expr) ? (void)0 : assfail(NULL, #expr, __FILE__, __LINE__))
fs/xfs/xfs_log_recover.c+1−1
@@ -122,7 +122,7 @@ xlog_do_io( · 121 unmodified lines
122122 xfs_daddr_t blk_no,
123123 unsigned int nbblks,
124124 char *data,
125− unsigned int op)
125+ enum req_op op)
126126 {
127127 int error;
128128
fs/zonefs/super.c+10−12
@@ -60,8 +60,7 @@ static void zonefs_account_active(struct inode *inode) · 59 unmodified lines
6060 }
6161 }
6262
63−static inline int zonefs_zone_mgmt(struct inode *inode,
64− enum req_opf op)
63+static inline int zonefs_zone_mgmt(struct inode *inode, enum req_op op)
6564 {
6665 struct zonefs_inode_info *zi = ZONEFS_I(inode);
6766 int ret;
@@ -525,7 +524,7 @@ static int zonefs_file_truncate(struct inode *inode, loff_t isize) · 457 unmodified lines
525524 {
526525 struct zonefs_inode_info *zi = ZONEFS_I(inode);
527526 loff_t old_isize;
528− enum req_opf op;
527+ enum req_op op;
529528 int ret = 0;
530529
531530 /*
@@ -1394,7 +1393,7 @@ static void zonefs_init_dir_inode(struct inode *parent, struct inode *inode, · 862 unmodified lines
13941393 {
13951394 struct super_block *sb = parent->i_sb;
13961395
1397− inode->i_ino = blkdev_nr_zones(sb->s_bdev->bd_disk) + type + 1;
1396+ inode->i_ino = bdev_nr_zones(sb->s_bdev) + type + 1;
13981397 inode_init_owner(&init_user_ns, inode, parent, S_IFDIR | 0555);
13991398 inode->i_op = &zonefs_dir_inode_operations;
14001399 inode->i_fop = &simple_dir_operations;
@@ -1540,7 +1539,7 @@ static int zonefs_create_zgroup(struct zonefs_zone_data *zd, · 139 unmodified lines
15401539 /*
15411540 * The first zone contains the super block: skip it.
15421541 */
1543− end = zd->zones + blkdev_nr_zones(sb->s_bdev->bd_disk);
1542+ end = zd->zones + bdev_nr_zones(sb->s_bdev);
15441543 for (zone = &zd->zones[1]; zone < end; zone = next) {
15451544
15461545 next = zone + 1;
@@ -1635,8 +1634,8 @@ static int zonefs_get_zone_info(struct zonefs_zone_data *zd) · 88 unmodified lines
16351634 struct block_device *bdev = zd->sb->s_bdev;
16361635 int ret;
16371636
1638− zd->zones = kvcalloc(blkdev_nr_zones(bdev->bd_disk),
1639− sizeof(struct blk_zone), GFP_KERNEL);
1637+ zd->zones = kvcalloc(bdev_nr_zones(bdev), sizeof(struct blk_zone),
1638+ GFP_KERNEL);
16401639 if (!zd->zones)
16411640 return -ENOMEM;
16421641
@@ -1648,9 +1647,9 @@ static int zonefs_get_zone_info(struct zonefs_zone_data *zd) · 5 unmodified lines
16481647 return ret;
16491648 }
16501649
1651− if (ret != blkdev_nr_zones(bdev->bd_disk)) {
1650+ if (ret != bdev_nr_zones(bdev)) {
16521651 zonefs_err(zd->sb, "Invalid zone report (%d/%u zones)\n",
1653− ret, blkdev_nr_zones(bdev->bd_disk));
1652+ ret, bdev_nr_zones(bdev));
16541653 return -EIO;
16551654 }
16561655
@@ -1816,8 +1815,7 @@ static int zonefs_fill_super(struct super_block *sb, void *data, int silent) · 159 unmodified lines
18161815 if (ret)
18171816 goto cleanup;
18181817
1819− zonefs_info(sb, "Mounting %u zones",
1820− blkdev_nr_zones(sb->s_bdev->bd_disk));
1818+ zonefs_info(sb, "Mounting %u zones", bdev_nr_zones(sb->s_bdev));
18211819
18221820 if (!sbi->s_max_wro_seq_files &&
18231821 !sbi->s_max_active_seq_files &&
@@ -1833,7 +1831,7 @@ static int zonefs_fill_super(struct super_block *sb, void *data, int silent) · 9 unmodified lines
18331831 if (!inode)
18341832 goto cleanup;
18351833
1836− inode->i_ino = blkdev_nr_zones(sb->s_bdev->bd_disk);
1834+ inode->i_ino = bdev_nr_zones(sb->s_bdev);
18371835 inode->i_mode = S_IFDIR | 0555;
18381836 inode->i_ctime = inode->i_mtime = inode->i_atime = current_time(inode);
18391837 inode->i_op = &zonefs_dir_inode_operations;
fs/zonefs/trace.h+2−2
@@ -20,12 +20,12 @@ · 19 unmodified lines
2020 #define show_dev(dev) MAJOR(dev), MINOR(dev)
2121
2222 TRACE_EVENT(zonefs_zone_mgmt,
23− TP_PROTO(struct inode *inode, enum req_opf op),
23+ TP_PROTO(struct inode *inode, enum req_op op),
2424 TP_ARGS(inode, op),
2525 TP_STRUCT__entry(
2626 __field(dev_t, dev)
2727 __field(ino_t, ino)
28− __field(int, op)
28+ __field(enum req_op, op)
2929 __field(sector_t, sector)
3030 __field(sector_t, nr_sectors)
3131 ),
include/linux/bio.h+5−5
@@ -405,7 +405,7 @@ extern void bioset_exit(struct bio_set *); · 404 unmodified lines
405405 extern int biovec_init_pool(mempool_t *pool, int pool_entries);
406406
407407 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
408− unsigned int opf, gfp_t gfp_mask,
408+ blk_opf_t opf, gfp_t gfp_mask,
409409 struct bio_set *bs);
410410 struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask);
411411 extern void bio_put(struct bio *);
@@ -418,7 +418,7 @@ int bio_init_clone(struct block_device *bdev, struct bio *bio, · 6 unmodified lines
418418 extern struct bio_set fs_bio_set;
419419
420420 static inline struct bio *bio_alloc(struct block_device *bdev,
421− unsigned short nr_vecs, unsigned int opf, gfp_t gfp_mask)
421+ unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask)
422422 {
423423 return bio_alloc_bioset(bdev, nr_vecs, opf, gfp_mask, &fs_bio_set);
424424 }
@@ -456,9 +456,9 @@ struct request_queue; · 31 unmodified lines
456456
457457 extern int submit_bio_wait(struct bio *bio);
458458 void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
459− unsigned short max_vecs, unsigned int opf);
459+ unsigned short max_vecs, blk_opf_t opf);
460460 extern void bio_uninit(struct bio *);
461−void bio_reset(struct bio *bio, struct block_device *bdev, unsigned int opf);
461+void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf);
462462 void bio_chain(struct bio *, struct bio *);
463463
464464 int bio_add_page(struct bio *, struct page *, unsigned len, unsigned off);
@@ -789,6 +789,6 @@ static inline void bio_clear_polled(struct bio *bio) · 324 unmodified lines
789789 }
790790
791791 struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
792− unsigned int nr_pages, unsigned int opf, gfp_t gfp);
792+ unsigned int nr_pages, blk_opf_t opf, gfp_t gfp);
793793
794794 #endif /* __LINUX_BIO_H */
include/linux/blk-mq.h+22−12
@@ -57,6 +57,7 @@ typedef __u32 __bitwise req_flags_t; · 56 unmodified lines
5757 #define RQF_TIMED_OUT ((__force req_flags_t)(1 << 21))
5858 /* queue has elevator attached */
5959 #define RQF_ELV ((__force req_flags_t)(1 << 22))
60+#define RQF_RESV ((__force req_flags_t)(1 << 23))
6061
6162 /* flags that prevent us from merging requests: */
6263 #define RQF_NOMERGE_FLAGS \
@@ -79,7 +80,7 @@ struct request { · 16 unmodified lines
7980 struct blk_mq_ctx *mq_ctx;
8081 struct blk_mq_hw_ctx *mq_hctx;
8182
82− unsigned int cmd_flags; /* op and common flags */
83+ blk_opf_t cmd_flags; /* op and common flags */
8384 req_flags_t rq_flags;
8485
8586 int tag;
@@ -197,8 +198,10 @@ struct request { · 111 unmodified lines
197198 void *end_io_data;
198199 };
199200
200−#define req_op(req) \
201− ((req)->cmd_flags & REQ_OP_MASK)
201+static inline enum req_op req_op(const struct request *req)
202+{
203+ return req->cmd_flags & REQ_OP_MASK;
204+}
202205
203206 static inline bool blk_rq_is_passthrough(struct request *rq)
204207 {
@@ -519,7 +522,7 @@ struct blk_mq_queue_data { · 314 unmodified lines
519522 bool last;
520523 };
521524
522−typedef bool (busy_tag_iter_fn)(struct request *, void *, bool);
525+typedef bool (busy_tag_iter_fn)(struct request *, void *);
523526
524527 /**
525528 * struct blk_mq_ops - Callback functions that implements block driver
@@ -574,7 +577,7 @@ struct blk_mq_ops { · 48 unmodified lines
574577 /**
575578 * @timeout: Called on request timeout.
576579 */
577− enum blk_eh_timer_return (*timeout)(struct request *, bool);
580+ enum blk_eh_timer_return (*timeout)(struct request *);
578581
579582 /**
580583 * @poll: Called to poll for completion of a specific tag.
@@ -686,10 +689,12 @@ struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata, · 105 unmodified lines
686689 \
687690 __blk_mq_alloc_disk(set, queuedata, &__key); \
688691 })
692+struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
693+ struct lock_class_key *lkclass);
689694 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *);
690695 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
691696 struct request_queue *q);
692−void blk_mq_unregister_dev(struct device *, struct request_queue *);
697+void blk_mq_destroy_queue(struct request_queue *);
693698
694699 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set);
695700 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
@@ -710,10 +715,10 @@ enum { · 14 unmodified lines
710715 BLK_MQ_REQ_PM = (__force blk_mq_req_flags_t)(1 << 2),
711716 };
712717
713−struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
718+struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
714719 blk_mq_req_flags_t flags);
715720 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
716− unsigned int op, blk_mq_req_flags_t flags,
721+ blk_opf_t opf, blk_mq_req_flags_t flags,
717722 unsigned int hctx_idx);
718723
719724 /*
@@ -823,6 +828,11 @@ static inline bool blk_mq_need_time_stamp(struct request *rq) · 103 unmodified lines
823828 return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV));
824829 }
825830
831+static inline bool blk_mq_is_reserved_rq(struct request *rq)
832+{
833+ return rq->rq_flags & RQF_RESV;
834+}
835+
826836 /*
827837 * Batched completions only work when there is no I/O error and no special
828838 * ->end_io handler.
@@ -1121,12 +1131,12 @@ void blk_dump_rq_flags(struct request *, char *); · 292 unmodified lines
11211131 #ifdef CONFIG_BLK_DEV_ZONED
11221132 static inline unsigned int blk_rq_zone_no(struct request *rq)
11231133 {
1124− return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
1134+ return disk_zone_no(rq->q->disk, blk_rq_pos(rq));
11251135 }
11261136
11271137 static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
11281138 {
1129− return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq));
1139+ return disk_zone_is_seq(rq->q->disk, blk_rq_pos(rq));
11301140 }
11311141
11321142 bool blk_req_needs_zone_write_lock(struct request *rq);
@@ -1148,8 +1158,8 @@ static inline void blk_req_zone_write_unlock(struct request *rq) · 15 unmodified lines
11481158
11491159 static inline bool blk_req_zone_is_write_locked(struct request *rq)
11501160 {
1151− return rq->q->seq_zones_wlock &&
1152− test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock);
1161+ return rq->q->disk->seq_zones_wlock &&
1162+ test_bit(blk_rq_zone_no(rq), rq->q->disk->seq_zones_wlock);
11531163 }
11541164
11551165 static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
include/linux/blk_types.h+63−56
@@ -240,6 +240,8 @@ static inline void bio_issue_init(struct bio_issue *issue, · 239 unmodified lines
240240 ((u64)size << BIO_ISSUE_SIZE_SHIFT));
241241 }
242242
243+typedef __u32 __bitwise blk_opf_t;
244+
243245 typedef unsigned int blk_qc_t;
244246 #define BLK_QC_T_NONE -1U
245247
@@ -250,7 +252,7 @@ typedef unsigned int blk_qc_t; · 4 unmodified lines
250252 struct bio {
251253 struct bio *bi_next; /* request queue link */
252254 struct block_device *bi_bdev;
253− unsigned int bi_opf; /* bottom bits REQ_OP, top bits
255+ blk_opf_t bi_opf; /* bottom bits REQ_OP, top bits
254256 * req_flags.
255257 */
256258 unsigned short bi_flags; /* BIO_* below */
@@ -337,8 +339,12 @@ enum { · 80 unmodified lines
337339
338340 typedef __u32 __bitwise blk_mq_req_flags_t;
339341
340−/*
341− * Operations and flags common to the bio and request structures.
342+#define REQ_OP_BITS 8
343+#define REQ_OP_MASK (__force blk_opf_t)((1 << REQ_OP_BITS) - 1)
344+#define REQ_FLAG_BITS 24
345+
346+/**
347+ * enum req_op - Operations common to the bio and request structures.
342348 * We use 8 bits for encoding the operation, and the remaining 24 for flags.
343349 *
344350 * The least significant bit of the operation number indicates the data
@@ -350,41 +356,37 @@ typedef __u32 __bitwise blk_mq_req_flags_t; · 5 unmodified lines
350356 * If a operation does not transfer data the least significant bit has no
351357 * meaning.
352358 */
353−#define REQ_OP_BITS 8
354−#define REQ_OP_MASK ((1 << REQ_OP_BITS) - 1)
355−#define REQ_FLAG_BITS 24
356−
357−enum req_opf {
359+enum req_op {
358360 /* read sectors from the device */
359− REQ_OP_READ = 0,
361+ REQ_OP_READ = (__force blk_opf_t)0,
360362 /* write sectors to the device */
361− REQ_OP_WRITE = 1,
363+ REQ_OP_WRITE = (__force blk_opf_t)1,
362364 /* flush the volatile write cache */
363− REQ_OP_FLUSH = 2,
365+ REQ_OP_FLUSH = (__force blk_opf_t)2,
364366 /* discard sectors */
365− REQ_OP_DISCARD = 3,
367+ REQ_OP_DISCARD = (__force blk_opf_t)3,
366368 /* securely erase sectors */
367− REQ_OP_SECURE_ERASE = 5,
369+ REQ_OP_SECURE_ERASE = (__force blk_opf_t)5,
368370 /* write the zero filled sector many times */
369− REQ_OP_WRITE_ZEROES = 9,
371+ REQ_OP_WRITE_ZEROES = (__force blk_opf_t)9,
370372 /* Open a zone */
371− REQ_OP_ZONE_OPEN = 10,
373+ REQ_OP_ZONE_OPEN = (__force blk_opf_t)10,
372374 /* Close a zone */
373− REQ_OP_ZONE_CLOSE = 11,
375+ REQ_OP_ZONE_CLOSE = (__force blk_opf_t)11,
374376 /* Transition a zone to full */
375− REQ_OP_ZONE_FINISH = 12,
377+ REQ_OP_ZONE_FINISH = (__force blk_opf_t)12,
376378 /* write data at the current zone write pointer */
377− REQ_OP_ZONE_APPEND = 13,
379+ REQ_OP_ZONE_APPEND = (__force blk_opf_t)13,
378380 /* reset a zone write pointer */
379− REQ_OP_ZONE_RESET = 15,
381+ REQ_OP_ZONE_RESET = (__force blk_opf_t)15,
380382 /* reset all the zone present on the device */
381− REQ_OP_ZONE_RESET_ALL = 17,
383+ REQ_OP_ZONE_RESET_ALL = (__force blk_opf_t)17,
382384
383385 /* Driver private requests */
384− REQ_OP_DRV_IN = 34,
385− REQ_OP_DRV_OUT = 35,
386+ REQ_OP_DRV_IN = (__force blk_opf_t)34,
387+ REQ_OP_DRV_OUT = (__force blk_opf_t)35,
386388
387− REQ_OP_LAST,
389+ REQ_OP_LAST = (__force blk_opf_t)36,
388390 };
389391
390392 enum req_flag_bits {
@@ -425,28 +427,31 @@ enum req_flag_bits { · 34 unmodified lines
425427 __REQ_NR_BITS, /* stops here */
426428 };
427429
428−#define REQ_FAILFAST_DEV (1ULL << __REQ_FAILFAST_DEV)
429−#define REQ_FAILFAST_TRANSPORT (1ULL << __REQ_FAILFAST_TRANSPORT)
430−#define REQ_FAILFAST_DRIVER (1ULL << __REQ_FAILFAST_DRIVER)
431−#define REQ_SYNC (1ULL << __REQ_SYNC)
432−#define REQ_META (1ULL << __REQ_META)
433−#define REQ_PRIO (1ULL << __REQ_PRIO)
434−#define REQ_NOMERGE (1ULL << __REQ_NOMERGE)
435−#define REQ_IDLE (1ULL << __REQ_IDLE)
436−#define REQ_INTEGRITY (1ULL << __REQ_INTEGRITY)
437−#define REQ_FUA (1ULL << __REQ_FUA)
438−#define REQ_PREFLUSH (1ULL << __REQ_PREFLUSH)
439−#define REQ_RAHEAD (1ULL << __REQ_RAHEAD)
440−#define REQ_BACKGROUND (1ULL << __REQ_BACKGROUND)
441−#define REQ_NOWAIT (1ULL << __REQ_NOWAIT)
442−#define REQ_CGROUP_PUNT (1ULL << __REQ_CGROUP_PUNT)
443−
444−#define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP)
445−#define REQ_POLLED (1ULL << __REQ_POLLED)
446−#define REQ_ALLOC_CACHE (1ULL << __REQ_ALLOC_CACHE)
447−
448−#define REQ_DRV (1ULL << __REQ_DRV)
449−#define REQ_SWAP (1ULL << __REQ_SWAP)
430+#define REQ_FAILFAST_DEV \
431+ (__force blk_opf_t)(1ULL << __REQ_FAILFAST_DEV)
432+#define REQ_FAILFAST_TRANSPORT \
433+ (__force blk_opf_t)(1ULL << __REQ_FAILFAST_TRANSPORT)
434+#define REQ_FAILFAST_DRIVER \
435+ (__force blk_opf_t)(1ULL << __REQ_FAILFAST_DRIVER)
436+#define REQ_SYNC (__force blk_opf_t)(1ULL << __REQ_SYNC)
437+#define REQ_META (__force blk_opf_t)(1ULL << __REQ_META)
438+#define REQ_PRIO (__force blk_opf_t)(1ULL << __REQ_PRIO)
439+#define REQ_NOMERGE (__force blk_opf_t)(1ULL << __REQ_NOMERGE)
440+#define REQ_IDLE (__force blk_opf_t)(1ULL << __REQ_IDLE)
441+#define REQ_INTEGRITY (__force blk_opf_t)(1ULL << __REQ_INTEGRITY)
442+#define REQ_FUA (__force blk_opf_t)(1ULL << __REQ_FUA)
443+#define REQ_PREFLUSH (__force blk_opf_t)(1ULL << __REQ_PREFLUSH)
444+#define REQ_RAHEAD (__force blk_opf_t)(1ULL << __REQ_RAHEAD)
445+#define REQ_BACKGROUND (__force blk_opf_t)(1ULL << __REQ_BACKGROUND)
446+#define REQ_NOWAIT (__force blk_opf_t)(1ULL << __REQ_NOWAIT)
447+#define REQ_CGROUP_PUNT (__force blk_opf_t)(1ULL << __REQ_CGROUP_PUNT)
448+
449+#define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP)
450+#define REQ_POLLED (__force blk_opf_t)(1ULL << __REQ_POLLED)
451+#define REQ_ALLOC_CACHE (__force blk_opf_t)(1ULL << __REQ_ALLOC_CACHE)
452+
453+#define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV)
454+#define REQ_SWAP (__force blk_opf_t)(1ULL << __REQ_SWAP)
450455
451456 #define REQ_FAILFAST_MASK \
452457 (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)
@@ -463,26 +468,28 @@ enum stat_group { · 10 unmodified lines
463468 NR_STAT_GROUPS
464469 };
465470
466−#define bio_op(bio) \
467− ((bio)->bi_opf & REQ_OP_MASK)
471+static inline enum req_op bio_op(const struct bio *bio)
472+{
473+ return bio->bi_opf & REQ_OP_MASK;
474+}
468475
469476 /* obsolete, don't use in new code */
470−static inline void bio_set_op_attrs(struct bio *bio, unsigned op,
471− unsigned op_flags)
477+static inline void bio_set_op_attrs(struct bio *bio, enum req_op op,
478+ blk_opf_t op_flags)
472479 {
473480 bio->bi_opf = op | op_flags;
474481 }
475482
476−static inline bool op_is_write(unsigned int op)
483+static inline bool op_is_write(blk_opf_t op)
477484 {
478− return (op & 1);
485+ return !!(op & (__force blk_opf_t)1);
479486 }
480487
481488 /*
482489 * Check if the bio or request is one that needs special treatment in the
483490 * flush state machine.
484491 */
485−static inline bool op_is_flush(unsigned int op)
492+static inline bool op_is_flush(blk_opf_t op)
486493 {
487494 return op & (REQ_FUA | REQ_PREFLUSH);
488495 }
@@ -492,13 +499,13 @@ static inline bool op_is_flush(unsigned int op) · 3 unmodified lines
492499 * PREFLUSH flag. Other operations may be marked as synchronous using the
493500 * REQ_SYNC flag.
494501 */
495−static inline bool op_is_sync(unsigned int op)
502+static inline bool op_is_sync(blk_opf_t op)
496503 {
497504 return (op & REQ_OP_MASK) == REQ_OP_READ ||
498505 (op & (REQ_SYNC | REQ_FUA | REQ_PREFLUSH));
499506 }
500507
501−static inline bool op_is_discard(unsigned int op)
508+static inline bool op_is_discard(blk_opf_t op)
502509 {
503510 return (op & REQ_OP_MASK) == REQ_OP_DISCARD;
504511 }
@@ -509,7 +516,7 @@ static inline bool op_is_discard(unsigned int op) · 4 unmodified lines
509516 * due to its different handling in the block layer and device response in
510517 * case of command failure.
511518 */
512−static inline bool op_is_zone_mgmt(enum req_opf op)
519+static inline bool op_is_zone_mgmt(enum req_op op)
513520 {
514521 switch (op & REQ_OP_MASK) {
515522 case REQ_OP_ZONE_RESET:
@@ -522,7 +529,7 @@ static inline bool op_is_zone_mgmt(enum req_opf op) · 6 unmodified lines
522529 }
523530 }
524531
525−static inline int op_stat_group(unsigned int op)
532+static inline int op_stat_group(enum req_op op)
526533 {
527534 if (op_is_discard(op))
528535 return STAT_DISCARD;
include/linux/blkdev.h+89−135
@@ -148,6 +148,7 @@ struct gendisk { · 147 unmodified lines
148148 #define GD_NATIVE_CAPACITY 3
149149 #define GD_ADDED 4
150150 #define GD_SUPPRESS_PART_SCAN 5
151+#define GD_OWNS_QUEUE 6
151152
152153 struct mutex open_mutex; /* open/close mutex */
153154 unsigned open_partitions; /* number of open partitions */
@@ -163,6 +164,29 @@ struct gendisk { · 9 unmodified lines
163164 #ifdef CONFIG_BLK_DEV_INTEGRITY
164165 struct kobject integrity_kobj;
165166 #endif /* CONFIG_BLK_DEV_INTEGRITY */
167+
168+#ifdef CONFIG_BLK_DEV_ZONED
169+ /*
170+ * Zoned block device information for request dispatch control.
171+ * nr_zones is the total number of zones of the device. This is always
172+ * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones
173+ * bits which indicates if a zone is conventional (bit set) or
174+ * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones
175+ * bits which indicates if a zone is write locked, that is, if a write
176+ * request targeting the zone was dispatched.
177+ *
178+ * Reads of this information must be protected with blk_queue_enter() /
179+ * blk_queue_exit(). Modifying this information is only allowed while
180+ * no requests are being processed. See also blk_mq_freeze_queue() and
181+ * blk_mq_unfreeze_queue().
182+ */
183+ unsigned int nr_zones;
184+ unsigned int max_open_zones;
185+ unsigned int max_active_zones;
186+ unsigned long *conv_zones_bitmap;
187+ unsigned long *seq_zones_wlock;
188+#endif /* CONFIG_BLK_DEV_ZONED */
189+
166190 #if IS_ENABLED(CONFIG_CDROM)
167191 struct cdrom_device_info *cdi;
168192 #endif
@@ -170,6 +194,12 @@ struct gendisk { · 1 unmodified line
170194 struct badblocks *bb;
171195 struct lockdep_map lockdep_map;
172196 u64 diskseq;
197+
198+ /*
199+ * Independent sector access ranges. This is always NULL for
200+ * devices that do not have multiple independent access ranges.
201+ */
202+ struct blk_independent_access_ranges *ia_ranges;
173203 };
174204
175205 static inline bool disk_live(struct gendisk *disk)
@@ -220,7 +250,7 @@ static inline int blk_validate_block_size(unsigned long bsize) · 44 unmodified lines
220250 return 0;
221251 }
222252
223−static inline bool blk_op_is_passthrough(unsigned int op)
253+static inline bool blk_op_is_passthrough(blk_opf_t op)
224254 {
225255 op &= REQ_OP_MASK;
226256 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
@@ -284,15 +314,15 @@ struct queue_limits { · 57 unmodified lines
284314 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
285315 void *data);
286316
287−void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
317+void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
288318
289319 #ifdef CONFIG_BLK_DEV_ZONED
290320
291321 #define BLK_ALL_ZONES ((unsigned int)-1)
292322 int blkdev_report_zones(struct block_device *bdev, sector_t sector,
293323 unsigned int nr_zones, report_zones_cb cb, void *data);
294−unsigned int blkdev_nr_zones(struct gendisk *disk);
295−extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
324+unsigned int bdev_nr_zones(struct block_device *bdev);
325+extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
296326 sector_t sectors, sector_t nr_sectors,
297327 gfp_t gfp_mask);
298328 int blk_revalidate_disk_zones(struct gendisk *disk,
@@ -305,7 +335,7 @@ extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, · 6 unmodified lines
305335
306336 #else /* CONFIG_BLK_DEV_ZONED */
307337
308−static inline unsigned int blkdev_nr_zones(struct gendisk *disk)
338+static inline unsigned int bdev_nr_zones(struct block_device *bdev)
309339 {
310340 return 0;
311341 }
@@ -424,6 +454,11 @@ struct request_queue { · 112 unmodified lines
424454 unsigned long nr_requests; /* Max # of requests */
425455
426456 unsigned int dma_pad_mask;
457+ /*
458+ * Drivers that set dma_alignment to less than 511 must be prepared to
459+ * handle individual bvec's that are not a multiple of a SECTOR_SIZE
460+ * due to possible offsets.
461+ */
427462 unsigned int dma_alignment;
428463
429464 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
@@ -455,31 +490,6 @@ struct request_queue { · 25 unmodified lines
455490
456491 unsigned int required_elevator_features;
457492
458−#ifdef CONFIG_BLK_DEV_ZONED
459− /*
460− * Zoned block device information for request dispatch control.
461− * nr_zones is the total number of zones of the device. This is always
462− * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones
463− * bits which indicates if a zone is conventional (bit set) or
464− * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones
465− * bits which indicates if a zone is write locked, that is, if a write
466− * request targeting the zone was dispatched. All three fields are
467− * initialized by the low level device driver (e.g. scsi/sd.c).
468− * Stacking drivers (device mappers) may or may not initialize
469− * these fields.
470− *
471− * Reads of this information must be protected with blk_queue_enter() /
472− * blk_queue_exit(). Modifying this information is only allowed while
473− * no requests are being processed. See also blk_mq_freeze_queue() and
474− * blk_mq_unfreeze_queue().
475− */
476− unsigned int nr_zones;
477− unsigned long *conv_zones_bitmap;
478− unsigned long *seq_zones_wlock;
479− unsigned int max_open_zones;
480− unsigned int max_active_zones;
481−#endif /* CONFIG_BLK_DEV_ZONED */
482−
483493 int node;
484494 #ifdef CONFIG_BLK_DEV_IO_TRACE
485495 struct blk_trace __rcu *blk_trace;
@@ -533,12 +543,6 @@ struct request_queue { · 47 unmodified lines
533543
534544 bool mq_sysfs_init_done;
535545
536− /*
537− * Independent sector access ranges. This is always NULL for
538− * devices that do not have multiple independent access ranges.
539− */
540− struct blk_independent_access_ranges *ia_ranges;
541−
542546 /**
543547 * @srcu: Sleepable RCU. Use as lock when type of the request queue
544548 * is blocking (BLK_MQ_F_BLOCKING). Must be the last member
@@ -559,7 +563,6 @@ struct request_queue { · 14 unmodified lines
559563 #define QUEUE_FLAG_NOXMERGES 9 /* No extended merges */
560564 #define QUEUE_FLAG_ADD_RANDOM 10 /* Contributes to random pool */
561565 #define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */
562−#define QUEUE_FLAG_DEAD 13 /* queue tear-down finished */
563566 #define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */
564567 #define QUEUE_FLAG_STABLE_WRITES 15 /* don't modify blks until WB is done */
565568 #define QUEUE_FLAG_POLL 16 /* IO polling enabled if set */
@@ -587,7 +590,6 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q); · 21 unmodified lines
587590 #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
588591 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
589592 #define blk_queue_has_srcu(q) test_bit(QUEUE_FLAG_HAS_SRCU, &(q)->queue_flags)
590−#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
591593 #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
592594 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
593595 #define blk_queue_noxmerges(q) \
@@ -663,76 +665,69 @@ static inline bool blk_queue_is_zoned(struct request_queue *q) · 69 unmodified lines
663665 }
664666 }
665667
666−static inline sector_t blk_queue_zone_sectors(struct request_queue *q)
667−{
668− return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
669−}
670−
671668 #ifdef CONFIG_BLK_DEV_ZONED
672−static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
669+static inline unsigned int disk_nr_zones(struct gendisk *disk)
673670 {
674− return blk_queue_is_zoned(q) ? q->nr_zones : 0;
671+ return blk_queue_is_zoned(disk->queue) ? disk->nr_zones : 0;
675672 }
676673
677−static inline unsigned int blk_queue_zone_no(struct request_queue *q,
678− sector_t sector)
674+static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
679675 {
680− if (!blk_queue_is_zoned(q))
676+ if (!blk_queue_is_zoned(disk->queue))
681677 return 0;
682− return sector >> ilog2(q->limits.chunk_sectors);
678+ return sector >> ilog2(disk->queue->limits.chunk_sectors);
683679 }
684680
685−static inline bool blk_queue_zone_is_seq(struct request_queue *q,
686− sector_t sector)
681+static inline bool disk_zone_is_seq(struct gendisk *disk, sector_t sector)
687682 {
688− if (!blk_queue_is_zoned(q))
683+ if (!blk_queue_is_zoned(disk->queue))
689684 return false;
690− if (!q->conv_zones_bitmap)
685+ if (!disk->conv_zones_bitmap)
691686 return true;
692− return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap);
687+ return !test_bit(disk_zone_no(disk, sector), disk->conv_zones_bitmap);
693688 }
694689
695−static inline void blk_queue_max_open_zones(struct request_queue *q,
690+static inline void disk_set_max_open_zones(struct gendisk *disk,
696691 unsigned int max_open_zones)
697692 {
698− q->max_open_zones = max_open_zones;
693+ disk->max_open_zones = max_open_zones;
699694 }
700695
701−static inline unsigned int queue_max_open_zones(const struct request_queue *q)
696+static inline void disk_set_max_active_zones(struct gendisk *disk,
697+ unsigned int max_active_zones)
702698 {
703− return q->max_open_zones;
699+ disk->max_active_zones = max_active_zones;
704700 }
705701
706−static inline void blk_queue_max_active_zones(struct request_queue *q,
707− unsigned int max_active_zones)
702+static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
708703 {
709− q->max_active_zones = max_active_zones;
704+ return bdev->bd_disk->max_open_zones;
710705 }
711706
712−static inline unsigned int queue_max_active_zones(const struct request_queue *q)
707+static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
713708 {
714− return q->max_active_zones;
709+ return bdev->bd_disk->max_active_zones;
715710 }
711+
716712 #else /* CONFIG_BLK_DEV_ZONED */
717−static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
713+static inline unsigned int disk_nr_zones(struct gendisk *disk)
718714 {
719715 return 0;
720716 }
721−static inline bool blk_queue_zone_is_seq(struct request_queue *q,
722− sector_t sector)
717+static inline bool disk_zone_is_seq(struct gendisk *disk, sector_t sector)
723718 {
724719 return false;
725720 }
726−static inline unsigned int blk_queue_zone_no(struct request_queue *q,
727− sector_t sector)
721+static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
728722 {
729723 return 0;
730724 }
731−static inline unsigned int queue_max_open_zones(const struct request_queue *q)
725+static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
732726 {
733727 return 0;
734728 }
735−static inline unsigned int queue_max_active_zones(const struct request_queue *q)
729+
730+static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
736731 {
737732 return 0;
738733 }
@@ -812,8 +807,6 @@ static inline u64 sb_bdev_nr_blocks(struct super_block *sb) · 73 unmodified lines
812807
813808 int bdev_disk_changed(struct gendisk *disk, bool invalidate);
814809
815−struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
816− struct lock_class_key *lkclass);
817810 void put_disk(struct gendisk *disk);
818811 struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass);
819812
@@ -832,7 +825,6 @@ struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass); · 12 unmodified lines
832825 \
833826 __blk_alloc_disk(node_id, &__key); \
834827 })
835−void blk_cleanup_disk(struct gendisk *disk);
836828
837829 int __register_blkdev(unsigned int major, const char *name,
838830 void (*probe)(dev_t devt));
@@ -880,7 +872,7 @@ extern void blk_queue_exit(struct request_queue *q); · 41 unmodified lines
880872 extern void blk_sync_queue(struct request_queue *q);
881873
882874 /* Helper to convert REQ_OP_XXX to its string format XXX */
883−extern const char *blk_op_str(unsigned int op);
875+extern const char *blk_op_str(enum req_op op);
884876
885877 int blk_status_to_errno(blk_status_t status);
886878 blk_status_t errno_to_blk_status(int errno);
@@ -898,64 +890,33 @@ static inline struct request_queue *bdev_get_queue(struct block_device *bdev) · 11 unmodified lines
898890 return bdev->bd_queue; /* this is never NULL */
899891 }
900892
901−#ifdef CONFIG_BLK_DEV_ZONED
902−
903893 /* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */
904894 const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
905895
906896 static inline unsigned int bio_zone_no(struct bio *bio)
907897 {
908− return blk_queue_zone_no(bdev_get_queue(bio->bi_bdev),
909− bio->bi_iter.bi_sector);
898+ return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
910899 }
911900
912901 static inline unsigned int bio_zone_is_seq(struct bio *bio)
913902 {
914− return blk_queue_zone_is_seq(bdev_get_queue(bio->bi_bdev),
915− bio->bi_iter.bi_sector);
916−}
917−#endif /* CONFIG_BLK_DEV_ZONED */
918−
919−static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
920− int op)
921−{
922− if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
923− return min(q->limits.max_discard_sectors,
924− UINT_MAX >> SECTOR_SHIFT);
925−
926− if (unlikely(op == REQ_OP_WRITE_ZEROES))
927− return q->limits.max_write_zeroes_sectors;
928−
929− return q->limits.max_sectors;
903+ return disk_zone_is_seq(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
930904 }
931905
932906 /*
933− * Return maximum size of a request at given offset. Only valid for
934− * file system requests.
907+ * Return how much of the chunk is left to be used for I/O at a given offset.
935908 */
936−static inline unsigned int blk_max_size_offset(struct request_queue *q,
937− sector_t offset,
938− unsigned int chunk_sectors)
939−{
940− if (!chunk_sectors) {
941− if (q->limits.chunk_sectors)
942− chunk_sectors = q->limits.chunk_sectors;
943− else
944− return q->limits.max_sectors;
945− }
946−
947− if (likely(is_power_of_2(chunk_sectors)))
948− chunk_sectors -= offset & (chunk_sectors - 1);
949− else
950− chunk_sectors -= sector_div(offset, chunk_sectors);
951−
952− return min(q->limits.max_sectors, chunk_sectors);
909+static inline unsigned int blk_chunk_sectors_left(sector_t offset,
910+ unsigned int chunk_sectors)
911+{
912+ if (unlikely(!is_power_of_2(chunk_sectors)))
913+ return chunk_sectors - sector_div(offset, chunk_sectors);
914+ return chunk_sectors - (offset & (chunk_sectors - 1));
953915 }
954916
955917 /*
956918 * Access functions for manipulating queue properties
957919 */
958−extern void blk_cleanup_queue(struct request_queue *);
959920 void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce limit);
960921 extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
961922 extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
@@ -1337,32 +1298,26 @@ static inline sector_t bdev_zone_sectors(struct block_device *bdev) · 375 unmodified lines
13371298 {
13381299 struct request_queue *q = bdev_get_queue(bdev);
13391300
1340− if (q)
1341− return blk_queue_zone_sectors(q);
1342− return 0;
1301+ if (!blk_queue_is_zoned(q))
1302+ return 0;
1303+ return q->limits.chunk_sectors;
13431304 }
13441305
1345−static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
1306+static inline int queue_dma_alignment(const struct request_queue *q)
13461307 {
1347− struct request_queue *q = bdev_get_queue(bdev);
1348−
1349− if (q)
1350− return queue_max_open_zones(q);
1351− return 0;
1308+ return q ? q->dma_alignment : 511;
13521309 }
13531310
1354−static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
1311+static inline unsigned int bdev_dma_alignment(struct block_device *bdev)
13551312 {
1356− struct request_queue *q = bdev_get_queue(bdev);
1357−
1358− if (q)
1359− return queue_max_active_zones(q);
1360− return 0;
1313+ return queue_dma_alignment(bdev_get_queue(bdev));
13611314 }
13621315
1363−static inline int queue_dma_alignment(const struct request_queue *q)
1316+static inline bool bdev_iter_is_aligned(struct block_device *bdev,
1317+ struct iov_iter *iter)
13641318 {
1365− return q ? q->dma_alignment : 511;
1319+ return iov_iter_is_aligned(iter, bdev_dma_alignment(bdev),
1320+ bdev_logical_block_size(bdev) - 1);
13661321 }
13671322
13681323 static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
@@ -1426,7 +1381,7 @@ struct block_device_operations { · 57 unmodified lines
14261381 unsigned int flags);
14271382 int (*open) (struct block_device *, fmode_t);
14281383 void (*release) (struct gendisk *, fmode_t);
1429− int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int);
1384+ int (*rw_page)(struct block_device *, sector_t, struct page *, enum req_op);
14301385 int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
14311386 int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
14321387 unsigned int (*check_events) (struct gendisk *disk,
@@ -1479,9 +1434,9 @@ static inline void blk_wake_io_task(struct task_struct *waiter) · 46 unmodified lines
14791434 }
14801435
14811436 unsigned long bdev_start_io_acct(struct block_device *bdev,
1482− unsigned int sectors, unsigned int op,
1437+ unsigned int sectors, enum req_op op,
14831438 unsigned long start_time);
1484−void bdev_end_io_acct(struct block_device *bdev, unsigned int op,
1439+void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
14851440 unsigned long start_time);
14861441
14871442 void bio_start_io_acct_time(struct bio *bio, unsigned long start_time);
@@ -1502,7 +1457,6 @@ static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) · 14 unmodified lines
15021457 int bdev_read_only(struct block_device *bdev);
15031458 int set_blocksize(struct block_device *bdev, int size);
15041459
1505−const char *bdevname(struct block_device *bdev, char *buffer);
15061460 int lookup_bdev(const char *pathname, dev_t *dev);
15071461
15081462 void blkdev_show(struct seq_file *seqf, off_t offset);
include/linux/blktrace_api.h+2−11
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include <linux/compat.h>
88 #include <uapi/linux/blktrace_api.h>
99 #include <linux/list.h>
10+#include <linux/blk_types.h>
1011
1112 #if defined(CONFIG_BLK_DEV_IO_TRACE)
1213
@@ -77,10 +78,6 @@ extern int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, · 64 unmodified lines
7778 char __user *arg);
7879 extern int blk_trace_startstop(struct request_queue *q, int start);
7980 extern int blk_trace_remove(struct request_queue *q);
80−extern void blk_trace_remove_sysfs(struct device *dev);
81−extern int blk_trace_init_sysfs(struct device *dev);
82−
83−extern struct attribute_group blk_trace_attr_group;
8481
8582 #else /* !CONFIG_BLK_DEV_IO_TRACE */
8683 # define blk_trace_ioctl(bdev, cmd, arg) (-ENOTTY)
@@ -91,13 +88,7 @@ extern struct attribute_group blk_trace_attr_group; · 4 unmodified lines
9188 # define blk_trace_remove(q) (-ENOTTY)
9289 # define blk_add_trace_msg(q, fmt, ...) do { } while (0)
9390 # define blk_add_cgroup_trace_msg(q, cg, fmt, ...) do { } while (0)
94−# define blk_trace_remove_sysfs(dev) do { } while (0)
9591 # define blk_trace_note_message_enabled(q) (false)
96−static inline int blk_trace_init_sysfs(struct device *dev)
97−{
98− return 0;
99−}
100−
10192 #endif /* CONFIG_BLK_DEV_IO_TRACE */
10293
10394 #ifdef CONFIG_COMPAT
@@ -115,7 +106,7 @@ struct compat_blk_user_trace_setup { · 11 unmodified lines
115106
116107 #endif
117108
118−void blk_fill_rwbs(char *rwbs, unsigned int op);
109+void blk_fill_rwbs(char *rwbs, blk_opf_t opf);
119110
120111 static inline sector_t blk_rq_trace_sector(struct request *rq)
121112 {
include/linux/buffer_head.h+5−4
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #define _LINUX_BUFFER_HEAD_H
1010
1111 #include <linux/types.h>
12+#include <linux/blk_types.h>
1213 #include <linux/fs.h>
1314 #include <linux/linkage.h>
1415 #include <linux/pagemap.h>
@@ -201,11 +202,11 @@ struct buffer_head *alloc_buffer_head(gfp_t gfp_flags); · 186 unmodified lines
201202 void free_buffer_head(struct buffer_head * bh);
202203 void unlock_buffer(struct buffer_head *bh);
203204 void __lock_buffer(struct buffer_head *bh);
204−void ll_rw_block(int, int, int, struct buffer_head * bh[]);
205+void ll_rw_block(blk_opf_t, int, struct buffer_head * bh[]);
205206 int sync_dirty_buffer(struct buffer_head *bh);
206−int __sync_dirty_buffer(struct buffer_head *bh, int op_flags);
207−void write_dirty_buffer(struct buffer_head *bh, int op_flags);
208−int submit_bh(int, int, struct buffer_head *);
207+int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
208+void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
209+int submit_bh(blk_opf_t, struct buffer_head *);
209210 void write_boundary_block(struct block_device *bdev,
210211 sector_t bblock, unsigned blocksize);
211212 int bh_uptodate_or_lock(struct buffer_head *bh);
include/linux/dm-io.h+2−2
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 #ifdef __KERNEL__
1414
1515 #include <linux/types.h>
16+#include <linux/blk_types.h>
1617
1718 struct dm_io_region {
1819 struct block_device *bdev;
@@ -57,8 +58,7 @@ struct dm_io_notify { · 38 unmodified lines
5758 */
5859 struct dm_io_client;
5960 struct dm_io_request {
60− int bi_op; /* REQ_OP */
61− int bi_op_flags; /* req_flag_bits */
61+ blk_opf_t bi_opf; /* Request type and flags */
6262 struct dm_io_memory mem; /* Memory to use for io */
6363 struct dm_io_notify notify; /* Synchronous if notify.fn is NULL */
6464 struct dm_io_client *client; /* Client memory handler */
include/linux/ioprio.h+10−14
@@ -11,7 +11,7 @@ · 10 unmodified lines
1111 /*
1212 * Default IO priority.
1313 */
14−#define IOPRIO_DEFAULT IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_BE_NORM)
14+#define IOPRIO_DEFAULT IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0)
1515
1616 /*
1717 * Check that a priority value has a valid class.
@@ -46,23 +46,19 @@ static inline int task_nice_ioclass(struct task_struct *task) · 28 unmodified lines
4646 return IOPRIO_CLASS_BE;
4747 }
4848
49−/*
50− * If the calling process has set an I/O priority, use that. Otherwise, return
51− * the default I/O priority.
52− */
53−static inline int get_current_ioprio(void)
49+#ifdef CONFIG_BLOCK
50+int __get_task_ioprio(struct task_struct *p);
51+#else
52+static inline int __get_task_ioprio(struct task_struct *p)
5453 {
55− struct io_context *ioc = current->io_context;
56−
57− if (ioc)
58− return ioc->ioprio;
5954 return IOPRIO_DEFAULT;
6055 }
56+#endif /* CONFIG_BLOCK */
6157
62−/*
63− * For inheritance, return the highest of the two given priorities
64− */
65−extern int ioprio_best(unsigned short aprio, unsigned short bprio);
58+static inline int get_current_ioprio(void)
59+{
60+ return __get_task_ioprio(current);
61+}
6662
6763 extern int set_task_ioprio(struct task_struct *task, int ioprio);
6864
include/linux/jbd2.h+1−1
@@ -1557,7 +1557,7 @@ extern int jbd2_journal_wipe (journal_t *, int); · 1556 unmodified lines
15571557 extern int jbd2_journal_skip_recovery (journal_t *);
15581558 extern void jbd2_journal_update_sb_errno(journal_t *);
15591559 extern int jbd2_journal_update_sb_log_tail (journal_t *, tid_t,
1560− unsigned long, int);
1560+ unsigned long, blk_opf_t);
15611561 extern void jbd2_journal_abort (journal_t *, int);
15621562 extern int jbd2_journal_errno (journal_t *);
15631563 extern void jbd2_journal_ack_err (journal_t *);
include/linux/uio.h+2−0
@@ -219,6 +219,8 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i); · 218 unmodified lines
219219 #endif
220220
221221 size_t iov_iter_zero(size_t bytes, struct iov_iter *);
222+bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask,
223+ unsigned len_mask);
222224 unsigned long iov_iter_alignment(const struct iov_iter *i);
223225 unsigned long iov_iter_gap_alignment(const struct iov_iter *i);
224226 void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov,
include/linux/writeback.h+2−2
@@ -101,9 +101,9 @@ struct writeback_control { · 100 unmodified lines
101101 #endif
102102 };
103103
104−static inline int wbc_to_write_flags(struct writeback_control *wbc)
104+static inline blk_opf_t wbc_to_write_flags(struct writeback_control *wbc)
105105 {
106− int flags = 0;
106+ blk_opf_t flags = 0;
107107
108108 if (wbc->punt_to_cgroup)
109109 flags = REQ_CGROUP_PUNT;
include/scsi/scsi_cmnd.h+2−2
@@ -386,7 +386,7 @@ static inline unsigned scsi_transfer_length(struct scsi_cmnd *scmd) · 385 unmodified lines
386386 extern void scsi_build_sense(struct scsi_cmnd *scmd, int desc,
387387 u8 key, u8 asc, u8 ascq);
388388
389−struct request *scsi_alloc_request(struct request_queue *q,
390− unsigned int op, blk_mq_req_flags_t flags);
389+struct request *scsi_alloc_request(struct request_queue *q, blk_opf_t opf,
390+ blk_mq_req_flags_t flags);
391391
392392 #endif /* _SCSI_SCSI_CMND_H */
include/scsi/scsi_device.h+1−1
@@ -457,7 +457,7 @@ extern void scsi_sanitize_inquiry_string(unsigned char *s, int len); · 456 unmodified lines
457457 extern int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
458458 int data_direction, void *buffer, unsigned bufflen,
459459 unsigned char *sense, struct scsi_sense_hdr *sshdr,
460− int timeout, int retries, u64 flags,
460+ int timeout, int retries, blk_opf_t flags,
461461 req_flags_t rq_flags, int *resid);
462462 /* Make sure any sense buffer is the correct size. */
463463 #define scsi_execute(sdev, cmd, data_direction, buffer, bufflen, sense, \
include/scsi/scsi_host.h+1−1
@@ -786,7 +786,7 @@ extern int scsi_host_block(struct Scsi_Host *shost); · 785 unmodified lines
786786 extern int scsi_host_unblock(struct Scsi_Host *shost, int new_state);
787787
788788 void scsi_host_busy_iter(struct Scsi_Host *,
789− bool (*fn)(struct scsi_cmnd *, void *, bool), void *priv);
789+ bool (*fn)(struct scsi_cmnd *, void *), void *priv);
790790
791791 struct class_container;
792792
include/trace/events/f2fs.h+11−11
@@ -66,7 +66,7 @@ TRACE_DEFINE_ENUM(CP_RESIZE); · 65 unmodified lines
6666
6767 #define F2FS_OP_FLAGS (REQ_RAHEAD | REQ_SYNC | REQ_META | REQ_PRIO | \
6868 REQ_PREFLUSH | REQ_FUA)
69−#define F2FS_BIO_FLAG_MASK(t) (t & F2FS_OP_FLAGS)
69+#define F2FS_BIO_FLAG_MASK(t) (__force u32)((t) & F2FS_OP_FLAGS)
7070
7171 #define show_bio_type(op,op_flags) show_bio_op(op), \
7272 show_bio_op_flags(op_flags)
@@ -75,12 +75,12 @@ TRACE_DEFINE_ENUM(CP_RESIZE); · 2 unmodified lines
7575
7676 #define show_bio_op_flags(flags) \
7777 __print_flags(F2FS_BIO_FLAG_MASK(flags), "|", \
78− { REQ_RAHEAD, "R" }, \
79− { REQ_SYNC, "S" }, \
80− { REQ_META, "M" }, \
81− { REQ_PRIO, "P" }, \
82− { REQ_PREFLUSH, "PF" }, \
83− { REQ_FUA, "FUA" })
78+ { (__force u32)REQ_RAHEAD, "R" }, \
79+ { (__force u32)REQ_SYNC, "S" }, \
80+ { (__force u32)REQ_META, "M" }, \
81+ { (__force u32)REQ_PRIO, "P" }, \
82+ { (__force u32)REQ_PREFLUSH, "PF" }, \
83+ { (__force u32)REQ_FUA, "FUA" })
8484
8585 #define show_data_type(type) \
8686 __print_symbolic(type, \
@@ -1036,8 +1036,8 @@ DECLARE_EVENT_CLASS(f2fs__submit_page_bio, · 949 unmodified lines
10361036 __field(pgoff_t, index)
10371037 __field(block_t, old_blkaddr)
10381038 __field(block_t, new_blkaddr)
1039− __field(int, op)
1040− __field(int, op_flags)
1039+ __field(enum req_op, op)
1040+ __field(blk_opf_t, op_flags)
10411041 __field(int, temp)
10421042 __field(int, type)
10431043 ),
@@ -1092,8 +1092,8 @@ DECLARE_EVENT_CLASS(f2fs__bio, · 48 unmodified lines
10921092 TP_STRUCT__entry(
10931093 __field(dev_t, dev)
10941094 __field(dev_t, target)
1095− __field(int, op)
1096− __field(int, op_flags)
1095+ __field(enum req_op, op)
1096+ __field(blk_opf_t, op_flags)
10971097 __field(int, type)
10981098 __field(sector_t, sector)
10991099 __field(unsigned int, size)
include/trace/events/jbd2.h+6−6
@@ -355,22 +355,22 @@ TRACE_EVENT(jbd2_update_log_tail, · 354 unmodified lines
355355
356356 TRACE_EVENT(jbd2_write_superblock,
357357
358− TP_PROTO(journal_t *journal, int write_op),
358+ TP_PROTO(journal_t *journal, blk_opf_t write_flags),
359359
360− TP_ARGS(journal, write_op),
360+ TP_ARGS(journal, write_flags),
361361
362362 TP_STRUCT__entry(
363363 __field( dev_t, dev )
364− __field( int, write_op )
364+ __field( blk_opf_t, write_flags )
365365 ),
366366
367367 TP_fast_assign(
368368 __entry->dev = journal->j_fs_dev->bd_dev;
369− __entry->write_op = write_op;
369+ __entry->write_flags = write_flags;
370370 ),
371371
372− TP_printk("dev %d,%d write_op %x", MAJOR(__entry->dev),
373− MINOR(__entry->dev), __entry->write_op)
372+ TP_printk("dev %d,%d write_flags %x", MAJOR(__entry->dev),
373+ MINOR(__entry->dev), (__force u32)__entry->write_flags)
374374 );
375375
376376 TRACE_EVENT(jbd2_lock_buffer_stall,
include/trace/events/nilfs2.h+2−2
@@ -192,7 +192,7 @@ TRACE_EVENT(nilfs2_mdt_submit_block, · 191 unmodified lines
192192 TP_PROTO(struct inode *inode,
193193 unsigned long ino,
194194 unsigned long blkoff,
195− int mode),
195+ enum req_op mode),
196196
197197 TP_ARGS(inode, ino, blkoff, mode),
198198
@@ -200,7 +200,7 @@ TRACE_EVENT(nilfs2_mdt_submit_block, · 1 unmodified line
200200 __field(struct inode *, inode)
201201 __field(unsigned long, ino)
202202 __field(unsigned long, blkoff)
203− __field(int, mode)
203+ __field(enum req_op, mode)
204204 ),
205205
206206 TP_fast_assign(
include/uapi/linux/ublk_cmd.h+161−0
@@ -0,0 +1,161 @@
1+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2+#ifndef USER_BLK_DRV_CMD_INC_H
3+#define USER_BLK_DRV_CMD_INC_H
4+
5+#include <linux/types.h>
6+
7+/* ublk server command definition */
8+
9+/*
10+ * Admin commands, issued by ublk server, and handled by ublk driver.
11+ */
12+#define UBLK_CMD_GET_QUEUE_AFFINITY 0x01
13+#define UBLK_CMD_GET_DEV_INFO 0x02
14+#define UBLK_CMD_ADD_DEV 0x04
15+#define UBLK_CMD_DEL_DEV 0x05
16+#define UBLK_CMD_START_DEV 0x06
17+#define UBLK_CMD_STOP_DEV 0x07
18+
19+/*
20+ * IO commands, issued by ublk server, and handled by ublk driver.
21+ *
22+ * FETCH_REQ: issued via sqe(URING_CMD) beforehand for fetching IO request
23+ * from ublk driver, should be issued only when starting device. After
24+ * the associated cqe is returned, request's tag can be retrieved via
25+ * cqe->userdata.
26+ *
27+ * COMMIT_AND_FETCH_REQ: issued via sqe(URING_CMD) after ublkserver handled
28+ * this IO request, request's handling result is committed to ublk
29+ * driver, meantime FETCH_REQ is piggyback, and FETCH_REQ has to be
30+ * handled before completing io request.
31+ */
32+#define UBLK_IO_FETCH_REQ 0x20
33+#define UBLK_IO_COMMIT_AND_FETCH_REQ 0x21
34+
35+/* only ABORT means that no re-fetch */
36+#define UBLK_IO_RES_OK 0
37+#define UBLK_IO_RES_ABORT (-ENODEV)
38+
39+#define UBLKSRV_CMD_BUF_OFFSET 0
40+#define UBLKSRV_IO_BUF_OFFSET 0x80000000
41+
42+/* tag bit is 12bit, so at most 4096 IOs for each queue */
43+#define UBLK_MAX_QUEUE_DEPTH 4096
44+
45+/*
46+ * zero copy requires 4k block size, and can remap ublk driver's io
47+ * request into ublksrv's vm space
48+ */
49+#define UBLK_F_SUPPORT_ZERO_COPY (1ULL << 0)
50+
51+/*
52+ * Force to complete io cmd via io_uring_cmd_complete_in_task so that
53+ * performance comparison is done easily with using task_work_add
54+ */
55+#define UBLK_F_URING_CMD_COMP_IN_TASK (1ULL << 1)
56+
57+/* device state */
58+#define UBLK_S_DEV_DEAD 0
59+#define UBLK_S_DEV_LIVE 1
60+
61+/* shipped via sqe->cmd of io_uring command */
62+struct ublksrv_ctrl_cmd {
63+ /* sent to which device, must be valid */
64+ __u32 dev_id;
65+
66+ /* sent to which queue, must be -1 if the cmd isn't for queue */
67+ __u16 queue_id;
68+ /*
69+ * cmd specific buffer, can be IN or OUT.
70+ */
71+ __u16 len;
72+ __u64 addr;
73+
74+ /* inline data */
75+ __u64 data[2];
76+};
77+
78+struct ublksrv_ctrl_dev_info {
79+ __u16 nr_hw_queues;
80+ __u16 queue_depth;
81+ __u16 block_size;
82+ __u16 state;
83+
84+ __u32 rq_max_blocks;
85+ __u32 dev_id;
86+
87+ __u64 dev_blocks;
88+
89+ __s32 ublksrv_pid;
90+ __s32 reserved0;
91+ __u64 flags;
92+ __u64 flags_reserved;
93+
94+ /* For ublksrv internal use, invisible to ublk driver */
95+ __u64 ublksrv_flags;
96+ __u64 reserved1[9];
97+};
98+
99+#define UBLK_IO_OP_READ 0
100+#define UBLK_IO_OP_WRITE 1
101+#define UBLK_IO_OP_FLUSH 2
102+#define UBLK_IO_OP_DISCARD 3
103+#define UBLK_IO_OP_WRITE_SAME 4
104+#define UBLK_IO_OP_WRITE_ZEROES 5
105+
106+#define UBLK_IO_F_FAILFAST_DEV (1U << 8)
107+#define UBLK_IO_F_FAILFAST_TRANSPORT (1U << 9)
108+#define UBLK_IO_F_FAILFAST_DRIVER (1U << 10)
109+#define UBLK_IO_F_META (1U << 11)
110+#define UBLK_IO_F_FUA (1U << 13)
111+#define UBLK_IO_F_NOUNMAP (1U << 15)
112+#define UBLK_IO_F_SWAP (1U << 16)
113+
114+/*
115+ * io cmd is described by this structure, and stored in share memory, indexed
116+ * by request tag.
117+ *
118+ * The data is stored by ublk driver, and read by ublksrv after one fetch command
119+ * returns.
120+ */
121+struct ublksrv_io_desc {
122+ /* op: bit 0-7, flags: bit 8-31 */
123+ __u32 op_flags;
124+
125+ __u32 nr_sectors;
126+
127+ /* start sector for this io */
128+ __u64 start_sector;
129+
130+ /* buffer address in ublksrv daemon vm space, from ublk driver */
131+ __u64 addr;
132+};
133+
134+static inline __u8 ublksrv_get_op(const struct ublksrv_io_desc *iod)
135+{
136+ return iod->op_flags & 0xff;
137+}
138+
139+static inline __u32 ublksrv_get_flags(const struct ublksrv_io_desc *iod)
140+{
141+ return iod->op_flags >> 8;
142+}
143+
144+/* issued to ublk driver via /dev/ublkcN */
145+struct ublksrv_io_cmd {
146+ __u16 q_id;
147+
148+ /* for fetch/commit which result */
149+ __u16 tag;
150+
151+ /* io result, it is valid for COMMIT* command only */
152+ __s32 result;
153+
154+ /*
155+ * userspace buffer address in ublksrv daemon process, valid for
156+ * FETCH* command only
157+ */
158+ __u64 addr;
159+};
160+
161+#endif
kernel/power/swap.c+13−16
@@ -269,15 +269,14 @@ static void hib_end_io(struct bio *bio) · 268 unmodified lines
269269 bio_put(bio);
270270 }
271271
272−static int hib_submit_io(int op, int op_flags, pgoff_t page_off, void *addr,
273− struct hib_bio_batch *hb)
272+static int hib_submit_io(blk_opf_t opf, pgoff_t page_off, void *addr,
273+ struct hib_bio_batch *hb)
274274 {
275275 struct page *page = virt_to_page(addr);
276276 struct bio *bio;
277277 int error = 0;
278278
279− bio = bio_alloc(hib_resume_bdev, 1, op | op_flags,
280− GFP_NOIO | __GFP_HIGH);
279+ bio = bio_alloc(hib_resume_bdev, 1, opf, GFP_NOIO | __GFP_HIGH);
281280 bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9);
282281
283282 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
@@ -317,8 +316,7 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) · 33 unmodified lines
317316 {
318317 int error;
319318
320− hib_submit_io(REQ_OP_READ, 0, swsusp_resume_block,
321− swsusp_header, NULL);
319+ hib_submit_io(REQ_OP_READ, swsusp_resume_block, swsusp_header, NULL);
322320 if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
323321 !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
324322 memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
@@ -331,7 +329,7 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) · 6 unmodified lines
331329 swsusp_header->flags = flags;
332330 if (flags & SF_CRC32_MODE)
333331 swsusp_header->crc32 = handle->crc32;
334− error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
332+ error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC,
335333 swsusp_resume_block, swsusp_header, NULL);
336334 } else {
337335 pr_err("Swap header not found!\n");
@@ -408,7 +406,7 @@ static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb) · 70 unmodified lines
408406 } else {
409407 src = buf;
410408 }
411− return hib_submit_io(REQ_OP_WRITE, REQ_SYNC, offset, src, hb);
409+ return hib_submit_io(REQ_OP_WRITE | REQ_SYNC, offset, src, hb);
412410 }
413411
414412 static void release_swap_writer(struct swap_map_handle *handle)
@@ -1003,7 +1001,7 @@ static int get_swap_reader(struct swap_map_handle *handle, · 588 unmodified lines
10031001 return -ENOMEM;
10041002 }
10051003
1006− error = hib_submit_io(REQ_OP_READ, 0, offset, tmp->map, NULL);
1004+ error = hib_submit_io(REQ_OP_READ, offset, tmp->map, NULL);
10071005 if (error) {
10081006 release_swap_reader(handle);
10091007 return error;
@@ -1027,7 +1025,7 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf, · 17 unmodified lines
10271025 offset = handle->cur->entries[handle->k];
10281026 if (!offset)
10291027 return -EFAULT;
1030− error = hib_submit_io(REQ_OP_READ, 0, offset, buf, hb);
1028+ error = hib_submit_io(REQ_OP_READ, offset, buf, hb);
10311029 if (error)
10321030 return error;
10331031 if (++handle->k >= MAP_PAGE_ENTRIES) {
@@ -1526,8 +1524,7 @@ int swsusp_check(void) · 492 unmodified lines
15261524 if (!IS_ERR(hib_resume_bdev)) {
15271525 set_blocksize(hib_resume_bdev, PAGE_SIZE);
15281526 clear_page(swsusp_header);
1529− error = hib_submit_io(REQ_OP_READ, 0,
1530− swsusp_resume_block,
1527+ error = hib_submit_io(REQ_OP_READ, swsusp_resume_block,
15311528 swsusp_header, NULL);
15321529 if (error)
15331530 goto put;
@@ -1535,7 +1532,7 @@ int swsusp_check(void) · 1 unmodified line
15351532 if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) {
15361533 memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
15371534 /* Reset swap signature now */
1538− error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
1535+ error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC,
15391536 swsusp_resume_block,
15401537 swsusp_header, NULL);
15411538 } else {
@@ -1586,11 +1583,11 @@ int swsusp_unmark(void) · 44 unmodified lines
15861583 {
15871584 int error;
15881585
1589− hib_submit_io(REQ_OP_READ, 0, swsusp_resume_block,
1590− swsusp_header, NULL);
1586+ hib_submit_io(REQ_OP_READ, swsusp_resume_block,
1587+ swsusp_header, NULL);
15911588 if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) {
15921589 memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10);
1593− error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
1590+ error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC,
15941591 swsusp_resume_block,
15951592 swsusp_header, NULL);
15961593 } else {
kernel/trace/blktrace.c+30−42
@@ -205,7 +205,7 @@ static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), · 204 unmodified lines
205205 #define BLK_TC_PREFLUSH BLK_TC_FLUSH
206206
207207 /* The ilog2() calls fall out because they're constant */
208−#define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \
208+#define MASK_TC_BIT(rw, __name) ((__force u32)(rw & REQ_ ## __name) << \
209209 (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name))
210210
211211 /*
@@ -213,8 +213,8 @@ static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), · 1 unmodified line
213213 * blk_io_trace structure and places it in a per-cpu subbuffer.
214214 */
215215 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
216− int op, int op_flags, u32 what, int error, int pdu_len,
217− void *pdu_data, u64 cgid)
216+ const blk_opf_t opf, u32 what, int error,
217+ int pdu_len, void *pdu_data, u64 cgid)
218218 {
219219 struct task_struct *tsk = current;
220220 struct ring_buffer_event *event = NULL;
@@ -227,16 +227,17 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, · 6 unmodified lines
227227 int cpu;
228228 bool blk_tracer = blk_tracer_enabled;
229229 ssize_t cgid_len = cgid ? sizeof(cgid) : 0;
230+ const enum req_op op = opf & REQ_OP_MASK;
230231
231232 if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))
232233 return;
233234
234235 what |= ddir_act[op_is_write(op) ? WRITE : READ];
235− what |= MASK_TC_BIT(op_flags, SYNC);
236− what |= MASK_TC_BIT(op_flags, RAHEAD);
237− what |= MASK_TC_BIT(op_flags, META);
238− what |= MASK_TC_BIT(op_flags, PREFLUSH);
239− what |= MASK_TC_BIT(op_flags, FUA);
236+ what |= MASK_TC_BIT(opf, SYNC);
237+ what |= MASK_TC_BIT(opf, RAHEAD);
238+ what |= MASK_TC_BIT(opf, META);
239+ what |= MASK_TC_BIT(opf, PREFLUSH);
240+ what |= MASK_TC_BIT(opf, FUA);
240241 if (op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)
241242 what |= BLK_TC_ACT(BLK_TC_DISCARD);
242243 if (op == REQ_OP_FLUSH)
@@ -736,12 +737,12 @@ int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) · 493 unmodified lines
736737
737738 switch (cmd) {
738739 case BLKTRACESETUP:
739− bdevname(bdev, b);
740+ snprintf(b, sizeof(b), "%pg", bdev);
740741 ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
741742 break;
742743 #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
743744 case BLKTRACESETUP32:
744− bdevname(bdev, b);
745+ snprintf(b, sizeof(b), "%pg", bdev);
745746 ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
746747 break;
747748 #endif
@@ -842,9 +843,8 @@ static void blk_add_trace_rq(struct request *rq, blk_status_t error, · 94 unmodified lines
842843 else
843844 what |= BLK_TC_ACT(BLK_TC_FS);
844845
845− __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq),
846− rq->cmd_flags, what, blk_status_to_errno(error), 0,
847− NULL, cgid);
846+ __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, rq->cmd_flags,
847+ what, blk_status_to_errno(error), 0, NULL, cgid);
848848 rcu_read_unlock();
849849 }
850850
@@ -903,7 +903,7 @@ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, · 52 unmodified lines
903903 }
904904
905905 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
906− bio_op(bio), bio->bi_opf, what, error, 0, NULL,
906+ bio->bi_opf, what, error, 0, NULL,
907907 blk_trace_bio_get_cgid(q, bio));
908908 rcu_read_unlock();
909909 }
@@ -949,7 +949,7 @@ static void blk_add_trace_plug(void *ignore, struct request_queue *q) · 39 unmodified lines
949949 rcu_read_lock();
950950 bt = rcu_dereference(q->blk_trace);
951951 if (bt)
952− __blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0);
952+ __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0);
953953 rcu_read_unlock();
954954 }
955955
@@ -969,7 +969,7 @@ static void blk_add_trace_unplug(void *ignore, struct request_queue *q, · 13 unmodified lines
969969 else
970970 what = BLK_TA_UNPLUG_TIMER;
971971
972− __blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0);
972+ __blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0);
973973 }
974974 rcu_read_unlock();
975975 }
@@ -985,8 +985,7 @@ static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) · 9 unmodified lines
985985 __be64 rpdu = cpu_to_be64(pdu);
986986
987987 __blk_add_trace(bt, bio->bi_iter.bi_sector,
988− bio->bi_iter.bi_size, bio_op(bio), bio->bi_opf,
989− BLK_TA_SPLIT,
988+ bio->bi_iter.bi_size, bio->bi_opf, BLK_TA_SPLIT,
990989 blk_status_to_errno(bio->bi_status),
991990 sizeof(rpdu), &rpdu,
992991 blk_trace_bio_get_cgid(q, bio));
@@ -1022,7 +1021,7 @@ static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, · 29 unmodified lines
10221021 r.sector_from = cpu_to_be64(from);
10231022
10241023 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
1025− bio_op(bio), bio->bi_opf, BLK_TA_REMAP,
1024+ bio->bi_opf, BLK_TA_REMAP,
10261025 blk_status_to_errno(bio->bi_status),
10271026 sizeof(r), &r, blk_trace_bio_get_cgid(q, bio));
10281027 rcu_read_unlock();
@@ -1058,7 +1057,7 @@ static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev, · 29 unmodified lines
10581057 r.sector_from = cpu_to_be64(from);
10591058
10601059 __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),
1061− rq_data_dir(rq), 0, BLK_TA_REMAP, 0,
1060+ rq->cmd_flags, BLK_TA_REMAP, 0,
10621061 sizeof(r), &r, blk_trace_request_get_cgid(rq));
10631062 rcu_read_unlock();
10641063 }
@@ -1084,7 +1083,7 @@ void blk_add_driver_data(struct request *rq, void *data, size_t len) · 19 unmodified lines
10841083 return;
10851084 }
10861085
1087− __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0,
1086+ __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0,
10881087 BLK_TA_DRV_DATA, 0, len, data,
10891088 blk_trace_request_get_cgid(rq));
10901089 rcu_read_unlock();
@@ -1867,17 +1866,6 @@ out_unlock_bdev: · 776 unmodified lines
18671866 out:
18681867 return ret ? ret : count;
18691868 }
1870−
1871−int blk_trace_init_sysfs(struct device *dev)
1872−{
1873− return sysfs_create_group(&dev->kobj, &blk_trace_attr_group);
1874−}
1875−
1876−void blk_trace_remove_sysfs(struct device *dev)
1877−{
1878− sysfs_remove_group(&dev->kobj, &blk_trace_attr_group);
1879−}
1880−
18811869 #endif /* CONFIG_BLK_DEV_IO_TRACE */
18821870
18831871 #ifdef CONFIG_EVENT_TRACING
@@ -1885,21 +1873,21 @@ void blk_trace_remove_sysfs(struct device *dev) · 1 unmodified line
18851873 /**
18861874 * blk_fill_rwbs - Fill the buffer rwbs by mapping op to character string.
18871875 * @rwbs: buffer to be filled
1888− * @op: REQ_OP_XXX for the tracepoint
1876+ * @opf: request operation type (REQ_OP_XXX) and flags for the tracepoint
18891877 *
18901878 * Description:
1891− * Maps the REQ_OP_XXX to character and fills the buffer provided by the
1892− * caller with resulting string.
1879+ * Maps each request operation and flag to a single character and fills the
1880+ * buffer provided by the caller with resulting string.
18931881 *
18941882 **/
1895−void blk_fill_rwbs(char *rwbs, unsigned int op)
1883+void blk_fill_rwbs(char *rwbs, blk_opf_t opf)
18961884 {
18971885 int i = 0;
18981886
1899− if (op & REQ_PREFLUSH)
1887+ if (opf & REQ_PREFLUSH)
19001888 rwbs[i++] = 'F';
19011889
1902− switch (op & REQ_OP_MASK) {
1890+ switch (opf & REQ_OP_MASK) {
19031891 case REQ_OP_WRITE:
19041892 rwbs[i++] = 'W';
19051893 break;
@@ -1920,13 +1908,13 @@ void blk_fill_rwbs(char *rwbs, unsigned int op) · 14 unmodified lines
19201908 rwbs[i++] = 'N';
19211909 }
19221910
1923− if (op & REQ_FUA)
1911+ if (opf & REQ_FUA)
19241912 rwbs[i++] = 'F';
1925− if (op & REQ_RAHEAD)
1913+ if (opf & REQ_RAHEAD)
19261914 rwbs[i++] = 'A';
1927− if (op & REQ_SYNC)
1915+ if (opf & REQ_SYNC)
19281916 rwbs[i++] = 'S';
1929− if (op & REQ_META)
1917+ if (opf & REQ_META)
19301918 rwbs[i++] = 'M';
19311919
19321920 rwbs[i] = '\0';
lib/iov_iter.c+92−0
@@ -1268,6 +1268,98 @@ void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) · 1267 unmodified lines
12681268 }
12691269 EXPORT_SYMBOL(iov_iter_discard);
12701270
1271+static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
1272+ unsigned len_mask)
1273+{
1274+ size_t size = i->count;
1275+ size_t skip = i->iov_offset;
1276+ unsigned k;
1277+
1278+ for (k = 0; k < i->nr_segs; k++, skip = 0) {
1279+ size_t len = i->iov[k].iov_len - skip;
1280+
1281+ if (len > size)
1282+ len = size;
1283+ if (len & len_mask)
1284+ return false;
1285+ if ((unsigned long)(i->iov[k].iov_base + skip) & addr_mask)
1286+ return false;
1287+
1288+ size -= len;
1289+ if (!size)
1290+ break;
1291+ }
1292+ return true;
1293+}
1294+
1295+static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask,
1296+ unsigned len_mask)
1297+{
1298+ size_t size = i->count;
1299+ unsigned skip = i->iov_offset;
1300+ unsigned k;
1301+
1302+ for (k = 0; k < i->nr_segs; k++, skip = 0) {
1303+ size_t len = i->bvec[k].bv_len - skip;
1304+
1305+ if (len > size)
1306+ len = size;
1307+ if (len & len_mask)
1308+ return false;
1309+ if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask)
1310+ return false;
1311+
1312+ size -= len;
1313+ if (!size)
1314+ break;
1315+ }
1316+ return true;
1317+}
1318+
1319+/**
1320+ * iov_iter_is_aligned() - Check if the addresses and lengths of each segments
1321+ * are aligned to the parameters.
1322+ *
1323+ * @i: &struct iov_iter to restore
1324+ * @addr_mask: bit mask to check against the iov element's addresses
1325+ * @len_mask: bit mask to check against the iov element's lengths
1326+ *
1327+ * Return: false if any addresses or lengths intersect with the provided masks
1328+ */
1329+bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask,
1330+ unsigned len_mask)
1331+{
1332+ if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
1333+ return iov_iter_aligned_iovec(i, addr_mask, len_mask);
1334+
1335+ if (iov_iter_is_bvec(i))
1336+ return iov_iter_aligned_bvec(i, addr_mask, len_mask);
1337+
1338+ if (iov_iter_is_pipe(i)) {
1339+ unsigned int p_mask = i->pipe->ring_size - 1;
1340+ size_t size = i->count;
1341+
1342+ if (size & len_mask)
1343+ return false;
1344+ if (size && allocated(&i->pipe->bufs[i->head & p_mask])) {
1345+ if (i->iov_offset & addr_mask)
1346+ return false;
1347+ }
1348+
1349+ return true;
1350+ }
1351+
1352+ if (iov_iter_is_xarray(i)) {
1353+ if (i->count & len_mask)
1354+ return false;
1355+ if ((i->xarray_start + i->iov_offset) & addr_mask)
1356+ return false;
1357+ }
1358+
1359+ return true;
1360+}
1361+EXPORT_SYMBOL_GPL(iov_iter_is_aligned);
1362+
12711363 static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
12721364 {
12731365 unsigned long res = 0;