History

Merge branch 'for-5.10/block' into for-5.10/drivers

* for-5.10/block: (140 commits) bdi: replace BDI_CAP_NO_{WRITEBACK,ACCT_DIRTY} with a single flag bdi: invert BDI_CAP_NO_ACCT_WB bdi: replace BDI_CAP_STABLE_WRITES with a queue and a sb flag mm: use SWP_SYNCHRONOUS_IO more intelligently bdi: remove BDI_CAP_SYNCHRONOUS_IO bdi: remove BDI_CAP_CGROUP_WRITEBACK block: lift setting the readahead size into the block layer md: update the optimal I/O size on reshape bdi: initialize ->ra_pages and ->io_pages in bdi_init aoe: set an optimal I/O size bcache: inherit the optimal I/O size drbd: remove dead code in device_to_statistics fs: remove the unused SB_I_MULTIROOT flag block: mark blkdev_get static PM: mm: cleanup swsusp_swap_check mm: split swap_type_of PM: rewrite is_hibernate_resume_dev to not require an inode mm: cleanup claim_swapfile ocfs2: cleanup o2hb_region_dev_store dasd: cleanup dasd_scan_partitions ...

Jens Axboe committed 6 years ago

Commit ac8f7a02 parents 805c6d3 f56753a117 files · +2663 −2094Browse files

Unified · Split

117 files
  1. Documentation/filesystems/locking.rst +0 −3
  2. block/Kconfig +0 −2
  3. block/bfq-iosched.c +7 −2
  4. block/blk-cgroup.c +26 −6
  5. block/blk-core.c +63 −176
  6. block/blk-integrity.c +2 −2
  7. block/blk-iocost.c +1180 −367
  8. block/blk-map.c +66 −111
  9. block/blk-merge.c +201 −0
  10. block/blk-mq-debugfs.c +6 −5
  11. block/blk-mq-sched.c +30 −94
  12. block/blk-mq-sched.h +0 −3
  13. block/blk-mq-tag.c +110 −42
  14. block/blk-mq-tag.h +17 −39
  15. block/blk-mq.c +54 −32
  16. block/blk-mq.h +73 −3
  17. block/blk-settings.c +28 −12
  18. block/blk-sysfs.c +68 −209
  19. block/blk-throttle.c +36 −23
  20. block/blk.h +18 −7
  21. block/bsg-lib.c +1 −1
  22. block/genhd.c +89 −67
  23. block/ioctl.c +16 −13
  24. block/ioprio.c +1 −1
  25. block/kyber-iosched.c +3 −3
  26. block/mq-deadline.c +6 −0
  27. block/partitions/core.c +15 −12
  28. block/scsi_ioctl.c +0 −2
  29. drivers/block/amiflop.c +1 −1
  30. drivers/block/aoe/aoeblk.c +1 −2
  31. drivers/block/aoe/aoecmd.c +1 −3
  32. drivers/block/ataflop.c +4 −3
  33. drivers/block/brd.c +0 −1
  34. drivers/block/drbd/drbd_nl.c +1 −15
  35. drivers/block/floppy.c +5 −3
  36. drivers/block/loop.c +2 −2
  37. drivers/block/nbd.c +8 −7
  38. drivers/block/paride/pcd.c +1 −1
  39. drivers/block/pktcdvd.c +14 −80
  40. drivers/block/rbd.c +2 −2
  41. drivers/block/rnbd/rnbd-clt.c +3 −9
  42. drivers/block/swim.c +2 −20
  43. drivers/block/swim3.c +2 −2
  44. drivers/block/virtio_blk.c +2 −2
  45. drivers/block/xsysace.c +10 −16
  46. drivers/block/zram/zram_drv.c +19 −11
  47. drivers/cdrom/gdrom.c +1 −1
  48. drivers/char/raw.c +24 −32
  49. drivers/ide/ide-cd.c +5 −11
  50. drivers/ide/ide-disk.c +1 −4
  51. drivers/ide/ide-floppy.c +0 −2
  52. drivers/ide/ide-gd.c +6 −42
  53. drivers/md/bcache/request.c +6 −4
  54. drivers/md/bcache/super.c +2 −3
  55. drivers/md/dm-raid.c +1 −1
  56. drivers/md/dm-table.c +4 −5
  57. drivers/md/dm.c +2 −13
  58. drivers/md/md-cluster.c +3 −3
  59. drivers/md/md-linear.c +1 −1
  60. drivers/md/md.c +10 −10
  61. drivers/md/md.h +1 −1
  62. drivers/md/raid0.c +0 −16
  63. drivers/md/raid10.c +15 −31
  64. drivers/md/raid5.c +13 −18
  65. drivers/mmc/core/queue.c +1 −2
  66. drivers/mtd/mtdcore.c +2 −0
  67. drivers/nvdimm/blk.c +1 −2
  68. drivers/nvdimm/btt.c +1 −4
  69. drivers/nvdimm/bus.c +3 −6
  70. drivers/nvdimm/nd.h +1 −1
  71. drivers/nvdimm/pmem.c +1 −3
  72. drivers/nvme/host/core.c +35 −18
  73. drivers/nvme/host/multipath.c +3 −7
  74. drivers/nvme/host/nvme.h +0 −13
  75. drivers/s390/block/dasd_genhd.c +4 −11
  76. drivers/s390/block/dasd_ioctl.c +2 −7
  77. drivers/scsi/iscsi_tcp.c +2 −2
  78. drivers/scsi/sd.c +8 −5
  79. drivers/scsi/sr.c +14 −22
  80. fs/9p/vfs_file.c +1 −1
  81. fs/9p/vfs_super.c +4 −2
  82. fs/afs/super.c +0 −1
  83. fs/block_dev.c +85 −90
  84. fs/btrfs/disk-io.c +0 −2
  85. fs/buffer.c +0 −16
  86. fs/fs-writeback.c +4 −3
  87. fs/fuse/inode.c +2 −2
  88. fs/namei.c +2 −2
  89. fs/nfs/super.c +1 −8
  90. fs/ocfs2/cluster/heartbeat.c +10 −18
  91. fs/super.c +2 −0
  92. fs/ubifs/super.c +2 −0
  93. fs/vboxsf/super.c +2 −0
  94. include/linux/backing-dev.h +13 −65
  95. include/linux/blk-mq.h +12 −3
  96. include/linux/blk_types.h +3 −4
  97. include/linux/blkdev.h +34 −10
  98. include/linux/fs.h +1 −1
  99. include/linux/genhd.h +7 −8
  100. include/linux/ide.h +0 −2
  101. include/linux/suspend.h +2 −2
  102. include/linux/swap.h +2 −1
  103. include/trace/events/iocost.h +9 −17
  104. include/uapi/linux/capability.h +2 −0
  105. kernel/power/swap.c +9 −12
  106. kernel/power/user.c +9 −17
  107. kernel/trace/blktrace.c +1 −1
  108. mm/backing-dev.c +7 −7
  109. mm/filemap.c +2 −2
  110. mm/memcontrol.c +1 −1
  111. mm/memory-failure.c +1 −1
  112. mm/migrate.c +1 −1
  113. mm/mmap.c +1 −1
  114. mm/page-writeback.c +9 −9
  115. mm/page_io.c +10 −8
  116. mm/swapfile.c +26 −23
  117. tools/cgroup/iocost_monitor.py +19 −35
Documentation/filesystems/locking.rst+0−3
@@ -488,9 +488,6 @@ getgeo: no · 487 unmodified lines
488488 swap_slot_free_notify: no (see below)
489489 ======================= ===================
490490
491−unlock_native_capacity and revalidate_disk are called only from
492−check_disk_change().
493−
494491 swap_slot_free_notify is called with swap_lock and sometimes the page lock
495492 held.
496493
block/Kconfig+0−2
@@ -161,8 +161,6 @@ config BLK_WBT_MQ · 160 unmodified lines
161161 depends on BLK_WBT
162162 help
163163 Enable writeback throttling by default on multiqueue devices.
164− Multiqueue currently doesn't have support for IO scheduling,
165− enabling this option is recommended.
166164
167165 config BLK_DEBUG_FS
168166 bool "Block layer debugging information in debugfs"
block/bfq-iosched.c+7−2
@@ -4640,6 +4640,9 @@ static bool bfq_has_work(struct blk_mq_hw_ctx *hctx) · 4639 unmodified lines
46404640 {
46414641 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
46424642
4643+ if (!atomic_read(&hctx->elevator_queued))
4644+ return false;
4645+
46434646 /*
46444647 * Avoiding lock: a race on bfqd->busy_queues should cause at
46454648 * most a call to dispatch for nothing
@@ -5554,6 +5557,7 @@ static void bfq_insert_requests(struct blk_mq_hw_ctx *hctx, · 908 unmodified lines
55545557 rq = list_first_entry(list, struct request, queuelist);
55555558 list_del_init(&rq->queuelist);
55565559 bfq_insert_request(hctx, rq, at_head);
5560+ atomic_inc(&hctx->elevator_queued);
55575561 }
55585562 }
55595563
@@ -5921,6 +5925,7 @@ static void bfq_finish_requeue_request(struct request *rq) · 361 unmodified lines
59215925
59225926 bfq_completed_request(bfqq, bfqd);
59235927 bfq_finish_requeue_request_body(bfqq);
5928+ atomic_dec(&rq->mq_hctx->elevator_queued);
59245929
59255930 spin_unlock_irqrestore(&bfqd->lock, flags);
59265931 } else {
@@ -6360,8 +6365,8 @@ static void bfq_depth_updated(struct blk_mq_hw_ctx *hctx) · 433 unmodified lines
63606365 struct blk_mq_tags *tags = hctx->sched_tags;
63616366 unsigned int min_shallow;
63626367
6363− min_shallow = bfq_update_depths(bfqd, &tags->bitmap_tags);
6364− sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, min_shallow);
6368+ min_shallow = bfq_update_depths(bfqd, tags->bitmap_tags);
6369+ sbitmap_queue_min_shallow_depth(tags->bitmap_tags, min_shallow);
63656370 }
63666371
63676372 static int bfq_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int index)
block/blk-cgroup.c+26−6
@@ -119,6 +119,8 @@ static void blkg_async_bio_workfn(struct work_struct *work) · 118 unmodified lines
119119 async_bio_work);
120120 struct bio_list bios = BIO_EMPTY_LIST;
121121 struct bio *bio;
122+ struct blk_plug plug;
123+ bool need_plug = false;
122124
123125 /* as long as there are pending bios, @blkg can't go away */
124126 spin_lock_bh(&blkg->async_bio_lock);
@@ -126,8 +128,15 @@ static void blkg_async_bio_workfn(struct work_struct *work) · 1 unmodified line
126128 bio_list_init(&blkg->async_bios);
127129 spin_unlock_bh(&blkg->async_bio_lock);
128130
131+ /* start plug only when bio_list contains at least 2 bios */
132+ if (bios.head && bios.head->bi_next) {
133+ need_plug = true;
134+ blk_start_plug(&plug);
135+ }
129136 while ((bio = bio_list_pop(&bios)))
130137 submit_bio(bio);
138+ if (need_plug)
139+ blk_finish_plug(&plug);
131140 }
132141
133142 /**
@@ -1613,16 +1622,24 @@ static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now) · 1479 unmodified lines
16131622 static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
16141623 {
16151624 unsigned long pflags;
1625+ bool clamp;
16161626 u64 now = ktime_to_ns(ktime_get());
16171627 u64 exp;
16181628 u64 delay_nsec = 0;
16191629 int tok;
16201630
16211631 while (blkg->parent) {
1622− if (atomic_read(&blkg->use_delay)) {
1632+ int use_delay = atomic_read(&blkg->use_delay);
1633+
1634+ if (use_delay) {
1635+ u64 this_delay;
1636+
16231637 blkcg_scale_delay(blkg, now);
1624− delay_nsec = max_t(u64, delay_nsec,
1625− atomic64_read(&blkg->delay_nsec));
1638+ this_delay = atomic64_read(&blkg->delay_nsec);
1639+ if (this_delay > delay_nsec) {
1640+ delay_nsec = this_delay;
1641+ clamp = use_delay > 0;
1642+ }
16261643 }
16271644 blkg = blkg->parent;
16281645 }
@@ -1634,10 +1651,13 @@ static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay) · 5 unmodified lines
16341651 * Let's not sleep for all eternity if we've amassed a huge delay.
16351652 * Swapping or metadata IO can accumulate 10's of seconds worth of
16361653 * delay, and we want userspace to be able to do _something_ so cap the
1637− * delays at 1 second. If there's 10's of seconds worth of delay then
1638− * the tasks will be delayed for 1 second for every syscall.
1654+ * delays at 0.25s. If there's 10's of seconds worth of delay then the
1655+ * tasks will be delayed for 0.25 second for every syscall. If
1656+ * blkcg_set_delay() was used as indicated by negative use_delay, the
1657+ * caller is responsible for regulating the range.
16391658 */
1640− delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
1659+ if (clamp)
1660+ delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
16411661
16421662 if (use_memdelay)
16431663 psi_memstall_enter(&pflags);
block/blk-core.c+63−176
@@ -116,8 +116,8 @@ void blk_rq_init(struct request_queue *q, struct request *rq) · 115 unmodified lines
116116 rq->__sector = (sector_t) -1;
117117 INIT_HLIST_NODE(&rq->hash);
118118 RB_CLEAR_NODE(&rq->rb_node);
119− rq->tag = -1;
120− rq->internal_tag = -1;
119+ rq->tag = BLK_MQ_NO_TAG;
120+ rq->internal_tag = BLK_MQ_NO_TAG;
121121 rq->start_time_ns = ktime_get_ns();
122122 rq->part = NULL;
123123 refcount_set(&rq->ref, 1);
@@ -538,11 +538,10 @@ struct request_queue *blk_alloc_queue(int node_id) · 414 unmodified lines
538538 if (!q->stats)
539539 goto fail_stats;
540540
541− q->backing_dev_info->ra_pages = VM_READAHEAD_PAGES;
542− q->backing_dev_info->io_pages = VM_READAHEAD_PAGES;
543− q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
544541 q->node = node_id;
545542
543+ atomic_set(&q->nr_active_requests_shared_sbitmap, 0);
544+
546545 timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
547546 laptop_mode_timer_fn, 0);
548547 timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
@@ -643,162 +642,6 @@ void blk_put_request(struct request *req) · 94 unmodified lines
643642 }
644643 EXPORT_SYMBOL(blk_put_request);
645644
646−static void blk_account_io_merge_bio(struct request *req)
647−{
648− if (!blk_do_io_stat(req))
649− return;
650−
651− part_stat_lock();
652− part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
653− part_stat_unlock();
654−}
655−
656−bool bio_attempt_back_merge(struct request *req, struct bio *bio,
657− unsigned int nr_segs)
658−{
659− const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
660−
661− if (!ll_back_merge_fn(req, bio, nr_segs))
662− return false;
663−
664− trace_block_bio_backmerge(req->q, req, bio);
665− rq_qos_merge(req->q, req, bio);
666−
667− if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
668− blk_rq_set_mixed_merge(req);
669−
670− req->biotail->bi_next = bio;
671− req->biotail = bio;
672− req->__data_len += bio->bi_iter.bi_size;
673−
674− bio_crypt_free_ctx(bio);
675−
676− blk_account_io_merge_bio(req);
677− return true;
678−}
679−
680−bool bio_attempt_front_merge(struct request *req, struct bio *bio,
681− unsigned int nr_segs)
682−{
683− const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
684−
685− if (!ll_front_merge_fn(req, bio, nr_segs))
686− return false;
687−
688− trace_block_bio_frontmerge(req->q, req, bio);
689− rq_qos_merge(req->q, req, bio);
690−
691− if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
692− blk_rq_set_mixed_merge(req);
693−
694− bio->bi_next = req->bio;
695− req->bio = bio;
696−
697− req->__sector = bio->bi_iter.bi_sector;
698− req->__data_len += bio->bi_iter.bi_size;
699−
700− bio_crypt_do_front_merge(req, bio);
701−
702− blk_account_io_merge_bio(req);
703− return true;
704−}
705−
706−bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
707− struct bio *bio)
708−{
709− unsigned short segments = blk_rq_nr_discard_segments(req);
710−
711− if (segments >= queue_max_discard_segments(q))
712− goto no_merge;
713− if (blk_rq_sectors(req) + bio_sectors(bio) >
714− blk_rq_get_max_sectors(req, blk_rq_pos(req)))
715− goto no_merge;
716−
717− rq_qos_merge(q, req, bio);
718−
719− req->biotail->bi_next = bio;
720− req->biotail = bio;
721− req->__data_len += bio->bi_iter.bi_size;
722− req->nr_phys_segments = segments + 1;
723−
724− blk_account_io_merge_bio(req);
725− return true;
726−no_merge:
727− req_set_nomerge(q, req);
728− return false;
729−}
730−
731−/**
732− * blk_attempt_plug_merge - try to merge with %current's plugged list
733− * @q: request_queue new bio is being queued at
734− * @bio: new bio being queued
735− * @nr_segs: number of segments in @bio
736− * @same_queue_rq: pointer to &struct request that gets filled in when
737− * another request associated with @q is found on the plug list
738− * (optional, may be %NULL)
739− *
740− * Determine whether @bio being queued on @q can be merged with a request
741− * on %current's plugged list. Returns %true if merge was successful,
742− * otherwise %false.
743− *
744− * Plugging coalesces IOs from the same issuer for the same purpose without
745− * going through @q->queue_lock. As such it's more of an issuing mechanism
746− * than scheduling, and the request, while may have elvpriv data, is not
747− * added on the elevator at this point. In addition, we don't have
748− * reliable access to the elevator outside queue lock. Only check basic
749− * merging parameters without querying the elevator.
750− *
751− * Caller must ensure !blk_queue_nomerges(q) beforehand.
752− */
753−bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
754− unsigned int nr_segs, struct request **same_queue_rq)
755−{
756− struct blk_plug *plug;
757− struct request *rq;
758− struct list_head *plug_list;
759−
760− plug = blk_mq_plug(q, bio);
761− if (!plug)
762− return false;
763−
764− plug_list = &plug->mq_list;
765−
766− list_for_each_entry_reverse(rq, plug_list, queuelist) {
767− bool merged = false;
768−
769− if (rq->q == q && same_queue_rq) {
770− /*
771− * Only blk-mq multiple hardware queues case checks the
772− * rq in the same queue, there should be only one such
773− * rq in a queue
774− **/
775− *same_queue_rq = rq;
776− }
777−
778− if (rq->q != q || !blk_rq_merge_ok(rq, bio))
779− continue;
780−
781− switch (blk_try_merge(rq, bio)) {
782− case ELEVATOR_BACK_MERGE:
783− merged = bio_attempt_back_merge(rq, bio, nr_segs);
784− break;
785− case ELEVATOR_FRONT_MERGE:
786− merged = bio_attempt_front_merge(rq, bio, nr_segs);
787− break;
788− case ELEVATOR_DISCARD_MERGE:
789− merged = bio_attempt_discard_merge(q, rq, bio);
790− break;
791− default:
792− break;
793− }
794−
795− if (merged)
796− return true;
797− }
798−
799− return false;
800−}
801−
802645 static void handle_bad_sector(struct bio *bio, sector_t maxsector)
803646 {
804647 char b[BDEVNAME_SIZE];
@@ -1301,14 +1144,28 @@ EXPORT_SYMBOL(submit_bio); · 496 unmodified lines
13011144 * limits when retrying requests on other queues. Those requests need
13021145 * to be checked against the new queue limits again during dispatch.
13031146 */
1304−static int blk_cloned_rq_check_limits(struct request_queue *q,
1147+static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q,
13051148 struct request *rq)
13061149 {
1307− if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
1150+ unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
1151+
1152+ if (blk_rq_sectors(rq) > max_sectors) {
1153+ /*
1154+ * SCSI device does not have a good way to return if
1155+ * Write Same/Zero is actually supported. If a device rejects
1156+ * a non-read/write command (discard, write same,etc.) the
1157+ * low-level device driver will set the relevant queue limit to
1158+ * 0 to prevent blk-lib from issuing more of the offending
1159+ * operations. Commands queued prior to the queue limit being
1160+ * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
1161+ * errors being propagated to upper layers.
1162+ */
1163+ if (max_sectors == 0)
1164+ return BLK_STS_NOTSUPP;
1165+
13081166 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
1309− __func__, blk_rq_sectors(rq),
1310− blk_queue_get_max_sectors(q, req_op(rq)));
1311− return -EIO;
1167+ __func__, blk_rq_sectors(rq), max_sectors);
1168+ return BLK_STS_IOERR;
13121169 }
13131170
13141171 /*
@@ -1321,10 +1178,10 @@ static int blk_cloned_rq_check_limits(struct request_queue *q, · 6 unmodified lines
13211178 if (rq->nr_phys_segments > queue_max_segments(q)) {
13221179 printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",
13231180 __func__, rq->nr_phys_segments, queue_max_segments(q));
1324− return -EIO;
1181+ return BLK_STS_IOERR;
13251182 }
13261183
1327− return 0;
1184+ return BLK_STS_OK;
13281185 }
13291186
13301187 /**
@@ -1334,8 +1191,11 @@ static int blk_cloned_rq_check_limits(struct request_queue *q, · 3 unmodified lines
13341191 */
13351192 blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
13361193 {
1337− if (blk_cloned_rq_check_limits(q, rq))
1338− return BLK_STS_IOERR;
1194+ blk_status_t ret;
1195+
1196+ ret = blk_cloned_rq_check_limits(q, rq);
1197+ if (ret != BLK_STS_OK)
1198+ return ret;
13391199
13401200 if (rq->rq_disk &&
13411201 should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
@@ -1461,10 +1321,9 @@ void blk_account_io_start(struct request *rq) · 119 unmodified lines
14611321 part_stat_unlock();
14621322 }
14631323
1464−unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
1465− unsigned int op)
1324+static unsigned long __part_start_io_acct(struct hd_struct *part,
1325+ unsigned int sectors, unsigned int op)
14661326 {
1467− struct hd_struct *part = &disk->part0;
14681327 const int sgrp = op_stat_group(op);
14691328 unsigned long now = READ_ONCE(jiffies);
14701329
@@ -1477,12 +1336,26 @@ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, · 6 unmodified lines
14771336
14781337 return now;
14791338 }
1339+
1340+unsigned long part_start_io_acct(struct gendisk *disk, struct hd_struct **part,
1341+ struct bio *bio)
1342+{
1343+ *part = disk_map_sector_rcu(disk, bio->bi_iter.bi_sector);
1344+
1345+ return __part_start_io_acct(*part, bio_sectors(bio), bio_op(bio));
1346+}
1347+EXPORT_SYMBOL_GPL(part_start_io_acct);
1348+
1349+unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
1350+ unsigned int op)
1351+{
1352+ return __part_start_io_acct(&disk->part0, sectors, op);
1353+}
14801354 EXPORT_SYMBOL(disk_start_io_acct);
14811355
1482−void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1483− unsigned long start_time)
1356+static void __part_end_io_acct(struct hd_struct *part, unsigned int op,
1357+ unsigned long start_time)
14841358 {
1485− struct hd_struct *part = &disk->part0;
14861359 const int sgrp = op_stat_group(op);
14871360 unsigned long now = READ_ONCE(jiffies);
14881361 unsigned long duration = now - start_time;
@@ -1493,6 +1366,20 @@ void disk_end_io_acct(struct gendisk *disk, unsigned int op, · 4 unmodified lines
14931366 part_stat_local_dec(part, in_flight[op_is_write(op)]);
14941367 part_stat_unlock();
14951368 }
1369+
1370+void part_end_io_acct(struct hd_struct *part, struct bio *bio,
1371+ unsigned long start_time)
1372+{
1373+ __part_end_io_acct(part, bio_op(bio), start_time);
1374+ hd_struct_put(part);
1375+}
1376+EXPORT_SYMBOL_GPL(part_end_io_acct);
1377+
1378+void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1379+ unsigned long start_time)
1380+{
1381+ __part_end_io_acct(&disk->part0, op, start_time);
1382+}
14961383 EXPORT_SYMBOL(disk_end_io_acct);
14971384
14981385 /*
block/blk-integrity.c+2−2
@@ -408,7 +408,7 @@ void blk_integrity_register(struct gendisk *disk, struct blk_integrity *template · 407 unmodified lines
408408 bi->tuple_size = template->tuple_size;
409409 bi->tag_size = template->tag_size;
410410
411− disk->queue->backing_dev_info->capabilities |= BDI_CAP_STABLE_WRITES;
411+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, disk->queue);
412412
413413 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
414414 if (disk->queue->ksm) {
@@ -428,7 +428,7 @@ EXPORT_SYMBOL(blk_integrity_register); · 13 unmodified lines
428428 */
429429 void blk_integrity_unregister(struct gendisk *disk)
430430 {
431− disk->queue->backing_dev_info->capabilities &= ~BDI_CAP_STABLE_WRITES;
431+ blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, disk->queue);
432432 memset(&disk->queue->integrity, 0, sizeof(struct blk_integrity));
433433 }
434434 EXPORT_SYMBOL(blk_integrity_unregister);
block/blk-iocost.c+1180−367
@@ -68,7 +68,7 @@ · 67 unmodified lines
6868 * gets 300/(100+300) or 75% share, and A0 and A1 equally splits the rest,
6969 * 12.5% each. The distribution mechanism only cares about these flattened
7070 * shares. They're called hweights (hierarchical weights) and always add
71− * upto 1 (HWEIGHT_WHOLE).
71+ * upto 1 (WEIGHT_ONE).
7272 *
7373 * A given cgroup's vtime runs slower in inverse proportion to its hweight.
7474 * For example, with 12.5% weight, A0's time runs 8 times slower (100/12.5)
@@ -179,6 +179,8 @@ · 104 unmodified lines
179179 #include <linux/parser.h>
180180 #include <linux/sched/signal.h>
181181 #include <linux/blk-cgroup.h>
182+#include <asm/local.h>
183+#include <asm/local64.h>
182184 #include "blk-rq-qos.h"
183185 #include "blk-stat.h"
184186 #include "blk-wbt.h"
@@ -215,36 +217,21 @@ enum { · 30 unmodified lines
215217 MAX_PERIOD = USEC_PER_SEC,
216218
217219 /*
218− * A cgroup's vtime can run 50% behind the device vtime, which
220+ * iocg->vtime is targeted at 50% behind the device vtime, which
219221 * serves as its IO credit buffer. Surplus weight adjustment is
220222 * immediately canceled if the vtime margin runs below 10%.
221223 */
222− MARGIN_PCT = 50,
223− INUSE_MARGIN_PCT = 10,
224+ MARGIN_MIN_PCT = 10,
225+ MARGIN_LOW_PCT = 20,
226+ MARGIN_TARGET_PCT = 50,
224227
225− /* Have some play in waitq timer operations */
226− WAITQ_TIMER_MARGIN_PCT = 5,
228+ INUSE_ADJ_STEP_PCT = 25,
227229
228− /*
229− * vtime can wrap well within a reasonable uptime when vrate is
230− * consistently raised. Don't trust recorded cgroup vtime if the
231− * period counter indicates that it's older than 5mins.
232− */
233− VTIME_VALID_DUR = 300 * USEC_PER_SEC,
234−
235− /*
236− * Remember the past three non-zero usages and use the max for
237− * surplus calculation. Three slots guarantee that we remember one
238− * full period usage from the last active stretch even after
239− * partial deactivation and re-activation periods. Don't start
240− * giving away weight before collecting two data points to prevent
241− * hweight adjustments based on one partial activation period.
242− */
243− NR_USAGE_SLOTS = 3,
244− MIN_VALID_USAGES = 2,
230+ /* Have some play in timer operations */
231+ TIMER_SLACK_PCT = 1,
245232
246233 /* 1/64k is granular enough and can easily be handled w/ u32 */
247− HWEIGHT_WHOLE = 1 << 16,
234+ WEIGHT_ONE = 1 << 16,
248235
249236 /*
250237 * As vtime is used to calculate the cost of each IO, it needs to
@@ -275,16 +262,40 @@ enum { · 24 unmodified lines
275262 /* unbusy hysterisis */
276263 UNBUSY_THR_PCT = 75,
277264
278− /* don't let cmds which take a very long time pin lagging for too long */
279− MAX_LAGGING_PERIODS = 10,
265+ /*
266+ * The effect of delay is indirect and non-linear and a huge amount of
267+ * future debt can accumulate abruptly while unthrottled. Linearly scale
268+ * up delay as debt is going up and then let it decay exponentially.
269+ * This gives us quick ramp ups while delay is accumulating and long
270+ * tails which can help reducing the frequency of debt explosions on
271+ * unthrottle. The parameters are experimentally determined.
272+ *
273+ * The delay mechanism provides adequate protection and behavior in many
274+ * cases. However, this is far from ideal and falls shorts on both
275+ * fronts. The debtors are often throttled too harshly costing a
276+ * significant level of fairness and possibly total work while the
277+ * protection against their impacts on the system can be choppy and
278+ * unreliable.
279+ *
280+ * The shortcoming primarily stems from the fact that, unlike for page
281+ * cache, the kernel doesn't have well-defined back-pressure propagation
282+ * mechanism and policies for anonymous memory. Fully addressing this
283+ * issue will likely require substantial improvements in the area.
284+ */
285+ MIN_DELAY_THR_PCT = 500,
286+ MAX_DELAY_THR_PCT = 25000,
287+ MIN_DELAY = 250,
288+ MAX_DELAY = 250 * USEC_PER_MSEC,
280289
281290 /*
282− * If usage% * 1.25 + 2% is lower than hweight% by more than 3%,
283− * donate the surplus.
291+ * Halve debts if total usage keeps staying under 25% w/o any shortages
292+ * for over 100ms.
284293 */
285− SURPLUS_SCALE_PCT = 125, /* * 125% */
286− SURPLUS_SCALE_ABS = HWEIGHT_WHOLE / 50, /* + 2% */
287− SURPLUS_MIN_ADJ_DELTA = HWEIGHT_WHOLE / 33, /* 3% */
294+ DEBT_BUSY_USAGE_PCT = 25,
295+ DEBT_REDUCTION_IDLE_DUR = 100 * USEC_PER_MSEC,
296+
297+ /* don't let cmds which take a very long time pin lagging for too long */
298+ MAX_LAGGING_PERIODS = 10,
288299
289300 /* switch iff the conditions are met for longer than this */
290301 AUTOP_CYCLE_NSEC = 10LLU * NSEC_PER_SEC,
@@ -372,9 +383,15 @@ struct ioc_params { · 81 unmodified lines
372383 u32 too_slow_vrate_pct;
373384 };
374385
386+struct ioc_margins {
387+ s64 min;
388+ s64 low;
389+ s64 target;
390+};
391+
375392 struct ioc_missed {
376− u32 nr_met;
377− u32 nr_missed;
393+ local_t nr_met;
394+ local_t nr_missed;
378395 u32 last_met;
379396 u32 last_missed;
380397 };
@@ -382,7 +399,7 @@ struct ioc_missed { · 1 unmodified line
382399 struct ioc_pcpu_stat {
383400 struct ioc_missed missed[2];
384401
385− u64 rq_wait_ns;
402+ local64_t rq_wait_ns;
386403 u64 last_rq_wait_ns;
387404 };
388405
@@ -393,8 +410,9 @@ struct ioc { · 4 unmodified lines
393410 bool enabled;
394411
395412 struct ioc_params params;
413+ struct ioc_margins margins;
396414 u32 period_us;
397− u32 margin_us;
415+ u32 timer_slack_ns;
398416 u64 vrate_min;
399417 u64 vrate_max;
400418
@@ -405,18 +423,22 @@ struct ioc { · 4 unmodified lines
405423
406424 enum ioc_running running;
407425 atomic64_t vtime_rate;
426+ u64 vtime_base_rate;
427+ s64 vtime_err;
408428
409429 seqcount_spinlock_t period_seqcount;
410− u32 period_at; /* wallclock starttime */
430+ u64 period_at; /* wallclock starttime */
411431 u64 period_at_vtime; /* vtime starttime */
412432
413433 atomic64_t cur_period; /* inc'd each period */
414434 int busy_level; /* saturation history */
415435
416− u64 inuse_margin_vtime;
417436 bool weights_updated;
418437 atomic_t hweight_gen; /* for lazy hweights */
419438
439+ /* the last time debt cancel condition wasn't met */
440+ u64 debt_busy_at;
441+
420442 u64 autop_too_fast_at;
421443 u64 autop_too_slow_at;
422444 int autop_idx;
@@ -424,6 +446,17 @@ struct ioc { · 1 unmodified line
424446 bool user_cost_model:1;
425447 };
426448
449+struct iocg_pcpu_stat {
450+ local64_t abs_vusage;
451+};
452+
453+struct iocg_stat {
454+ u64 usage_us;
455+ u64 wait_us;
456+ u64 indebt_us;
457+ u64 indelay_us;
458+};
459+
427460 /* per device-cgroup pair */
428461 struct ioc_gq {
429462 struct blkg_policy_data pd;
@@ -443,12 +476,17 @@ struct ioc_gq { · 13 unmodified lines
443476 *
444477 * `last_inuse` remembers `inuse` while an iocg is idle to persist
445478 * surplus adjustments.
479+ *
480+ * `inuse` may be adjusted dynamically during period. `saved_*` are used
481+ * to determine and track adjustments.
446482 */
447483 u32 cfg_weight;
448484 u32 weight;
449485 u32 active;
450486 u32 inuse;
487+
451488 u32 last_inuse;
489+ s64 saved_margin;
452490
453491 sector_t cursor; /* to detect randio */
454492
@@ -461,14 +499,14 @@ struct ioc_gq { · 6 unmodified lines
461499 * `vtime_done` is the same but progressed on completion rather
462500 * than issue. The delta behind `vtime` represents the cost of
463501 * currently in-flight IOs.
464− *
465− * `last_vtime` is used to remember `vtime` at the end of the last
466− * period to calculate utilization.
467502 */
468503 atomic64_t vtime;
469504 atomic64_t done_vtime;
470505 u64 abs_vdebt;
471− u64 last_vtime;
506+
507+ /* current delay in effect and when it started */
508+ u64 delay;
509+ u64 delay_at;
472510
473511 /*
474512 * The period this iocg was last active in. Used for deactivation
@@ -477,21 +515,35 @@ struct ioc_gq { · 2 unmodified lines
477515 atomic64_t active_period;
478516 struct list_head active_list;
479517
480− /* see __propagate_active_weight() and current_hweight() for details */
518+ /* see __propagate_weights() and current_hweight() for details */
481519 u64 child_active_sum;
482520 u64 child_inuse_sum;
521+ u64 child_adjusted_sum;
483522 int hweight_gen;
484523 u32 hweight_active;
485524 u32 hweight_inuse;
486− bool has_surplus;
525+ u32 hweight_donating;
526+ u32 hweight_after_donation;
527+
528+ struct list_head walk_list;
529+ struct list_head surplus_list;
487530
488531 struct wait_queue_head waitq;
489532 struct hrtimer waitq_timer;
490− struct hrtimer delay_timer;
491533
492− /* usage is recorded as fractions of HWEIGHT_WHOLE */
493− int usage_idx;
494− u32 usages[NR_USAGE_SLOTS];
534+ /* timestamp at the latest activation */
535+ u64 activated_at;
536+
537+ /* statistics */
538+ struct iocg_pcpu_stat __percpu *pcpu_stat;
539+ struct iocg_stat local_stat;
540+ struct iocg_stat desc_stat;
541+ struct iocg_stat last_stat;
542+ u64 last_stat_abs_vusage;
543+ u64 usage_delta_us;
544+ u64 wait_since;
545+ u64 indebt_since;
546+ u64 indelay_since;
495547
496548 /* this iocg's depth in the hierarchy and ancestors including self */
497549 int level;
@@ -506,7 +558,7 @@ struct ioc_cgrp { · 8 unmodified lines
506558
507559 struct ioc_now {
508560 u64 now_ns;
509− u32 now;
561+ u64 now;
510562 u64 vnow;
511563 u64 vrate;
512564 };
@@ -656,7 +708,7 @@ static struct ioc_cgrp *blkcg_to_iocc(struct blkcg *blkcg) · 143 unmodified lines
656708 */
657709 static u64 abs_cost_to_cost(u64 abs_cost, u32 hw_inuse)
658710 {
659− return DIV64_U64_ROUND_UP(abs_cost * HWEIGHT_WHOLE, hw_inuse);
711+ return DIV64_U64_ROUND_UP(abs_cost * WEIGHT_ONE, hw_inuse);
660712 }
661713
662714 /*
@@ -664,18 +716,56 @@ static u64 abs_cost_to_cost(u64 abs_cost, u32 hw_inuse) · 1 unmodified line
664716 */
665717 static u64 cost_to_abs_cost(u64 cost, u32 hw_inuse)
666718 {
667− return DIV64_U64_ROUND_UP(cost * hw_inuse, HWEIGHT_WHOLE);
719+ return DIV64_U64_ROUND_UP(cost * hw_inuse, WEIGHT_ONE);
668720 }
669721
670−static void iocg_commit_bio(struct ioc_gq *iocg, struct bio *bio, u64 cost)
722+static void iocg_commit_bio(struct ioc_gq *iocg, struct bio *bio,
723+ u64 abs_cost, u64 cost)
671724 {
725+ struct iocg_pcpu_stat *gcs;
726+
672727 bio->bi_iocost_cost = cost;
673728 atomic64_add(cost, &iocg->vtime);
729+
730+ gcs = get_cpu_ptr(iocg->pcpu_stat);
731+ local64_add(abs_cost, &gcs->abs_vusage);
732+ put_cpu_ptr(gcs);
733+}
734+
735+static void iocg_lock(struct ioc_gq *iocg, bool lock_ioc, unsigned long *flags)
736+{
737+ if (lock_ioc) {
738+ spin_lock_irqsave(&iocg->ioc->lock, *flags);
739+ spin_lock(&iocg->waitq.lock);
740+ } else {
741+ spin_lock_irqsave(&iocg->waitq.lock, *flags);
742+ }
743+}
744+
745+static void iocg_unlock(struct ioc_gq *iocg, bool unlock_ioc, unsigned long *flags)
746+{
747+ if (unlock_ioc) {
748+ spin_unlock(&iocg->waitq.lock);
749+ spin_unlock_irqrestore(&iocg->ioc->lock, *flags);
750+ } else {
751+ spin_unlock_irqrestore(&iocg->waitq.lock, *flags);
752+ }
674753 }
675754
676755 #define CREATE_TRACE_POINTS
677756 #include <trace/events/iocost.h>
678757
758+static void ioc_refresh_margins(struct ioc *ioc)
759+{
760+ struct ioc_margins *margins = &ioc->margins;
761+ u32 period_us = ioc->period_us;
762+ u64 vrate = ioc->vtime_base_rate;
763+
764+ margins->min = (period_us * MARGIN_MIN_PCT / 100) * vrate;
765+ margins->low = (period_us * MARGIN_LOW_PCT / 100) * vrate;
766+ margins->target = (period_us * MARGIN_TARGET_PCT / 100) * vrate;
767+}
768+
679769 /* latency Qos params changed, update period_us and all the dependent params */
680770 static void ioc_refresh_period_us(struct ioc *ioc)
681771 {
@@ -709,9 +799,10 @@ static void ioc_refresh_period_us(struct ioc *ioc) · 27 unmodified lines
709799
710800 /* calculate dependent params */
711801 ioc->period_us = period_us;
712− ioc->margin_us = period_us * MARGIN_PCT / 100;
713− ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(
714− period_us * VTIME_PER_USEC * INUSE_MARGIN_PCT, 100);
802+ ioc->timer_slack_ns = div64_u64(
803+ (u64)period_us * NSEC_PER_USEC * TIMER_SLACK_PCT,
804+ 100);
805+ ioc_refresh_margins(ioc);
715806 }
716807
717808 static int ioc_autop_idx(struct ioc *ioc)
@@ -738,8 +829,7 @@ static int ioc_autop_idx(struct ioc *ioc) · 20 unmodified lines
738829 return idx;
739830
740831 /* step up/down based on the vrate */
741− vrate_pct = div64_u64(atomic64_read(&ioc->vtime_rate) * 100,
742− VTIME_PER_USEC);
832+ vrate_pct = div64_u64(ioc->vtime_base_rate * 100, VTIME_PER_USEC);
743833 now_ns = ktime_get_ns();
744834
745835 if (p->too_fast_vrate_pct && p->too_fast_vrate_pct <= vrate_pct) {
@@ -847,6 +937,43 @@ static bool ioc_refresh_params(struct ioc *ioc, bool force) · 101 unmodified lines
847937 return true;
848938 }
849939
940+/*
941+ * When an iocg accumulates too much vtime or gets deactivated, we throw away
942+ * some vtime, which lowers the overall device utilization. As the exact amount
943+ * which is being thrown away is known, we can compensate by accelerating the
944+ * vrate accordingly so that the extra vtime generated in the current period
945+ * matches what got lost.
946+ */
947+static void ioc_refresh_vrate(struct ioc *ioc, struct ioc_now *now)
948+{
949+ s64 pleft = ioc->period_at + ioc->period_us - now->now;
950+ s64 vperiod = ioc->period_us * ioc->vtime_base_rate;
951+ s64 vcomp, vcomp_min, vcomp_max;
952+
953+ lockdep_assert_held(&ioc->lock);
954+
955+ /* we need some time left in this period */
956+ if (pleft <= 0)
957+ goto done;
958+
959+ /*
960+ * Calculate how much vrate should be adjusted to offset the error.
961+ * Limit the amount of adjustment and deduct the adjusted amount from
962+ * the error.
963+ */
964+ vcomp = -div64_s64(ioc->vtime_err, pleft);
965+ vcomp_min = -(ioc->vtime_base_rate >> 1);
966+ vcomp_max = ioc->vtime_base_rate;
967+ vcomp = clamp(vcomp, vcomp_min, vcomp_max);
968+
969+ ioc->vtime_err += vcomp * pleft;
970+
971+ atomic64_set(&ioc->vtime_rate, ioc->vtime_base_rate + vcomp);
972+done:
973+ /* bound how much error can accumulate */
974+ ioc->vtime_err = clamp(ioc->vtime_err, -vperiod, vperiod);
975+}
976+
850977 /* take a snapshot of the current [v]time and vrate */
851978 static void ioc_now(struct ioc *ioc, struct ioc_now *now)
852979 {
@@ -886,16 +1013,25 @@ static void ioc_start_period(struct ioc *ioc, struct ioc_now *now) · 33 unmodified lines
8861013
8871014 /*
8881015 * Update @iocg's `active` and `inuse` to @active and @inuse, update level
889− * weight sums and propagate upwards accordingly.
1016+ * weight sums and propagate upwards accordingly. If @save, the current margin
1017+ * is saved to be used as reference for later inuse in-period adjustments.
8901018 */
891−static void __propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse)
1019+static void __propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
1020+ bool save, struct ioc_now *now)
8921021 {
8931022 struct ioc *ioc = iocg->ioc;
8941023 int lvl;
8951024
8961025 lockdep_assert_held(&ioc->lock);
8971026
898− inuse = min(active, inuse);
1027+ inuse = clamp_t(u32, inuse, 1, active);
1028+
1029+ iocg->last_inuse = iocg->inuse;
1030+ if (save)
1031+ iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);
1032+
1033+ if (active == iocg->active && inuse == iocg->inuse)
1034+ return;
8991035
9001036 for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
9011037 struct ioc_gq *parent = iocg->ancestors[lvl];
@@ -933,7 +1069,7 @@ static void __propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse · 31 unmodified lines
9331069 ioc->weights_updated = true;
9341070 }
9351071
936−static void commit_active_weights(struct ioc *ioc)
1072+static void commit_weights(struct ioc *ioc)
9371073 {
9381074 lockdep_assert_held(&ioc->lock);
9391075
@@ -945,10 +1081,11 @@ static void commit_active_weights(struct ioc *ioc) · 5 unmodified lines
9451081 }
9461082 }
9471083
948−static void propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse)
1084+static void propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
1085+ bool save, struct ioc_now *now)
9491086 {
950− __propagate_active_weight(iocg, active, inuse);
951− commit_active_weights(iocg->ioc);
1087+ __propagate_weights(iocg, active, inuse, save, now);
1088+ commit_weights(iocg->ioc);
9521089 }
9531090
9541091 static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep)
@@ -964,9 +1101,9 @@ static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep · 9 unmodified lines
9641101 goto out;
9651102
9661103 /*
967− * Paired with wmb in commit_active_weights(). If we saw the
968− * updated hweight_gen, all the weight updates from
969− * __propagate_active_weight() are visible too.
1104+ * Paired with wmb in commit_weights(). If we saw the updated
1105+ * hweight_gen, all the weight updates from __propagate_weights() are
1106+ * visible too.
9701107 *
9711108 * We can race with weight updates during calculation and get it
9721109 * wrong. However, hweight_gen would have changed and a future
@@ -975,12 +1112,12 @@ static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep · 2 unmodified lines
9751112 */
9761113 smp_rmb();
9771114
978− hwa = hwi = HWEIGHT_WHOLE;
1115+ hwa = hwi = WEIGHT_ONE;
9791116 for (lvl = 0; lvl <= iocg->level - 1; lvl++) {
9801117 struct ioc_gq *parent = iocg->ancestors[lvl];
9811118 struct ioc_gq *child = iocg->ancestors[lvl + 1];
982− u32 active_sum = READ_ONCE(parent->child_active_sum);
983− u32 inuse_sum = READ_ONCE(parent->child_inuse_sum);
1119+ u64 active_sum = READ_ONCE(parent->child_active_sum);
1120+ u64 inuse_sum = READ_ONCE(parent->child_inuse_sum);
9841121 u32 active = READ_ONCE(child->active);
9851122 u32 inuse = READ_ONCE(child->inuse);
9861123
@@ -988,11 +1125,11 @@ static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep · 1 unmodified line
9881125 if (!active_sum || !inuse_sum)
9891126 continue;
9901127
991− active_sum = max(active, active_sum);
992− hwa = hwa * active / active_sum; /* max 16bits * 10000 */
1128+ active_sum = max_t(u64, active, active_sum);
1129+ hwa = div64_u64((u64)hwa * active, active_sum);
9931130
994− inuse_sum = max(inuse, inuse_sum);
995− hwi = hwi * inuse / inuse_sum; /* max 16bits * 10000 */
1131+ inuse_sum = max_t(u64, inuse, inuse_sum);
1132+ hwi = div64_u64((u64)hwi * inuse, inuse_sum);
9961133 }
9971134
9981135 iocg->hweight_active = max_t(u32, hwa, 1);
@@ -1005,7 +1142,33 @@ out: · 6 unmodified lines
10051142 *hw_inusep = iocg->hweight_inuse;
10061143 }
10071144
1008−static void weight_updated(struct ioc_gq *iocg)
1145+/*
1146+ * Calculate the hweight_inuse @iocg would get with max @inuse assuming all the
1147+ * other weights stay unchanged.
1148+ */
1149+static u32 current_hweight_max(struct ioc_gq *iocg)
1150+{
1151+ u32 hwm = WEIGHT_ONE;
1152+ u32 inuse = iocg->active;
1153+ u64 child_inuse_sum;
1154+ int lvl;
1155+
1156+ lockdep_assert_held(&iocg->ioc->lock);
1157+
1158+ for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
1159+ struct ioc_gq *parent = iocg->ancestors[lvl];
1160+ struct ioc_gq *child = iocg->ancestors[lvl + 1];
1161+
1162+ child_inuse_sum = parent->child_inuse_sum + inuse - child->inuse;
1163+ hwm = div64_u64((u64)hwm * inuse, child_inuse_sum);
1164+ inuse = DIV64_U64_ROUND_UP(parent->active * child_inuse_sum,
1165+ parent->child_active_sum);
1166+ }
1167+
1168+ return max_t(u32, hwm, 1);
1169+}
1170+
1171+static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now)
10091172 {
10101173 struct ioc *ioc = iocg->ioc;
10111174 struct blkcg_gq *blkg = iocg_to_blkg(iocg);
@@ -1016,16 +1179,15 @@ static void weight_updated(struct ioc_gq *iocg) · 4 unmodified lines
10161179
10171180 weight = iocg->cfg_weight ?: iocc->dfl_weight;
10181181 if (weight != iocg->weight && iocg->active)
1019− propagate_active_weight(iocg, weight,
1020− DIV64_U64_ROUND_UP(iocg->inuse * weight, iocg->weight));
1182+ propagate_weights(iocg, weight, iocg->inuse, true, now);
10211183 iocg->weight = weight;
10221184 }
10231185
10241186 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
10251187 {
10261188 struct ioc *ioc = iocg->ioc;
1027− u64 last_period, cur_period, max_period_delta;
1028− u64 vtime, vmargin, vmin;
1189+ u64 last_period, cur_period;
1190+ u64 vtime, vtarget;
10291191 int i;
10301192
10311193 /*
@@ -1064,22 +1226,15 @@ static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now) · 32 unmodified lines
10641226 goto fail_unlock;
10651227
10661228 /*
1067− * vtime may wrap when vrate is raised substantially due to
1068− * underestimated IO costs. Look at the period and ignore its
1069− * vtime if the iocg has been idle for too long. Also, cap the
1070− * budget it can start with to the margin.
1229+ * Always start with the target budget. On deactivation, we throw away
1230+ * anything above it.
10711231 */
1072− max_period_delta = DIV64_U64_ROUND_UP(VTIME_VALID_DUR, ioc->period_us);
1232+ vtarget = now->vnow - ioc->margins.target;
10731233 vtime = atomic64_read(&iocg->vtime);
1074− vmargin = ioc->margin_us * now->vrate;
1075− vmin = now->vnow - vmargin;
10761234
1077− if (last_period + max_period_delta < cur_period ||
1078− time_before64(vtime, vmin)) {
1079− atomic64_add(vmin - vtime, &iocg->vtime);
1080− atomic64_add(vmin - vtime, &iocg->done_vtime);
1081− vtime = vmin;
1082− }
1235+ atomic64_add(vtarget - vtime, &iocg->vtime);
1236+ atomic64_add(vtarget - vtime, &iocg->done_vtime);
1237+ vtime = vtarget;
10831238
10841239 /*
10851240 * Activate, propagate weight and start period timer if not
@@ -1088,16 +1243,18 @@ static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now) · 2 unmodified lines
10881243 */
10891244 iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;
10901245 list_add(&iocg->active_list, &ioc->active_iocgs);
1091− propagate_active_weight(iocg, iocg->weight,
1092− iocg->last_inuse ?: iocg->weight);
1246+
1247+ propagate_weights(iocg, iocg->weight,
1248+ iocg->last_inuse ?: iocg->weight, true, now);
10931249
10941250 TRACE_IOCG_PATH(iocg_activate, iocg, now,
10951251 last_period, cur_period, vtime);
10961252
1097− iocg->last_vtime = vtime;
1253+ iocg->activated_at = now->now;
10981254
10991255 if (ioc->running == IOC_IDLE) {
11001256 ioc->running = IOC_RUNNING;
1257+ ioc->debt_busy_at = now->now;
11011258 ioc_start_period(ioc, now);
11021259 }
11031260
@@ -1110,6 +1267,110 @@ fail_unlock: · 6 unmodified lines
11101267 return false;
11111268 }
11121269
1270+static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)
1271+{
1272+ struct ioc *ioc = iocg->ioc;
1273+ struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1274+ u64 tdelta, delay, new_delay;
1275+ s64 vover, vover_pct;
1276+ u32 hwa;
1277+
1278+ lockdep_assert_held(&iocg->waitq.lock);
1279+
1280+ /* calculate the current delay in effect - 1/2 every second */
1281+ tdelta = now->now - iocg->delay_at;
1282+ if (iocg->delay)
1283+ delay = iocg->delay >> div64_u64(tdelta, USEC_PER_SEC);
1284+ else
1285+ delay = 0;
1286+
1287+ /* calculate the new delay from the debt amount */
1288+ current_hweight(iocg, &hwa, NULL);
1289+ vover = atomic64_read(&iocg->vtime) +
1290+ abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;
1291+ vover_pct = div64_s64(100 * vover,
1292+ ioc->period_us * ioc->vtime_base_rate);
1293+
1294+ if (vover_pct <= MIN_DELAY_THR_PCT)
1295+ new_delay = 0;
1296+ else if (vover_pct >= MAX_DELAY_THR_PCT)
1297+ new_delay = MAX_DELAY;
1298+ else
1299+ new_delay = MIN_DELAY +
1300+ div_u64((MAX_DELAY - MIN_DELAY) *
1301+ (vover_pct - MIN_DELAY_THR_PCT),
1302+ MAX_DELAY_THR_PCT - MIN_DELAY_THR_PCT);
1303+
1304+ /* pick the higher one and apply */
1305+ if (new_delay > delay) {
1306+ iocg->delay = new_delay;
1307+ iocg->delay_at = now->now;
1308+ delay = new_delay;
1309+ }
1310+
1311+ if (delay >= MIN_DELAY) {
1312+ if (!iocg->indelay_since)
1313+ iocg->indelay_since = now->now;
1314+ blkcg_set_delay(blkg, delay * NSEC_PER_USEC);
1315+ return true;
1316+ } else {
1317+ if (iocg->indelay_since) {
1318+ iocg->local_stat.indelay_us += now->now - iocg->indelay_since;
1319+ iocg->indelay_since = 0;
1320+ }
1321+ iocg->delay = 0;
1322+ blkcg_clear_delay(blkg);
1323+ return false;
1324+ }
1325+}
1326+
1327+static void iocg_incur_debt(struct ioc_gq *iocg, u64 abs_cost,
1328+ struct ioc_now *now)
1329+{
1330+ struct iocg_pcpu_stat *gcs;
1331+
1332+ lockdep_assert_held(&iocg->ioc->lock);
1333+ lockdep_assert_held(&iocg->waitq.lock);
1334+ WARN_ON_ONCE(list_empty(&iocg->active_list));
1335+
1336+ /*
1337+ * Once in debt, debt handling owns inuse. @iocg stays at the minimum
1338+ * inuse donating all of it share to others until its debt is paid off.
1339+ */
1340+ if (!iocg->abs_vdebt && abs_cost) {
1341+ iocg->indebt_since = now->now;
1342+ propagate_weights(iocg, iocg->active, 0, false, now);
1343+ }
1344+
1345+ iocg->abs_vdebt += abs_cost;
1346+
1347+ gcs = get_cpu_ptr(iocg->pcpu_stat);
1348+ local64_add(abs_cost, &gcs->abs_vusage);
1349+ put_cpu_ptr(gcs);
1350+}
1351+
1352+static void iocg_pay_debt(struct ioc_gq *iocg, u64 abs_vpay,
1353+ struct ioc_now *now)
1354+{
1355+ lockdep_assert_held(&iocg->ioc->lock);
1356+ lockdep_assert_held(&iocg->waitq.lock);
1357+
1358+ /* make sure that nobody messed with @iocg */
1359+ WARN_ON_ONCE(list_empty(&iocg->active_list));
1360+ WARN_ON_ONCE(iocg->inuse > 1);
1361+
1362+ iocg->abs_vdebt -= min(abs_vpay, iocg->abs_vdebt);
1363+
1364+ /* if debt is paid in full, restore inuse */
1365+ if (!iocg->abs_vdebt) {
1366+ iocg->local_stat.indebt_us += now->now - iocg->indebt_since;
1367+ iocg->indebt_since = 0;
1368+
1369+ propagate_weights(iocg, iocg->active, iocg->last_inuse,
1370+ false, now);
1371+ }
1372+}
1373+
11131374 static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode,
11141375 int flags, void *key)
11151376 {
@@ -1122,7 +1383,7 @@ static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode, · 6 unmodified lines
11221383 if (ctx->vbudget < 0)
11231384 return -1;
11241385
1125− iocg_commit_bio(ctx->iocg, wait->bio, cost);
1386+ iocg_commit_bio(ctx->iocg, wait->bio, wait->abs_cost, cost);
11261387
11271388 /*
11281389 * autoremove_wake_function() removes the wait entry only when it
@@ -1136,132 +1397,106 @@ static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode, · 7 unmodified lines
11361397 return 0;
11371398 }
11381399
1139−static void iocg_kick_waitq(struct ioc_gq *iocg, struct ioc_now *now)
1400+/*
1401+ * Calculate the accumulated budget, pay debt if @pay_debt and wake up waiters
1402+ * accordingly. When @pay_debt is %true, the caller must be holding ioc->lock in
1403+ * addition to iocg->waitq.lock.
1404+ */
1405+static void iocg_kick_waitq(struct ioc_gq *iocg, bool pay_debt,
1406+ struct ioc_now *now)
11401407 {
11411408 struct ioc *ioc = iocg->ioc;
11421409 struct iocg_wake_ctx ctx = { .iocg = iocg };
1143− u64 margin_ns = (u64)(ioc->period_us *
1144− WAITQ_TIMER_MARGIN_PCT / 100) * NSEC_PER_USEC;
1145− u64 vdebt, vshortage, expires, oexpires;
1410+ u64 vshortage, expires, oexpires;
11461411 s64 vbudget;
1147− u32 hw_inuse;
1412+ u32 hwa;
11481413
11491414 lockdep_assert_held(&iocg->waitq.lock);
11501415
1151− current_hweight(iocg, NULL, &hw_inuse);
1416+ current_hweight(iocg, &hwa, NULL);
11521417 vbudget = now->vnow - atomic64_read(&iocg->vtime);
11531418
11541419 /* pay off debt */
1155− vdebt = abs_cost_to_cost(iocg->abs_vdebt, hw_inuse);
1156− if (vdebt && vbudget > 0) {
1157− u64 delta = min_t(u64, vbudget, vdebt);
1158− u64 abs_delta = min(cost_to_abs_cost(delta, hw_inuse),
1159− iocg->abs_vdebt);
1420+ if (pay_debt && iocg->abs_vdebt && vbudget > 0) {
1421+ u64 abs_vbudget = cost_to_abs_cost(vbudget, hwa);
1422+ u64 abs_vpay = min_t(u64, abs_vbudget, iocg->abs_vdebt);
1423+ u64 vpay = abs_cost_to_cost(abs_vpay, hwa);
1424+
1425+ lockdep_assert_held(&ioc->lock);
1426+
1427+ atomic64_add(vpay, &iocg->vtime);
1428+ atomic64_add(vpay, &iocg->done_vtime);
1429+ iocg_pay_debt(iocg, abs_vpay, now);
1430+ vbudget -= vpay;
1431+ }
1432+
1433+ if (iocg->abs_vdebt || iocg->delay)
1434+ iocg_kick_delay(iocg, now);
11601435
1161− atomic64_add(delta, &iocg->vtime);
1162− atomic64_add(delta, &iocg->done_vtime);
1163− iocg->abs_vdebt -= abs_delta;
1436+ /*
1437+ * Debt can still be outstanding if we haven't paid all yet or the
1438+ * caller raced and called without @pay_debt. Shouldn't wake up waiters
1439+ * under debt. Make sure @vbudget reflects the outstanding amount and is
1440+ * not positive.
1441+ */
1442+ if (iocg->abs_vdebt) {
1443+ s64 vdebt = abs_cost_to_cost(iocg->abs_vdebt, hwa);
1444+ vbudget = min_t(s64, 0, vbudget - vdebt);
11641445 }
11651446
11661447 /*
1167− * Wake up the ones which are due and see how much vtime we'll need
1168− * for the next one.
1448+ * Wake up the ones which are due and see how much vtime we'll need for
1449+ * the next one. As paying off debt restores hw_inuse, it must be read
1450+ * after the above debt payment.
11691451 */
1170− ctx.hw_inuse = hw_inuse;
1171− ctx.vbudget = vbudget - vdebt;
1452+ ctx.vbudget = vbudget;
1453+ current_hweight(iocg, NULL, &ctx.hw_inuse);
1454+
11721455 __wake_up_locked_key(&iocg->waitq, TASK_NORMAL, &ctx);
1173− if (!waitqueue_active(&iocg->waitq))
1456+
1457+ if (!waitqueue_active(&iocg->waitq)) {
1458+ if (iocg->wait_since) {
1459+ iocg->local_stat.wait_us += now->now - iocg->wait_since;
1460+ iocg->wait_since = 0;
1461+ }
11741462 return;
1463+ }
1464+
1465+ if (!iocg->wait_since)
1466+ iocg->wait_since = now->now;
1467+
11751468 if (WARN_ON_ONCE(ctx.vbudget >= 0))
11761469 return;
11771470
1178− /* determine next wakeup, add a quarter margin to guarantee chunking */
1471+ /* determine next wakeup, add a timer margin to guarantee chunking */
11791472 vshortage = -ctx.vbudget;
11801473 expires = now->now_ns +
1181− DIV64_U64_ROUND_UP(vshortage, now->vrate) * NSEC_PER_USEC;
1182− expires += margin_ns / 4;
1474+ DIV64_U64_ROUND_UP(vshortage, ioc->vtime_base_rate) *
1475+ NSEC_PER_USEC;
1476+ expires += ioc->timer_slack_ns;
11831477
11841478 /* if already active and close enough, don't bother */
11851479 oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->waitq_timer));
11861480 if (hrtimer_is_queued(&iocg->waitq_timer) &&
1187− abs(oexpires - expires) <= margin_ns / 4)
1481+ abs(oexpires - expires) <= ioc->timer_slack_ns)
11881482 return;
11891483
11901484 hrtimer_start_range_ns(&iocg->waitq_timer, ns_to_ktime(expires),
1191− margin_ns / 4, HRTIMER_MODE_ABS);
1485+ ioc->timer_slack_ns, HRTIMER_MODE_ABS);
11921486 }
11931487
11941488 static enum hrtimer_restart iocg_waitq_timer_fn(struct hrtimer *timer)
11951489 {
11961490 struct ioc_gq *iocg = container_of(timer, struct ioc_gq, waitq_timer);
1491+ bool pay_debt = READ_ONCE(iocg->abs_vdebt);
11971492 struct ioc_now now;
11981493 unsigned long flags;
11991494
12001495 ioc_now(iocg->ioc, &now);
12011496
1202− spin_lock_irqsave(&iocg->waitq.lock, flags);
1203− iocg_kick_waitq(iocg, &now);
1204− spin_unlock_irqrestore(&iocg->waitq.lock, flags);
1205−
1206− return HRTIMER_NORESTART;
1207−}
1208−
1209−static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)
1210−{
1211− struct ioc *ioc = iocg->ioc;
1212− struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1213− u64 vtime = atomic64_read(&iocg->vtime);
1214− u64 vmargin = ioc->margin_us * now->vrate;
1215− u64 margin_ns = ioc->margin_us * NSEC_PER_USEC;
1216− u64 delta_ns, expires, oexpires;
1217− u32 hw_inuse;
1218−
1219− lockdep_assert_held(&iocg->waitq.lock);
1220−
1221− /* debt-adjust vtime */
1222− current_hweight(iocg, NULL, &hw_inuse);
1223− vtime += abs_cost_to_cost(iocg->abs_vdebt, hw_inuse);
1224−
1225− /*
1226− * Clear or maintain depending on the overage. Non-zero vdebt is what
1227− * guarantees that @iocg is online and future iocg_kick_delay() will
1228− * clear use_delay. Don't leave it on when there's no vdebt.
1229− */
1230− if (!iocg->abs_vdebt || time_before_eq64(vtime, now->vnow)) {
1231− blkcg_clear_delay(blkg);
1232− return false;
1233− }
1234− if (!atomic_read(&blkg->use_delay) &&
1235− time_before_eq64(vtime, now->vnow + vmargin))
1236− return false;
1237−
1238− /* use delay */
1239− delta_ns = DIV64_U64_ROUND_UP(vtime - now->vnow,
1240− now->vrate) * NSEC_PER_USEC;
1241− blkcg_set_delay(blkg, delta_ns);
1242− expires = now->now_ns + delta_ns;
1243−
1244− /* if already active and close enough, don't bother */
1245− oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->delay_timer));
1246− if (hrtimer_is_queued(&iocg->delay_timer) &&
1247− abs(oexpires - expires) <= margin_ns / 4)
1248− return true;
1249−
1250− hrtimer_start_range_ns(&iocg->delay_timer, ns_to_ktime(expires),
1251− margin_ns / 4, HRTIMER_MODE_ABS);
1252− return true;
1253−}
1254−
1255−static enum hrtimer_restart iocg_delay_timer_fn(struct hrtimer *timer)
1256−{
1257− struct ioc_gq *iocg = container_of(timer, struct ioc_gq, delay_timer);
1258− struct ioc_now now;
1259− unsigned long flags;
1260−
1261− spin_lock_irqsave(&iocg->waitq.lock, flags);
1262− ioc_now(iocg->ioc, &now);
1263− iocg_kick_delay(iocg, &now);
1264− spin_unlock_irqrestore(&iocg->waitq.lock, flags);
1497+ iocg_lock(iocg, pay_debt, &flags);
1498+ iocg_kick_waitq(iocg, pay_debt, &now);
1499+ iocg_unlock(iocg, pay_debt, &flags);
12651500
12661501 return HRTIMER_NORESTART;
12671502 }
@@ -1278,8 +1513,8 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p · 10 unmodified lines
12781513 u64 this_rq_wait_ns;
12791514
12801515 for (rw = READ; rw <= WRITE; rw++) {
1281− u32 this_met = READ_ONCE(stat->missed[rw].nr_met);
1282− u32 this_missed = READ_ONCE(stat->missed[rw].nr_missed);
1516+ u32 this_met = local_read(&stat->missed[rw].nr_met);
1517+ u32 this_missed = local_read(&stat->missed[rw].nr_missed);
12831518
12841519 nr_met[rw] += this_met - stat->missed[rw].last_met;
12851520 nr_missed[rw] += this_missed - stat->missed[rw].last_missed;
@@ -1287,7 +1522,7 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p · 1 unmodified line
12871522 stat->missed[rw].last_missed = this_missed;
12881523 }
12891524
1290− this_rq_wait_ns = READ_ONCE(stat->rq_wait_ns);
1525+ this_rq_wait_ns = local64_read(&stat->rq_wait_ns);
12911526 rq_wait_ns += this_rq_wait_ns - stat->last_rq_wait_ns;
12921527 stat->last_rq_wait_ns = this_rq_wait_ns;
12931528 }
@@ -1322,18 +1557,426 @@ static bool iocg_is_idle(struct ioc_gq *iocg) · 28 unmodified lines
13221557 return true;
13231558 }
13241559
1325−/* returns usage with margin added if surplus is large enough */
1326−static u32 surplus_adjusted_hweight_inuse(u32 usage, u32 hw_inuse)
1560+/*
1561+ * Call this function on the target leaf @iocg's to build pre-order traversal
1562+ * list of all the ancestors in @inner_walk. The inner nodes are linked through
1563+ * ->walk_list and the caller is responsible for dissolving the list after use.
1564+ */
1565+static void iocg_build_inner_walk(struct ioc_gq *iocg,
1566+ struct list_head *inner_walk)
13271567 {
1328− /* add margin */
1329− usage = DIV_ROUND_UP(usage * SURPLUS_SCALE_PCT, 100);
1330− usage += SURPLUS_SCALE_ABS;
1568+ int lvl;
13311569
1332− /* don't bother if the surplus is too small */
1333− if (usage + SURPLUS_MIN_ADJ_DELTA > hw_inuse)
1334− return 0;
1570+ WARN_ON_ONCE(!list_empty(&iocg->walk_list));
1571+
1572+ /* find the first ancestor which hasn't been visited yet */
1573+ for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
1574+ if (!list_empty(&iocg->ancestors[lvl]->walk_list))
1575+ break;
1576+ }
1577+
1578+ /* walk down and visit the inner nodes to get pre-order traversal */
1579+ while (++lvl <= iocg->level - 1) {
1580+ struct ioc_gq *inner = iocg->ancestors[lvl];
1581+
1582+ /* record traversal order */
1583+ list_add_tail(&inner->walk_list, inner_walk);
1584+ }
1585+}
1586+
1587+/* collect per-cpu counters and propagate the deltas to the parent */
1588+static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now)
1589+{
1590+ struct ioc *ioc = iocg->ioc;
1591+ struct iocg_stat new_stat;
1592+ u64 abs_vusage = 0;
1593+ u64 vusage_delta;
1594+ int cpu;
1595+
1596+ lockdep_assert_held(&iocg->ioc->lock);
1597+
1598+ /* collect per-cpu counters */
1599+ for_each_possible_cpu(cpu) {
1600+ abs_vusage += local64_read(
1601+ per_cpu_ptr(&iocg->pcpu_stat->abs_vusage, cpu));
1602+ }
1603+ vusage_delta = abs_vusage - iocg->last_stat_abs_vusage;
1604+ iocg->last_stat_abs_vusage = abs_vusage;
1605+
1606+ iocg->usage_delta_us = div64_u64(vusage_delta, ioc->vtime_base_rate);
1607+ iocg->local_stat.usage_us += iocg->usage_delta_us;
1608+
1609+ /* propagate upwards */
1610+ new_stat.usage_us =
1611+ iocg->local_stat.usage_us + iocg->desc_stat.usage_us;
1612+ new_stat.wait_us =
1613+ iocg->local_stat.wait_us + iocg->desc_stat.wait_us;
1614+ new_stat.indebt_us =
1615+ iocg->local_stat.indebt_us + iocg->desc_stat.indebt_us;
1616+ new_stat.indelay_us =
1617+ iocg->local_stat.indelay_us + iocg->desc_stat.indelay_us;
1618+
1619+ /* propagate the deltas to the parent */
1620+ if (iocg->level > 0) {
1621+ struct iocg_stat *parent_stat =
1622+ &iocg->ancestors[iocg->level - 1]->desc_stat;
1623+
1624+ parent_stat->usage_us +=
1625+ new_stat.usage_us - iocg->last_stat.usage_us;
1626+ parent_stat->wait_us +=
1627+ new_stat.wait_us - iocg->last_stat.wait_us;
1628+ parent_stat->indebt_us +=
1629+ new_stat.indebt_us - iocg->last_stat.indebt_us;
1630+ parent_stat->indelay_us +=
1631+ new_stat.indelay_us - iocg->last_stat.indelay_us;
1632+ }
1633+
1634+ iocg->last_stat = new_stat;
1635+}
13351636
1336− return usage;
1637+/* get stat counters ready for reading on all active iocgs */
1638+static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)
1639+{
1640+ LIST_HEAD(inner_walk);
1641+ struct ioc_gq *iocg, *tiocg;
1642+
1643+ /* flush leaves and build inner node walk list */
1644+ list_for_each_entry(iocg, target_iocgs, active_list) {
1645+ iocg_flush_stat_one(iocg, now);
1646+ iocg_build_inner_walk(iocg, &inner_walk);
1647+ }
1648+
1649+ /* keep flushing upwards by walking the inner list backwards */
1650+ list_for_each_entry_safe_reverse(iocg, tiocg, &inner_walk, walk_list) {
1651+ iocg_flush_stat_one(iocg, now);
1652+ list_del_init(&iocg->walk_list);
1653+ }
1654+}
1655+
1656+/*
1657+ * Determine what @iocg's hweight_inuse should be after donating unused
1658+ * capacity. @hwm is the upper bound and used to signal no donation. This
1659+ * function also throws away @iocg's excess budget.
1660+ */
1661+static u32 hweight_after_donation(struct ioc_gq *iocg, u32 old_hwi, u32 hwm,
1662+ u32 usage, struct ioc_now *now)
1663+{
1664+ struct ioc *ioc = iocg->ioc;
1665+ u64 vtime = atomic64_read(&iocg->vtime);
1666+ s64 excess, delta, target, new_hwi;
1667+
1668+ /* debt handling owns inuse for debtors */
1669+ if (iocg->abs_vdebt)
1670+ return 1;
1671+
1672+ /* see whether minimum margin requirement is met */
1673+ if (waitqueue_active(&iocg->waitq) ||
1674+ time_after64(vtime, now->vnow - ioc->margins.min))
1675+ return hwm;
1676+
1677+ /* throw away excess above target */
1678+ excess = now->vnow - vtime - ioc->margins.target;
1679+ if (excess > 0) {
1680+ atomic64_add(excess, &iocg->vtime);
1681+ atomic64_add(excess, &iocg->done_vtime);
1682+ vtime += excess;
1683+ ioc->vtime_err -= div64_u64(excess * old_hwi, WEIGHT_ONE);
1684+ }
1685+
1686+ /*
1687+ * Let's say the distance between iocg's and device's vtimes as a
1688+ * fraction of period duration is delta. Assuming that the iocg will
1689+ * consume the usage determined above, we want to determine new_hwi so
1690+ * that delta equals MARGIN_TARGET at the end of the next period.
1691+ *
1692+ * We need to execute usage worth of IOs while spending the sum of the
1693+ * new budget (1 - MARGIN_TARGET) and the leftover from the last period
1694+ * (delta):
1695+ *
1696+ * usage = (1 - MARGIN_TARGET + delta) * new_hwi
1697+ *
1698+ * Therefore, the new_hwi is:
1699+ *
1700+ * new_hwi = usage / (1 - MARGIN_TARGET + delta)
1701+ */
1702+ delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),
1703+ now->vnow - ioc->period_at_vtime);
1704+ target = WEIGHT_ONE * MARGIN_TARGET_PCT / 100;
1705+ new_hwi = div64_s64(WEIGHT_ONE * usage, WEIGHT_ONE - target + delta);
1706+
1707+ return clamp_t(s64, new_hwi, 1, hwm);
1708+}
1709+
1710+/*
1711+ * For work-conservation, an iocg which isn't using all of its share should
1712+ * donate the leftover to other iocgs. There are two ways to achieve this - 1.
1713+ * bumping up vrate accordingly 2. lowering the donating iocg's inuse weight.
1714+ *
1715+ * #1 is mathematically simpler but has the drawback of requiring synchronous
1716+ * global hweight_inuse updates when idle iocg's get activated or inuse weights
1717+ * change due to donation snapbacks as it has the possibility of grossly
1718+ * overshooting what's allowed by the model and vrate.
1719+ *
1720+ * #2 is inherently safe with local operations. The donating iocg can easily
1721+ * snap back to higher weights when needed without worrying about impacts on
1722+ * other nodes as the impacts will be inherently correct. This also makes idle
1723+ * iocg activations safe. The only effect activations have is decreasing
1724+ * hweight_inuse of others, the right solution to which is for those iocgs to
1725+ * snap back to higher weights.
1726+ *
1727+ * So, we go with #2. The challenge is calculating how each donating iocg's
1728+ * inuse should be adjusted to achieve the target donation amounts. This is done
1729+ * using Andy's method described in the following pdf.
1730+ *
1731+ * https://drive.google.com/file/d/1PsJwxPFtjUnwOY1QJ5AeICCcsL7BM3bo
1732+ *
1733+ * Given the weights and target after-donation hweight_inuse values, Andy's
1734+ * method determines how the proportional distribution should look like at each
1735+ * sibling level to maintain the relative relationship between all non-donating
1736+ * pairs. To roughly summarize, it divides the tree into donating and
1737+ * non-donating parts, calculates global donation rate which is used to
1738+ * determine the target hweight_inuse for each node, and then derives per-level
1739+ * proportions.
1740+ *
1741+ * The following pdf shows that global distribution calculated this way can be
1742+ * achieved by scaling inuse weights of donating leaves and propagating the
1743+ * adjustments upwards proportionally.
1744+ *
1745+ * https://drive.google.com/file/d/1vONz1-fzVO7oY5DXXsLjSxEtYYQbOvsE
1746+ *
1747+ * Combining the above two, we can determine how each leaf iocg's inuse should
1748+ * be adjusted to achieve the target donation.
1749+ *
1750+ * https://drive.google.com/file/d/1WcrltBOSPN0qXVdBgnKm4mdp9FhuEFQN
1751+ *
1752+ * The inline comments use symbols from the last pdf.
1753+ *
1754+ * b is the sum of the absolute budgets in the subtree. 1 for the root node.
1755+ * f is the sum of the absolute budgets of non-donating nodes in the subtree.
1756+ * t is the sum of the absolute budgets of donating nodes in the subtree.
1757+ * w is the weight of the node. w = w_f + w_t
1758+ * w_f is the non-donating portion of w. w_f = w * f / b
1759+ * w_b is the donating portion of w. w_t = w * t / b
1760+ * s is the sum of all sibling weights. s = Sum(w) for siblings
1761+ * s_f and s_t are the non-donating and donating portions of s.
1762+ *
1763+ * Subscript p denotes the parent's counterpart and ' the adjusted value - e.g.
1764+ * w_pt is the donating portion of the parent's weight and w'_pt the same value
1765+ * after adjustments. Subscript r denotes the root node's values.
1766+ */
1767+static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)
1768+{
1769+ LIST_HEAD(over_hwa);
1770+ LIST_HEAD(inner_walk);
1771+ struct ioc_gq *iocg, *tiocg, *root_iocg;
1772+ u32 after_sum, over_sum, over_target, gamma;
1773+
1774+ /*
1775+ * It's pretty unlikely but possible for the total sum of
1776+ * hweight_after_donation's to be higher than WEIGHT_ONE, which will
1777+ * confuse the following calculations. If such condition is detected,
1778+ * scale down everyone over its full share equally to keep the sum below
1779+ * WEIGHT_ONE.
1780+ */
1781+ after_sum = 0;
1782+ over_sum = 0;
1783+ list_for_each_entry(iocg, surpluses, surplus_list) {
1784+ u32 hwa;
1785+
1786+ current_hweight(iocg, &hwa, NULL);
1787+ after_sum += iocg->hweight_after_donation;
1788+
1789+ if (iocg->hweight_after_donation > hwa) {
1790+ over_sum += iocg->hweight_after_donation;
1791+ list_add(&iocg->walk_list, &over_hwa);
1792+ }
1793+ }
1794+
1795+ if (after_sum >= WEIGHT_ONE) {
1796+ /*
1797+ * The delta should be deducted from the over_sum, calculate
1798+ * target over_sum value.
1799+ */
1800+ u32 over_delta = after_sum - (WEIGHT_ONE - 1);
1801+ WARN_ON_ONCE(over_sum <= over_delta);
1802+ over_target = over_sum - over_delta;
1803+ } else {
1804+ over_target = 0;
1805+ }
1806+
1807+ list_for_each_entry_safe(iocg, tiocg, &over_hwa, walk_list) {
1808+ if (over_target)
1809+ iocg->hweight_after_donation =
1810+ div_u64((u64)iocg->hweight_after_donation *
1811+ over_target, over_sum);
1812+ list_del_init(&iocg->walk_list);
1813+ }
1814+
1815+ /*
1816+ * Build pre-order inner node walk list and prepare for donation
1817+ * adjustment calculations.
1818+ */
1819+ list_for_each_entry(iocg, surpluses, surplus_list) {
1820+ iocg_build_inner_walk(iocg, &inner_walk);
1821+ }
1822+
1823+ root_iocg = list_first_entry(&inner_walk, struct ioc_gq, walk_list);
1824+ WARN_ON_ONCE(root_iocg->level > 0);
1825+
1826+ list_for_each_entry(iocg, &inner_walk, walk_list) {
1827+ iocg->child_adjusted_sum = 0;
1828+ iocg->hweight_donating = 0;
1829+ iocg->hweight_after_donation = 0;
1830+ }
1831+
1832+ /*
1833+ * Propagate the donating budget (b_t) and after donation budget (b'_t)
1834+ * up the hierarchy.
1835+ */
1836+ list_for_each_entry(iocg, surpluses, surplus_list) {
1837+ struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1838+
1839+ parent->hweight_donating += iocg->hweight_donating;
1840+ parent->hweight_after_donation += iocg->hweight_after_donation;
1841+ }
1842+
1843+ list_for_each_entry_reverse(iocg, &inner_walk, walk_list) {
1844+ if (iocg->level > 0) {
1845+ struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1846+
1847+ parent->hweight_donating += iocg->hweight_donating;
1848+ parent->hweight_after_donation += iocg->hweight_after_donation;
1849+ }
1850+ }
1851+
1852+ /*
1853+ * Calculate inner hwa's (b) and make sure the donation values are
1854+ * within the accepted ranges as we're doing low res calculations with
1855+ * roundups.
1856+ */
1857+ list_for_each_entry(iocg, &inner_walk, walk_list) {
1858+ if (iocg->level) {
1859+ struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1860+
1861+ iocg->hweight_active = DIV64_U64_ROUND_UP(
1862+ (u64)parent->hweight_active * iocg->active,
1863+ parent->child_active_sum);
1864+
1865+ }
1866+
1867+ iocg->hweight_donating = min(iocg->hweight_donating,
1868+ iocg->hweight_active);
1869+ iocg->hweight_after_donation = min(iocg->hweight_after_donation,
1870+ iocg->hweight_donating - 1);
1871+ if (WARN_ON_ONCE(iocg->hweight_active <= 1 ||
1872+ iocg->hweight_donating <= 1 ||
1873+ iocg->hweight_after_donation == 0)) {
1874+ pr_warn("iocg: invalid donation weights in ");
1875+ pr_cont_cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup);
1876+ pr_cont(": active=%u donating=%u after=%u\n",
1877+ iocg->hweight_active, iocg->hweight_donating,
1878+ iocg->hweight_after_donation);
1879+ }
1880+ }
1881+
1882+ /*
1883+ * Calculate the global donation rate (gamma) - the rate to adjust
1884+ * non-donating budgets by.
1885+ *
1886+ * No need to use 64bit multiplication here as the first operand is
1887+ * guaranteed to be smaller than WEIGHT_ONE (1<<16).
1888+ *
1889+ * We know that there are beneficiary nodes and the sum of the donating
1890+ * hweights can't be whole; however, due to the round-ups during hweight
1891+ * calculations, root_iocg->hweight_donating might still end up equal to
1892+ * or greater than whole. Limit the range when calculating the divider.
1893+ *
1894+ * gamma = (1 - t_r') / (1 - t_r)
1895+ */
1896+ gamma = DIV_ROUND_UP(
1897+ (WEIGHT_ONE - root_iocg->hweight_after_donation) * WEIGHT_ONE,
1898+ WEIGHT_ONE - min_t(u32, root_iocg->hweight_donating, WEIGHT_ONE - 1));
1899+
1900+ /*
1901+ * Calculate adjusted hwi, child_adjusted_sum and inuse for the inner
1902+ * nodes.
1903+ */
1904+ list_for_each_entry(iocg, &inner_walk, walk_list) {
1905+ struct ioc_gq *parent;
1906+ u32 inuse, wpt, wptp;
1907+ u64 st, sf;
1908+
1909+ if (iocg->level == 0) {
1910+ /* adjusted weight sum for 1st level: s' = s * b_pf / b'_pf */
1911+ iocg->child_adjusted_sum = DIV64_U64_ROUND_UP(
1912+ iocg->child_active_sum * (WEIGHT_ONE - iocg->hweight_donating),
1913+ WEIGHT_ONE - iocg->hweight_after_donation);
1914+ continue;
1915+ }
1916+
1917+ parent = iocg->ancestors[iocg->level - 1];
1918+
1919+ /* b' = gamma * b_f + b_t' */
1920+ iocg->hweight_inuse = DIV64_U64_ROUND_UP(
1921+ (u64)gamma * (iocg->hweight_active - iocg->hweight_donating),
1922+ WEIGHT_ONE) + iocg->hweight_after_donation;
1923+
1924+ /* w' = s' * b' / b'_p */
1925+ inuse = DIV64_U64_ROUND_UP(
1926+ (u64)parent->child_adjusted_sum * iocg->hweight_inuse,
1927+ parent->hweight_inuse);
1928+
1929+ /* adjusted weight sum for children: s' = s_f + s_t * w'_pt / w_pt */
1930+ st = DIV64_U64_ROUND_UP(
1931+ iocg->child_active_sum * iocg->hweight_donating,
1932+ iocg->hweight_active);
1933+ sf = iocg->child_active_sum - st;
1934+ wpt = DIV64_U64_ROUND_UP(
1935+ (u64)iocg->active * iocg->hweight_donating,
1936+ iocg->hweight_active);
1937+ wptp = DIV64_U64_ROUND_UP(
1938+ (u64)inuse * iocg->hweight_after_donation,
1939+ iocg->hweight_inuse);
1940+
1941+ iocg->child_adjusted_sum = sf + DIV64_U64_ROUND_UP(st * wptp, wpt);
1942+ }
1943+
1944+ /*
1945+ * All inner nodes now have ->hweight_inuse and ->child_adjusted_sum and
1946+ * we can finally determine leaf adjustments.
1947+ */
1948+ list_for_each_entry(iocg, surpluses, surplus_list) {
1949+ struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
1950+ u32 inuse;
1951+
1952+ /*
1953+ * In-debt iocgs participated in the donation calculation with
1954+ * the minimum target hweight_inuse. Configuring inuse
1955+ * accordingly would work fine but debt handling expects
1956+ * @iocg->inuse stay at the minimum and we don't wanna
1957+ * interfere.
1958+ */
1959+ if (iocg->abs_vdebt) {
1960+ WARN_ON_ONCE(iocg->inuse > 1);
1961+ continue;
1962+ }
1963+
1964+ /* w' = s' * b' / b'_p, note that b' == b'_t for donating leaves */
1965+ inuse = DIV64_U64_ROUND_UP(
1966+ parent->child_adjusted_sum * iocg->hweight_after_donation,
1967+ parent->hweight_inuse);
1968+
1969+ TRACE_IOCG_PATH(inuse_transfer, iocg, now,
1970+ iocg->inuse, inuse,
1971+ iocg->hweight_inuse,
1972+ iocg->hweight_after_donation);
1973+
1974+ __propagate_weights(iocg, iocg->active, inuse, true, now);
1975+ }
1976+
1977+ /* walk list should be dissolved after use */
1978+ list_for_each_entry_safe(iocg, tiocg, &inner_walk, walk_list)
1979+ list_del_init(&iocg->walk_list);
13371980 }
13381981
13391982 static void ioc_timer_fn(struct timer_list *timer)
@@ -1341,12 +1984,14 @@ static void ioc_timer_fn(struct timer_list *timer) · 1 unmodified line
13411984 struct ioc *ioc = container_of(timer, struct ioc, timer);
13421985 struct ioc_gq *iocg, *tiocg;
13431986 struct ioc_now now;
1344− int nr_surpluses = 0, nr_shortages = 0, nr_lagging = 0;
1987+ LIST_HEAD(surpluses);
1988+ int nr_debtors = 0, nr_shortages = 0, nr_lagging = 0;
1989+ u64 usage_us_sum = 0;
13451990 u32 ppm_rthr = MILLION - ioc->params.qos[QOS_RPPM];
13461991 u32 ppm_wthr = MILLION - ioc->params.qos[QOS_WPPM];
13471992 u32 missed_ppm[2], rq_wait_pct;
13481993 u64 period_vtime;
1349− int prev_busy_level, i;
1994+ int prev_busy_level;
13501995
13511996 /* how were the latencies during the period? */
13521997 ioc_lat_stat(ioc, missed_ppm, &rq_wait_pct);
@@ -1370,30 +2015,71 @@ static void ioc_timer_fn(struct timer_list *timer) · 17 unmodified lines
13702015 */
13712016 list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {
13722017 if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
1373− !iocg_is_idle(iocg))
2018+ !iocg->delay && !iocg_is_idle(iocg))
13742019 continue;
13752020
13762021 spin_lock(&iocg->waitq.lock);
13772022
1378− if (waitqueue_active(&iocg->waitq) || iocg->abs_vdebt) {
2023+ /* flush wait and indebt stat deltas */
2024+ if (iocg->wait_since) {
2025+ iocg->local_stat.wait_us += now.now - iocg->wait_since;
2026+ iocg->wait_since = now.now;
2027+ }
2028+ if (iocg->indebt_since) {
2029+ iocg->local_stat.indebt_us +=
2030+ now.now - iocg->indebt_since;
2031+ iocg->indebt_since = now.now;
2032+ }
2033+ if (iocg->indelay_since) {
2034+ iocg->local_stat.indelay_us +=
2035+ now.now - iocg->indelay_since;
2036+ iocg->indelay_since = now.now;
2037+ }
2038+
2039+ if (waitqueue_active(&iocg->waitq) || iocg->abs_vdebt ||
2040+ iocg->delay) {
13792041 /* might be oversleeping vtime / hweight changes, kick */
1380− iocg_kick_waitq(iocg, &now);
1381− iocg_kick_delay(iocg, &now);
2042+ iocg_kick_waitq(iocg, true, &now);
2043+ if (iocg->abs_vdebt)
2044+ nr_debtors++;
13822045 } else if (iocg_is_idle(iocg)) {
13832046 /* no waiter and idle, deactivate */
1384− iocg->last_inuse = iocg->inuse;
1385− __propagate_active_weight(iocg, 0, 0);
2047+ u64 vtime = atomic64_read(&iocg->vtime);
2048+ s64 excess;
2049+
2050+ /*
2051+ * @iocg has been inactive for a full duration and will
2052+ * have a high budget. Account anything above target as
2053+ * error and throw away. On reactivation, it'll start
2054+ * with the target budget.
2055+ */
2056+ excess = now.vnow - vtime - ioc->margins.target;
2057+ if (excess > 0) {
2058+ u32 old_hwi;
2059+
2060+ current_hweight(iocg, NULL, &old_hwi);
2061+ ioc->vtime_err -= div64_u64(excess * old_hwi,
2062+ WEIGHT_ONE);
2063+ }
2064+
2065+ __propagate_weights(iocg, 0, 0, false, &now);
13862066 list_del_init(&iocg->active_list);
13872067 }
13882068
13892069 spin_unlock(&iocg->waitq.lock);
13902070 }
1391− commit_active_weights(ioc);
2071+ commit_weights(ioc);
2072+
2073+ /*
2074+ * Wait and indebt stat are flushed above and the donation calculation
2075+ * below needs updated usage stat. Let's bring stat up-to-date.
2076+ */
2077+ iocg_flush_stat(&ioc->active_iocgs, &now);
13922078
1393− /* calc usages and see whether some weights need to be moved around */
2079+ /* calc usage and see whether some weights need to be moved around */
13942080 list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
1395− u64 vdone, vtime, vusage, vmargin, vmin;
1396− u32 hw_active, hw_inuse, usage;
2081+ u64 vdone, vtime, usage_us, usage_dur;
2082+ u32 usage, hw_active, hw_inuse;
13972083
13982084 /*
13992085 * Collect unused and wind vtime closer to vnow to prevent
@@ -1417,116 +2103,105 @@ static void ioc_timer_fn(struct timer_list *timer) · 17 unmodified lines
14172103 time_before64(vdone, now.vnow - period_vtime))
14182104 nr_lagging++;
14192105
1420− if (waitqueue_active(&iocg->waitq))
1421− vusage = now.vnow - iocg->last_vtime;
1422− else if (time_before64(iocg->last_vtime, vtime))
1423− vusage = vtime - iocg->last_vtime;
1424− else
1425− vusage = 0;
1426−
1427− iocg->last_vtime += vusage;
14282106 /*
1429− * Factor in in-flight vtime into vusage to avoid
1430− * high-latency completions appearing as idle. This should
1431− * be done after the above ->last_time adjustment.
2107+ * Determine absolute usage factoring in in-flight IOs to avoid
2108+ * high-latency completions appearing as idle.
14322109 */
1433− vusage = max(vusage, vtime - vdone);
1434−
1435− /* calculate hweight based usage ratio and record */
1436− if (vusage) {
1437− usage = DIV64_U64_ROUND_UP(vusage * hw_inuse,
1438− period_vtime);
1439− iocg->usage_idx = (iocg->usage_idx + 1) % NR_USAGE_SLOTS;
1440− iocg->usages[iocg->usage_idx] = usage;
1441− } else {
1442− usage = 0;
2110+ usage_us = iocg->usage_delta_us;
2111+ usage_us_sum += usage_us;
2112+
2113+ if (vdone != vtime) {
2114+ u64 inflight_us = DIV64_U64_ROUND_UP(
2115+ cost_to_abs_cost(vtime - vdone, hw_inuse),
2116+ ioc->vtime_base_rate);
2117+ usage_us = max(usage_us, inflight_us);
14432118 }
14442119
2120+ /* convert to hweight based usage ratio */
2121+ if (time_after64(iocg->activated_at, ioc->period_at))
2122+ usage_dur = max_t(u64, now.now - iocg->activated_at, 1);
2123+ else
2124+ usage_dur = max_t(u64, now.now - ioc->period_at, 1);
2125+
2126+ usage = clamp_t(u32,
2127+ DIV64_U64_ROUND_UP(usage_us * WEIGHT_ONE,
2128+ usage_dur),
2129+ 1, WEIGHT_ONE);
2130+
14452131 /* see whether there's surplus vtime */
1446− vmargin = ioc->margin_us * now.vrate;
1447− vmin = now.vnow - vmargin;
1448−
1449− iocg->has_surplus = false;
1450−
1451− if (!waitqueue_active(&iocg->waitq) &&
1452− time_before64(vtime, vmin)) {
1453− u64 delta = vmin - vtime;
1454−
1455− /* throw away surplus vtime */
1456− atomic64_add(delta, &iocg->vtime);
1457− atomic64_add(delta, &iocg->done_vtime);
1458− iocg->last_vtime += delta;
1459− /* if usage is sufficiently low, maybe it can donate */
1460− if (surplus_adjusted_hweight_inuse(usage, hw_inuse)) {
1461− iocg->has_surplus = true;
1462− nr_surpluses++;
1463− }
1464− } else if (hw_inuse < hw_active) {
1465− u32 new_hwi, new_inuse;
2132+ WARN_ON_ONCE(!list_empty(&iocg->surplus_list));
2133+ if (hw_inuse < hw_active ||
2134+ (!waitqueue_active(&iocg->waitq) &&
2135+ time_before64(vtime, now.vnow - ioc->margins.low))) {
2136+ u32 hwa, old_hwi, hwm, new_hwi;
14662137
1467− /* was donating but might need to take back some */
1468− if (waitqueue_active(&iocg->waitq)) {
1469− new_hwi = hw_active;
2138+ /*
2139+ * Already donating or accumulated enough to start.
2140+ * Determine the donation amount.
2141+ */
2142+ current_hweight(iocg, &hwa, &old_hwi);
2143+ hwm = current_hweight_max(iocg);
2144+ new_hwi = hweight_after_donation(iocg, old_hwi, hwm,
2145+ usage, &now);
2146+ if (new_hwi < hwm) {
2147+ iocg->hweight_donating = hwa;
2148+ iocg->hweight_after_donation = new_hwi;
2149+ list_add(&iocg->surplus_list, &surpluses);
14702150 } else {
1471− new_hwi = max(hw_inuse,
1472− usage * SURPLUS_SCALE_PCT / 100 +
1473− SURPLUS_SCALE_ABS);
1474− }
2151+ TRACE_IOCG_PATH(inuse_shortage, iocg, &now,
2152+ iocg->inuse, iocg->active,
2153+ iocg->hweight_inuse, new_hwi);
14752154
1476− new_inuse = div64_u64((u64)iocg->inuse * new_hwi,
1477− hw_inuse);
1478− new_inuse = clamp_t(u32, new_inuse, 1, iocg->active);
1479−
1480− if (new_inuse > iocg->inuse) {
1481− TRACE_IOCG_PATH(inuse_takeback, iocg, &now,
1482− iocg->inuse, new_inuse,
1483− hw_inuse, new_hwi);
1484− __propagate_active_weight(iocg, iocg->weight,
1485− new_inuse);
2155+ __propagate_weights(iocg, iocg->active,
2156+ iocg->active, true, &now);
2157+ nr_shortages++;
14862158 }
14872159 } else {
1488− /* genuninely out of vtime */
2160+ /* genuinely short on vtime */
14892161 nr_shortages++;
14902162 }
14912163 }
14922164
1493− if (!nr_shortages || !nr_surpluses)
1494− goto skip_surplus_transfers;
2165+ if (!list_empty(&surpluses) && nr_shortages)
2166+ transfer_surpluses(&surpluses, &now);
14952167
1496− /* there are both shortages and surpluses, transfer surpluses */
1497− list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
1498− u32 usage, hw_active, hw_inuse, new_hwi, new_inuse;
1499− int nr_valid = 0;
2168+ commit_weights(ioc);
15002169
1501− if (!iocg->has_surplus)
1502− continue;
2170+ /* surplus list should be dissolved after use */
2171+ list_for_each_entry_safe(iocg, tiocg, &surpluses, surplus_list)
2172+ list_del_init(&iocg->surplus_list);
15032173
1504− /* base the decision on max historical usage */
1505− for (i = 0, usage = 0; i < NR_USAGE_SLOTS; i++) {
1506− if (iocg->usages[i]) {
1507− usage = max(usage, iocg->usages[i]);
1508− nr_valid++;
2174+ /*
2175+ * A low weight iocg can amass a large amount of debt, for example, when
2176+ * anonymous memory gets reclaimed aggressively. If the system has a lot
2177+ * of memory paired with a slow IO device, the debt can span multiple
2178+ * seconds or more. If there are no other subsequent IO issuers, the
2179+ * in-debt iocg may end up blocked paying its debt while the IO device
2180+ * is idle.
2181+ *
2182+ * The following protects against such pathological cases. If the device
2183+ * has been sufficiently idle for a substantial amount of time, the
2184+ * debts are halved. The criteria are on the conservative side as we
2185+ * want to resolve the rare extreme cases without impacting regular
2186+ * operation by forgiving debts too readily.
2187+ */
2188+ if (nr_shortages ||
2189+ div64_u64(100 * usage_us_sum, now.now - ioc->period_at) >=
2190+ DEBT_BUSY_USAGE_PCT)
2191+ ioc->debt_busy_at = now.now;
2192+
2193+ if (nr_debtors &&
2194+ now.now - ioc->debt_busy_at >= DEBT_REDUCTION_IDLE_DUR) {
2195+ list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
2196+ if (iocg->abs_vdebt) {
2197+ spin_lock(&iocg->waitq.lock);
2198+ iocg->abs_vdebt /= 2;
2199+ iocg_kick_waitq(iocg, true, &now);
2200+ spin_unlock(&iocg->waitq.lock);
15092201 }
15102202 }
1511− if (nr_valid < MIN_VALID_USAGES)
1512− continue;
1513−
1514− current_hweight(iocg, &hw_active, &hw_inuse);
1515− new_hwi = surplus_adjusted_hweight_inuse(usage, hw_inuse);
1516− if (!new_hwi)
1517− continue;
1518−
1519− new_inuse = DIV64_U64_ROUND_UP((u64)iocg->inuse * new_hwi,
1520− hw_inuse);
1521− if (new_inuse < iocg->inuse) {
1522− TRACE_IOCG_PATH(inuse_giveaway, iocg, &now,
1523− iocg->inuse, new_inuse,
1524− hw_inuse, new_hwi);
1525− __propagate_active_weight(iocg, iocg->weight, new_inuse);
1526− }
2203+ ioc->debt_busy_at = now.now;
15272204 }
1528−skip_surplus_transfers:
1529− commit_active_weights(ioc);
15302205
15312206 /*
15322207 * If q is getting clogged or we're missing too much, we're issuing
@@ -1554,11 +2229,9 @@ skip_surplus_transfers: · 21 unmodified lines
15542229
15552230 /*
15562231 * If there are IOs spanning multiple periods, wait
1557− * them out before pushing the device harder. If
1558− * there are surpluses, let redistribution work it
1559− * out first.
2232+ * them out before pushing the device harder.
15602233 */
1561− if (!nr_lagging && !nr_surpluses)
2234+ if (!nr_lagging)
15622235 ioc->busy_level--;
15632236 } else {
15642237 /*
@@ -1577,7 +2250,7 @@ skip_surplus_transfers: · 12 unmodified lines
15772250 ioc->busy_level = clamp(ioc->busy_level, -1000, 1000);
15782251
15792252 if (ioc->busy_level > 0 || (ioc->busy_level < 0 && !nr_lagging)) {
1580− u64 vrate = atomic64_read(&ioc->vtime_rate);
2253+ u64 vrate = ioc->vtime_base_rate;
15812254 u64 vrate_min = ioc->vrate_min, vrate_max = ioc->vrate_max;
15822255
15832256 /* rq_wait signal is always reliable, ignore user vrate_min */
@@ -1612,16 +2285,14 @@ skip_surplus_transfers: · 28 unmodified lines
16122285 }
16132286
16142287 trace_iocost_ioc_vrate_adj(ioc, vrate, missed_ppm, rq_wait_pct,
1615− nr_lagging, nr_shortages,
1616− nr_surpluses);
2288+ nr_lagging, nr_shortages);
16172289
1618− atomic64_set(&ioc->vtime_rate, vrate);
1619− ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(
1620− ioc->period_us * vrate * INUSE_MARGIN_PCT, 100);
2290+ ioc->vtime_base_rate = vrate;
2291+ ioc_refresh_margins(ioc);
16212292 } else if (ioc->busy_level != prev_busy_level || nr_lagging) {
16222293 trace_iocost_ioc_vrate_adj(ioc, atomic64_read(&ioc->vtime_rate),
16232294 missed_ppm, rq_wait_pct, nr_lagging,
1624− nr_shortages, nr_surpluses);
2295+ nr_shortages);
16252296 }
16262297
16272298 ioc_refresh_params(ioc, false);
@@ -1637,11 +2308,74 @@ skip_surplus_transfers: · 9 unmodified lines
16372308 ioc_start_period(ioc, &now);
16382309 } else {
16392310 ioc->busy_level = 0;
2311+ ioc->vtime_err = 0;
16402312 ioc->running = IOC_IDLE;
16412313 }
2314+
2315+ ioc_refresh_vrate(ioc, &now);
2316+ }
2317+
2318+ spin_unlock_irq(&ioc->lock);
2319+}
2320+
2321+static u64 adjust_inuse_and_calc_cost(struct ioc_gq *iocg, u64 vtime,
2322+ u64 abs_cost, struct ioc_now *now)
2323+{
2324+ struct ioc *ioc = iocg->ioc;
2325+ struct ioc_margins *margins = &ioc->margins;
2326+ u32 __maybe_unused old_inuse = iocg->inuse, __maybe_unused old_hwi;
2327+ u32 hwi, adj_step;
2328+ s64 margin;
2329+ u64 cost, new_inuse;
2330+
2331+ current_hweight(iocg, NULL, &hwi);
2332+ old_hwi = hwi;
2333+ cost = abs_cost_to_cost(abs_cost, hwi);
2334+ margin = now->vnow - vtime - cost;
2335+
2336+ /* debt handling owns inuse for debtors */
2337+ if (iocg->abs_vdebt)
2338+ return cost;
2339+
2340+ /*
2341+ * We only increase inuse during period and do so iff the margin has
2342+ * deteriorated since the previous adjustment.
2343+ */
2344+ if (margin >= iocg->saved_margin || margin >= margins->low ||
2345+ iocg->inuse == iocg->active)
2346+ return cost;
2347+
2348+ spin_lock_irq(&ioc->lock);
2349+
2350+ /* we own inuse only when @iocg is in the normal active state */
2351+ if (iocg->abs_vdebt || list_empty(&iocg->active_list)) {
2352+ spin_unlock_irq(&ioc->lock);
2353+ return cost;
16422354 }
16432355
2356+ /*
2357+ * Bump up inuse till @abs_cost fits in the existing budget.
2358+ * adj_step must be determined after acquiring ioc->lock - we might
2359+ * have raced and lost to another thread for activation and could
2360+ * be reading 0 iocg->active before ioc->lock which will lead to
2361+ * infinite loop.
2362+ */
2363+ new_inuse = iocg->inuse;
2364+ adj_step = DIV_ROUND_UP(iocg->active * INUSE_ADJ_STEP_PCT, 100);
2365+ do {
2366+ new_inuse = new_inuse + adj_step;
2367+ propagate_weights(iocg, iocg->active, new_inuse, true, now);
2368+ current_hweight(iocg, NULL, &hwi);
2369+ cost = abs_cost_to_cost(abs_cost, hwi);
2370+ } while (time_after64(vtime + cost, now->vnow) &&
2371+ iocg->inuse != iocg->active);
2372+
16442373 spin_unlock_irq(&ioc->lock);
2374+
2375+ TRACE_IOCG_PATH(inuse_adjust, iocg, now,
2376+ old_inuse, iocg->inuse, old_hwi, hwi);
2377+
2378+ return cost;
16452379 }
16462380
16472381 static void calc_vtime_cost_builtin(struct bio *bio, struct ioc_gq *iocg,
@@ -1725,38 +2459,25 @@ static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio) · 77 unmodified lines
17252459 struct ioc_gq *iocg = blkg_to_iocg(blkg);
17262460 struct ioc_now now;
17272461 struct iocg_wait wait;
1728− u32 hw_active, hw_inuse;
17292462 u64 abs_cost, cost, vtime;
2463+ bool use_debt, ioc_locked;
2464+ unsigned long flags;
17302465
17312466 /* bypass IOs if disabled or for root cgroup */
17322467 if (!ioc->enabled || !iocg->level)
17332468 return;
17342469
1735− /* always activate so that even 0 cost IOs get protected to some level */
1736− if (!iocg_activate(iocg, &now))
1737− return;
1738−
17392470 /* calculate the absolute vtime cost */
17402471 abs_cost = calc_vtime_cost(bio, iocg, false);
17412472 if (!abs_cost)
17422473 return;
17432474
1744− iocg->cursor = bio_end_sector(bio);
2475+ if (!iocg_activate(iocg, &now))
2476+ return;
17452477
2478+ iocg->cursor = bio_end_sector(bio);
17462479 vtime = atomic64_read(&iocg->vtime);
1747− current_hweight(iocg, &hw_active, &hw_inuse);
1748−
1749− if (hw_inuse < hw_active &&
1750− time_after_eq64(vtime + ioc->inuse_margin_vtime, now.vnow)) {
1751− TRACE_IOCG_PATH(inuse_reset, iocg, &now,
1752− iocg->inuse, iocg->weight, hw_inuse, hw_active);
1753− spin_lock_irq(&ioc->lock);
1754− propagate_active_weight(iocg, iocg->weight, iocg->weight);
1755− spin_unlock_irq(&ioc->lock);
1756− current_hweight(iocg, &hw_active, &hw_inuse);
1757− }
1758−
1759− cost = abs_cost_to_cost(abs_cost, hw_inuse);
2480+ cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
17602481
17612482 /*
17622483 * If no one's waiting and within budget, issue right away. The
@@ -1765,21 +2486,32 @@ static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio) · 2 unmodified lines
17652486 */
17662487 if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
17672488 time_before_eq64(vtime + cost, now.vnow)) {
1768− iocg_commit_bio(iocg, bio, cost);
2489+ iocg_commit_bio(iocg, bio, abs_cost, cost);
17692490 return;
17702491 }
17712492
17722493 /*
1773− * We activated above but w/o any synchronization. Deactivation is
1774− * synchronized with waitq.lock and we won't get deactivated as long
1775− * as we're waiting or has debt, so we're good if we're activated
1776− * here. In the unlikely case that we aren't, just issue the IO.
2494+ * We're over budget. This can be handled in two ways. IOs which may
2495+ * cause priority inversions are punted to @ioc->aux_iocg and charged as
2496+ * debt. Otherwise, the issuer is blocked on @iocg->waitq. Debt handling
2497+ * requires @ioc->lock, waitq handling @iocg->waitq.lock. Determine
2498+ * whether debt handling is needed and acquire locks accordingly.
17772499 */
1778− spin_lock_irq(&iocg->waitq.lock);
2500+ use_debt = bio_issue_as_root_blkg(bio) || fatal_signal_pending(current);
2501+ ioc_locked = use_debt || READ_ONCE(iocg->abs_vdebt);
2502+retry_lock:
2503+ iocg_lock(iocg, ioc_locked, &flags);
17792504
2505+ /*
2506+ * @iocg must stay activated for debt and waitq handling. Deactivation
2507+ * is synchronized against both ioc->lock and waitq.lock and we won't
2508+ * get deactivated as long as we're waiting or has debt, so we're good
2509+ * if we're activated here. In the unlikely cases that we aren't, just
2510+ * issue the IO.
2511+ */
17802512 if (unlikely(list_empty(&iocg->active_list))) {
1781− spin_unlock_irq(&iocg->waitq.lock);
1782− iocg_commit_bio(iocg, bio, cost);
2513+ iocg_unlock(iocg, ioc_locked, &flags);
2514+ iocg_commit_bio(iocg, bio, abs_cost, cost);
17832515 return;
17842516 }
17852517
@@ -1800,15 +2532,26 @@ static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio) · 14 unmodified lines
18002532 * clear them and leave @iocg inactive w/ dangling use_delay heavily
18012533 * penalizing the cgroup and its descendants.
18022534 */
1803− if (bio_issue_as_root_blkg(bio) || fatal_signal_pending(current)) {
1804− iocg->abs_vdebt += abs_cost;
2535+ if (use_debt) {
2536+ iocg_incur_debt(iocg, abs_cost, &now);
18052537 if (iocg_kick_delay(iocg, &now))
18062538 blkcg_schedule_throttle(rqos->q,
18072539 (bio->bi_opf & REQ_SWAP) == REQ_SWAP);
1808− spin_unlock_irq(&iocg->waitq.lock);
2540+ iocg_unlock(iocg, ioc_locked, &flags);
18092541 return;
18102542 }
18112543
2544+ /* guarantee that iocgs w/ waiters have maximum inuse */
2545+ if (!iocg->abs_vdebt && iocg->inuse != iocg->active) {
2546+ if (!ioc_locked) {
2547+ iocg_unlock(iocg, false, &flags);
2548+ ioc_locked = true;
2549+ goto retry_lock;
2550+ }
2551+ propagate_weights(iocg, iocg->active, iocg->active, true,
2552+ &now);
2553+ }
2554+
18122555 /*
18132556 * Append self to the waitq and schedule the wakeup timer if we're
18142557 * the first waiter. The timer duration is calculated based on the
@@ -1829,9 +2572,9 @@ static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio) · 14 unmodified lines
18292572 wait.committed = false; /* will be set true by waker */
18302573
18312574 __add_wait_queue_entry_tail(&iocg->waitq, &wait.wait);
1832− iocg_kick_waitq(iocg, &now);
2575+ iocg_kick_waitq(iocg, ioc_locked, &now);
18332576
1834− spin_unlock_irq(&iocg->waitq.lock);
2577+ iocg_unlock(iocg, ioc_locked, &flags);
18352578
18362579 while (true) {
18372580 set_current_state(TASK_UNINTERRUPTIBLE);
@@ -1851,8 +2594,7 @@ static void ioc_rqos_merge(struct rq_qos *rqos, struct request *rq, · 13 unmodified lines
18512594 struct ioc *ioc = iocg->ioc;
18522595 sector_t bio_end = bio_end_sector(bio);
18532596 struct ioc_now now;
1854− u32 hw_inuse;
1855− u64 abs_cost, cost;
2597+ u64 vtime, abs_cost, cost;
18562598 unsigned long flags;
18572599
18582600 /* bypass if disabled or for root cgroup */
@@ -1864,8 +2606,9 @@ static void ioc_rqos_merge(struct rq_qos *rqos, struct request *rq, · 5 unmodified lines
18642606 return;
18652607
18662608 ioc_now(ioc, &now);
1867− current_hweight(iocg, NULL, &hw_inuse);
1868− cost = abs_cost_to_cost(abs_cost, hw_inuse);
2609+
2610+ vtime = atomic64_read(&iocg->vtime);
2611+ cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
18692612
18702613 /* update cursor if backmerging into the request at the cursor */
18712614 if (blk_rq_pos(rq) < bio_end &&
@@ -1878,7 +2621,7 @@ static void ioc_rqos_merge(struct rq_qos *rqos, struct request *rq, · 6 unmodified lines
18782621 */
18792622 if (rq->bio && rq->bio->bi_iocost_cost &&
18802623 time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {
1881− iocg_commit_bio(iocg, bio, cost);
2624+ iocg_commit_bio(iocg, bio, abs_cost, cost);
18822625 return;
18832626 }
18842627
@@ -1887,14 +2630,20 @@ static void ioc_rqos_merge(struct rq_qos *rqos, struct request *rq, · 2 unmodified lines
18872630 * be for the vast majority of cases. See debt handling in
18882631 * ioc_rqos_throttle() for details.
18892632 */
1890− spin_lock_irqsave(&iocg->waitq.lock, flags);
2633+ spin_lock_irqsave(&ioc->lock, flags);
2634+ spin_lock(&iocg->waitq.lock);
2635+
18912636 if (likely(!list_empty(&iocg->active_list))) {
1892− iocg->abs_vdebt += abs_cost;
1893− iocg_kick_delay(iocg, &now);
2637+ iocg_incur_debt(iocg, abs_cost, &now);
2638+ if (iocg_kick_delay(iocg, &now))
2639+ blkcg_schedule_throttle(rqos->q,
2640+ (bio->bi_opf & REQ_SWAP) == REQ_SWAP);
18942641 } else {
1895− iocg_commit_bio(iocg, bio, cost);
2642+ iocg_commit_bio(iocg, bio, abs_cost, cost);
18962643 }
1897− spin_unlock_irqrestore(&iocg->waitq.lock, flags);
2644+
2645+ spin_unlock(&iocg->waitq.lock);
2646+ spin_unlock_irqrestore(&ioc->lock, flags);
18982647 }
18992648
19002649 static void ioc_rqos_done_bio(struct rq_qos *rqos, struct bio *bio)
@@ -1908,6 +2657,7 @@ static void ioc_rqos_done_bio(struct rq_qos *rqos, struct bio *bio) · 7 unmodified lines
19082657 static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
19092658 {
19102659 struct ioc *ioc = rqos_to_ioc(rqos);
2660+ struct ioc_pcpu_stat *ccs;
19112661 u64 on_q_ns, rq_wait_ns, size_nsec;
19122662 int pidx, rw;
19132663
@@ -1931,13 +2681,17 @@ static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq) · 17 unmodified lines
19312681 rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;
19322682 size_nsec = div64_u64(calc_size_vtime_cost(rq, ioc), VTIME_PER_NSEC);
19332683
2684+ ccs = get_cpu_ptr(ioc->pcpu_stat);
2685+
19342686 if (on_q_ns <= size_nsec ||
19352687 on_q_ns - size_nsec <= ioc->params.qos[pidx] * NSEC_PER_USEC)
1936− this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);
2688+ local_inc(&ccs->missed[rw].nr_met);
19372689 else
1938− this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);
2690+ local_inc(&ccs->missed[rw].nr_missed);
2691+
2692+ local64_add(rq_wait_ns, &ccs->rq_wait_ns);
19392693
1940− this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);
2694+ put_cpu_ptr(ccs);
19412695 }
19422696
19432697 static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)
@@ -1977,7 +2731,7 @@ static int blk_iocost_init(struct request_queue *q) · 33 unmodified lines
19772731 {
19782732 struct ioc *ioc;
19792733 struct rq_qos *rqos;
1980− int ret;
2734+ int i, cpu, ret;
19812735
19822736 ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
19832737 if (!ioc)
@@ -1989,6 +2743,16 @@ static int blk_iocost_init(struct request_queue *q) · 5 unmodified lines
19892743 return -ENOMEM;
19902744 }
19912745
2746+ for_each_possible_cpu(cpu) {
2747+ struct ioc_pcpu_stat *ccs = per_cpu_ptr(ioc->pcpu_stat, cpu);
2748+
2749+ for (i = 0; i < ARRAY_SIZE(ccs->missed); i++) {
2750+ local_set(&ccs->missed[i].nr_met, 0);
2751+ local_set(&ccs->missed[i].nr_missed, 0);
2752+ }
2753+ local64_set(&ccs->rq_wait_ns, 0);
2754+ }
2755+
19922756 rqos = &ioc->rqos;
19932757 rqos->id = RQ_QOS_COST;
19942758 rqos->ops = &ioc_rqos_ops;
@@ -1999,6 +2763,7 @@ static int blk_iocost_init(struct request_queue *q) · 4 unmodified lines
19992763 INIT_LIST_HEAD(&ioc->active_iocgs);
20002764
20012765 ioc->running = IOC_IDLE;
2766+ ioc->vtime_base_rate = VTIME_PER_USEC;
20022767 atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);
20032768 seqcount_spinlock_init(&ioc->period_seqcount, &ioc->lock);
20042769 ioc->period_at = ktime_to_us(ktime_get());
@@ -2029,7 +2794,7 @@ static struct blkcg_policy_data *ioc_cpd_alloc(gfp_t gfp) · 24 unmodified lines
20292794 if (!iocc)
20302795 return NULL;
20312796
2032− iocc->dfl_weight = CGROUP_WEIGHT_DFL;
2797+ iocc->dfl_weight = CGROUP_WEIGHT_DFL * WEIGHT_ONE;
20332798 return &iocc->cpd;
20342799 }
20352800
@@ -2048,6 +2813,12 @@ static struct blkg_policy_data *ioc_pd_alloc(gfp_t gfp, struct request_queue *q, · 12 unmodified lines
20482813 if (!iocg)
20492814 return NULL;
20502815
2816+ iocg->pcpu_stat = alloc_percpu_gfp(struct iocg_pcpu_stat, gfp);
2817+ if (!iocg->pcpu_stat) {
2818+ kfree(iocg);
2819+ return NULL;
2820+ }
2821+
20512822 return &iocg->pd;
20522823 }
20532824
@@ -2067,14 +2838,14 @@ static void ioc_pd_init(struct blkg_policy_data *pd) · 13 unmodified lines
20672838 atomic64_set(&iocg->done_vtime, now.vnow);
20682839 atomic64_set(&iocg->active_period, atomic64_read(&ioc->cur_period));
20692840 INIT_LIST_HEAD(&iocg->active_list);
2070− iocg->hweight_active = HWEIGHT_WHOLE;
2071− iocg->hweight_inuse = HWEIGHT_WHOLE;
2841+ INIT_LIST_HEAD(&iocg->walk_list);
2842+ INIT_LIST_HEAD(&iocg->surplus_list);
2843+ iocg->hweight_active = WEIGHT_ONE;
2844+ iocg->hweight_inuse = WEIGHT_ONE;
20722845
20732846 init_waitqueue_head(&iocg->waitq);
20742847 hrtimer_init(&iocg->waitq_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
20752848 iocg->waitq_timer.function = iocg_waitq_timer_fn;
2076− hrtimer_init(&iocg->delay_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2077− iocg->delay_timer.function = iocg_delay_timer_fn;
20782849
20792850 iocg->level = blkg->blkcg->css.cgroup->level;
20802851
@@ -2084,7 +2855,7 @@ static void ioc_pd_init(struct blkg_policy_data *pd) · 3 unmodified lines
20842855 }
20852856
20862857 spin_lock_irqsave(&ioc->lock, flags);
2087− weight_updated(iocg);
2858+ weight_updated(iocg, &now);
20882859 spin_unlock_irqrestore(&ioc->lock, flags);
20892860 }
20902861
@@ -2096,18 +2867,56 @@ static void ioc_pd_free(struct blkg_policy_data *pd) · 5 unmodified lines
20962867
20972868 if (ioc) {
20982869 spin_lock_irqsave(&ioc->lock, flags);
2870+
20992871 if (!list_empty(&iocg->active_list)) {
2100− propagate_active_weight(iocg, 0, 0);
2872+ struct ioc_now now;
2873+
2874+ ioc_now(ioc, &now);
2875+ propagate_weights(iocg, 0, 0, false, &now);
21012876 list_del_init(&iocg->active_list);
21022877 }
2878+
2879+ WARN_ON_ONCE(!list_empty(&iocg->walk_list));
2880+ WARN_ON_ONCE(!list_empty(&iocg->surplus_list));
2881+
21032882 spin_unlock_irqrestore(&ioc->lock, flags);
21042883
21052884 hrtimer_cancel(&iocg->waitq_timer);
2106− hrtimer_cancel(&iocg->delay_timer);
21072885 }
2886+ free_percpu(iocg->pcpu_stat);
21082887 kfree(iocg);
21092888 }
21102889
2890+static size_t ioc_pd_stat(struct blkg_policy_data *pd, char *buf, size_t size)
2891+{
2892+ struct ioc_gq *iocg = pd_to_iocg(pd);
2893+ struct ioc *ioc = iocg->ioc;
2894+ size_t pos = 0;
2895+
2896+ if (!ioc->enabled)
2897+ return 0;
2898+
2899+ if (iocg->level == 0) {
2900+ unsigned vp10k = DIV64_U64_ROUND_CLOSEST(
2901+ ioc->vtime_base_rate * 10000,
2902+ VTIME_PER_USEC);
2903+ pos += scnprintf(buf + pos, size - pos, " cost.vrate=%u.%02u",
2904+ vp10k / 100, vp10k % 100);
2905+ }
2906+
2907+ pos += scnprintf(buf + pos, size - pos, " cost.usage=%llu",
2908+ iocg->last_stat.usage_us);
2909+
2910+ if (blkcg_debug_stats)
2911+ pos += scnprintf(buf + pos, size - pos,
2912+ " cost.wait=%llu cost.indebt=%llu cost.indelay=%llu",
2913+ iocg->last_stat.wait_us,
2914+ iocg->last_stat.indebt_us,
2915+ iocg->last_stat.indelay_us);
2916+
2917+ return pos;
2918+}
2919+
21112920 static u64 ioc_weight_prfill(struct seq_file *sf, struct blkg_policy_data *pd,
21122921 int off)
21132922 {
@@ -2115,7 +2924,7 @@ static u64 ioc_weight_prfill(struct seq_file *sf, struct blkg_policy_data *pd, · 1 unmodified line
21152924 struct ioc_gq *iocg = pd_to_iocg(pd);
21162925
21172926 if (dname && iocg->cfg_weight)
2118− seq_printf(sf, "%s %u\n", dname, iocg->cfg_weight);
2927+ seq_printf(sf, "%s %u\n", dname, iocg->cfg_weight / WEIGHT_ONE);
21192928 return 0;
21202929 }
21212930
@@ -2125,7 +2934,7 @@ static int ioc_weight_show(struct seq_file *sf, void *v) · 3 unmodified lines
21252934 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
21262935 struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
21272936
2128− seq_printf(sf, "default %u\n", iocc->dfl_weight);
2937+ seq_printf(sf, "default %u\n", iocc->dfl_weight / WEIGHT_ONE);
21292938 blkcg_print_blkgs(sf, blkcg, ioc_weight_prfill,
21302939 &blkcg_policy_iocost, seq_cft(sf)->private, false);
21312940 return 0;
@@ -2137,6 +2946,7 @@ static ssize_t ioc_weight_write(struct kernfs_open_file *of, char *buf, · 5 unmodified lines
21372946 struct blkcg *blkcg = css_to_blkcg(of_css(of));
21382947 struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
21392948 struct blkg_conf_ctx ctx;
2949+ struct ioc_now now;
21402950 struct ioc_gq *iocg;
21412951 u32 v;
21422952 int ret;
@@ -2151,13 +2961,14 @@ static ssize_t ioc_weight_write(struct kernfs_open_file *of, char *buf, · 8 unmodified lines
21512961 return -EINVAL;
21522962
21532963 spin_lock(&blkcg->lock);
2154− iocc->dfl_weight = v;
2964+ iocc->dfl_weight = v * WEIGHT_ONE;
21552965 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
21562966 struct ioc_gq *iocg = blkg_to_iocg(blkg);
21572967
21582968 if (iocg) {
21592969 spin_lock_irq(&iocg->ioc->lock);
2160− weight_updated(iocg);
2970+ ioc_now(iocg->ioc, &now);
2971+ weight_updated(iocg, &now);
21612972 spin_unlock_irq(&iocg->ioc->lock);
21622973 }
21632974 }
@@ -2182,8 +2993,9 @@ static ssize_t ioc_weight_write(struct kernfs_open_file *of, char *buf, · 18 unmodified lines
21822993 }
21832994
21842995 spin_lock(&iocg->ioc->lock);
2185− iocg->cfg_weight = v;
2186− weight_updated(iocg);
2996+ iocg->cfg_weight = v * WEIGHT_ONE;
2997+ ioc_now(iocg->ioc, &now);
2998+ weight_updated(iocg, &now);
21872999 spin_unlock(&iocg->ioc->lock);
21883000
21893001 blkg_conf_finish(&ctx);
@@ -2521,6 +3333,7 @@ static struct blkcg_policy blkcg_policy_iocost = { · 331 unmodified lines
25213333 .pd_alloc_fn = ioc_pd_alloc,
25223334 .pd_init_fn = ioc_pd_init,
25233335 .pd_free_fn = ioc_pd_free,
3336+ .pd_stat_fn = ioc_pd_stat,
25243337 };
25253338
25263339 static int __init ioc_init(void)
block/blk-map.c+66−111
@@ -12,7 +12,8 @@ · 11 unmodified lines
1212 #include "blk.h"
1313
1414 struct bio_map_data {
15− int is_our_pages;
15+ bool is_our_pages : 1;
16+ bool is_null_mapped : 1;
1617 struct iov_iter iter;
1718 struct iovec iov[];
1819 };
@@ -108,7 +109,7 @@ static int bio_uncopy_user(struct bio *bio) · 89 unmodified lines
108109 struct bio_map_data *bmd = bio->bi_private;
109110 int ret = 0;
110111
111− if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
112+ if (!bmd->is_null_mapped) {
112113 /*
113114 * if we're in a workqueue, the request is orphaned, so
114115 * don't copy into a random user address space, just free
@@ -126,24 +127,12 @@ static int bio_uncopy_user(struct bio *bio) · 11 unmodified lines
126127 return ret;
127128 }
128129
129−/**
130− * bio_copy_user_iov - copy user data to bio
131− * @q: destination block queue
132− * @map_data: pointer to the rq_map_data holding pages (if necessary)
133− * @iter: iovec iterator
134− * @gfp_mask: memory allocation flags
135− *
136− * Prepares and returns a bio for indirect user io, bouncing data
137− * to/from kernel pages as necessary. Must be paired with
138− * call bio_uncopy_user() on io completion.
139− */
140−static struct bio *bio_copy_user_iov(struct request_queue *q,
141− struct rq_map_data *map_data, struct iov_iter *iter,
142− gfp_t gfp_mask)
130+static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data,
131+ struct iov_iter *iter, gfp_t gfp_mask)
143132 {
144133 struct bio_map_data *bmd;
145134 struct page *page;
146− struct bio *bio;
135+ struct bio *bio, *bounce_bio;
147136 int i = 0, ret;
148137 int nr_pages;
149138 unsigned int len = iter->count;
@@ -151,14 +140,15 @@ static struct bio *bio_copy_user_iov(struct request_queue *q, · 1 unmodified line
151140
152141 bmd = bio_alloc_map_data(iter, gfp_mask);
153142 if (!bmd)
154− return ERR_PTR(-ENOMEM);
143+ return -ENOMEM;
155144
156145 /*
157146 * We need to do a deep copy of the iov_iter including the iovecs.
158147 * The caller provided iov might point to an on-stack or otherwise
159148 * shortlived one.
160149 */
161− bmd->is_our_pages = map_data ? 0 : 1;
150+ bmd->is_our_pages = !map_data;
151+ bmd->is_null_mapped = (map_data && map_data->null_mapped);
162152
163153 nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);
164154 if (nr_pages > BIO_MAX_PAGES)
@@ -168,8 +158,7 @@ static struct bio *bio_copy_user_iov(struct request_queue *q, · 3 unmodified lines
168158 bio = bio_kmalloc(gfp_mask, nr_pages);
169159 if (!bio)
170160 goto out_bmd;
171−
172− ret = 0;
161+ bio->bi_opf |= req_op(rq);
173162
174163 if (map_data) {
175164 nr_pages = 1 << map_data->page_order;
@@ -186,7 +175,7 @@ static struct bio *bio_copy_user_iov(struct request_queue *q, · 10 unmodified lines
186175 if (map_data) {
187176 if (i == map_data->nr_entries * nr_pages) {
188177 ret = -ENOMEM;
189− break;
178+ goto cleanup;
190179 }
191180
192181 page = map_data->pages[i / nr_pages];
@@ -194,14 +183,14 @@ static struct bio *bio_copy_user_iov(struct request_queue *q, · 1 unmodified line
194183
195184 i++;
196185 } else {
197− page = alloc_page(q->bounce_gfp | gfp_mask);
186+ page = alloc_page(rq->q->bounce_gfp | gfp_mask);
198187 if (!page) {
199188 ret = -ENOMEM;
200− break;
189+ goto cleanup;
201190 }
202191 }
203192
204− if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) {
193+ if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) {
205194 if (!map_data)
206195 __free_page(page);
207196 break;
@@ -211,9 +200,6 @@ static struct bio *bio_copy_user_iov(struct request_queue *q, · 3 unmodified lines
211200 offset = 0;
212201 }
213202
214− if (ret)
215− goto cleanup;
216−
217203 if (map_data)
218204 map_data->offset += bio->bi_iter.bi_size;
219205
@@ -233,41 +219,42 @@ static struct bio *bio_copy_user_iov(struct request_queue *q, · 13 unmodified lines
233219 }
234220
235221 bio->bi_private = bmd;
236− if (map_data && map_data->null_mapped)
237− bio_set_flag(bio, BIO_NULL_MAPPED);
238− return bio;
222+
223+ bounce_bio = bio;
224+ ret = blk_rq_append_bio(rq, &bounce_bio);
225+ if (ret)
226+ goto cleanup;
227+
228+ /*
229+ * We link the bounce buffer in and could have to traverse it later, so
230+ * we have to get a ref to prevent it from being freed
231+ */
232+ bio_get(bounce_bio);
233+ return 0;
239234 cleanup:
240235 if (!map_data)
241236 bio_free_pages(bio);
242237 bio_put(bio);
243238 out_bmd:
244239 kfree(bmd);
245− return ERR_PTR(ret);
240+ return ret;
246241 }
247242
248−/**
249− * bio_map_user_iov - map user iovec into bio
250− * @q: the struct request_queue for the bio
251− * @iter: iovec iterator
252− * @gfp_mask: memory allocation flags
253− *
254− * Map the user space address into a bio suitable for io to a block
255− * device. Returns an error pointer in case of error.
256− */
257−static struct bio *bio_map_user_iov(struct request_queue *q,
258− struct iov_iter *iter, gfp_t gfp_mask)
243+static int bio_map_user_iov(struct request *rq, struct iov_iter *iter,
244+ gfp_t gfp_mask)
259245 {
260− unsigned int max_sectors = queue_max_hw_sectors(q);
261− int j;
262− struct bio *bio;
246+ unsigned int max_sectors = queue_max_hw_sectors(rq->q);
247+ struct bio *bio, *bounce_bio;
263248 int ret;
249+ int j;
264250
265251 if (!iov_iter_count(iter))
266− return ERR_PTR(-EINVAL);
252+ return -EINVAL;
267253
268254 bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES));
269255 if (!bio)
270− return ERR_PTR(-ENOMEM);
256+ return -ENOMEM;
257+ bio->bi_opf |= req_op(rq);
271258
272259 while (iov_iter_count(iter)) {
273260 struct page **pages;
@@ -283,7 +270,7 @@ static struct bio *bio_map_user_iov(struct request_queue *q, · 9 unmodified lines
283270
284271 npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE);
285272
286− if (unlikely(offs & queue_dma_alignment(q))) {
273+ if (unlikely(offs & queue_dma_alignment(rq->q))) {
287274 ret = -EINVAL;
288275 j = 0;
289276 } else {
@@ -295,7 +282,7 @@ static struct bio *bio_map_user_iov(struct request_queue *q, · 5 unmodified lines
295282 if (n > bytes)
296283 n = bytes;
297284
298− if (!bio_add_hw_page(q, bio, page, n, offs,
285+ if (!bio_add_hw_page(rq->q, bio, page, n, offs,
299286 max_sectors, &same_page)) {
300287 if (same_page)
301288 put_page(page);
@@ -319,21 +306,31 @@ static struct bio *bio_map_user_iov(struct request_queue *q, · 17 unmodified lines
319306 break;
320307 }
321308
322− bio_set_flag(bio, BIO_USER_MAPPED);
323−
324309 /*
325− * subtle -- if bio_map_user_iov() ended up bouncing a bio,
326− * it would normally disappear when its bi_end_io is run.
327− * however, we need it for the unmap, so grab an extra
328− * reference to it
310+ * Subtle: if we end up needing to bounce a bio, it would normally
311+ * disappear when its bi_end_io is run. However, we need the original
312+ * bio for the unmap, so grab an extra reference to it
329313 */
330314 bio_get(bio);
331− return bio;
332315
316+ bounce_bio = bio;
317+ ret = blk_rq_append_bio(rq, &bounce_bio);
318+ if (ret)
319+ goto out_put_orig;
320+
321+ /*
322+ * We link the bounce buffer in and could have to traverse it
323+ * later, so we have to get a ref to prevent it from being freed
324+ */
325+ bio_get(bounce_bio);
326+ return 0;
327+
328+ out_put_orig:
329+ bio_put(bio);
333330 out_unmap:
334331 bio_release_pages(bio, false);
335332 bio_put(bio);
336− return ERR_PTR(ret);
333+ return ret;
337334 }
338335
339336 /**
@@ -557,55 +554,6 @@ int blk_rq_append_bio(struct request *rq, struct bio **bio) · 217 unmodified lines
557554 }
558555 EXPORT_SYMBOL(blk_rq_append_bio);
559556
560−static int __blk_rq_unmap_user(struct bio *bio)
561−{
562− int ret = 0;
563−
564− if (bio) {
565− if (bio_flagged(bio, BIO_USER_MAPPED))
566− bio_unmap_user(bio);
567− else
568− ret = bio_uncopy_user(bio);
569− }
570−
571− return ret;
572−}
573−
574−static int __blk_rq_map_user_iov(struct request *rq,
575− struct rq_map_data *map_data, struct iov_iter *iter,
576− gfp_t gfp_mask, bool copy)
577−{
578− struct request_queue *q = rq->q;
579− struct bio *bio, *orig_bio;
580− int ret;
581−
582− if (copy)
583− bio = bio_copy_user_iov(q, map_data, iter, gfp_mask);
584− else
585− bio = bio_map_user_iov(q, iter, gfp_mask);
586−
587− if (IS_ERR(bio))
588− return PTR_ERR(bio);
589−
590− bio->bi_opf &= ~REQ_OP_MASK;
591− bio->bi_opf |= req_op(rq);
592−
593− orig_bio = bio;
594−
595− /*
596− * We link the bounce buffer in and could have to traverse it
597− * later so we have to get a ref to prevent it from being freed
598− */
599− ret = blk_rq_append_bio(rq, &bio);
600− if (ret) {
601− __blk_rq_unmap_user(orig_bio);
602− return ret;
603− }
604− bio_get(bio);
605−
606− return 0;
607−}
608−
609557 /**
610558 * blk_rq_map_user_iov - map user data to a request, for passthrough requests
611559 * @q: request queue where request should be inserted
@@ -649,7 +597,10 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, · 37 unmodified lines
649597
650598 i = *iter;
651599 do {
652− ret =__blk_rq_map_user_iov(rq, map_data, &i, gfp_mask, copy);
600+ if (copy)
601+ ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask);
602+ else
603+ ret = bio_map_user_iov(rq, &i, gfp_mask);
653604 if (ret)
654605 goto unmap_rq;
655606 if (!bio)
@@ -700,9 +651,13 @@ int blk_rq_unmap_user(struct bio *bio) · 44 unmodified lines
700651 if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
701652 mapped_bio = bio->bi_private;
702653
703− ret2 = __blk_rq_unmap_user(mapped_bio);
704− if (ret2 && !ret)
705− ret = ret2;
654+ if (bio->bi_private) {
655+ ret2 = bio_uncopy_user(mapped_bio);
656+ if (ret2 && !ret)
657+ ret = ret2;
658+ } else {
659+ bio_unmap_user(mapped_bio);
660+ }
706661
707662 mapped_bio = bio;
708663 bio = bio->bi_next;
block/blk-merge.c+201−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 #include <trace/events/block.h>
1212
1313 #include "blk.h"
14+#include "blk-rq-qos.h"
1415
1516 static inline bool bio_will_gap(struct request_queue *q,
1617 struct request *prev_rq, struct bio *prev, struct bio *next)
@@ -895,3 +896,203 @@ enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) · 878 unmodified lines
895896 return ELEVATOR_FRONT_MERGE;
896897 return ELEVATOR_NO_MERGE;
897898 }
899+
900+static void blk_account_io_merge_bio(struct request *req)
901+{
902+ if (!blk_do_io_stat(req))
903+ return;
904+
905+ part_stat_lock();
906+ part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
907+ part_stat_unlock();
908+}
909+
910+enum bio_merge_status bio_attempt_back_merge(struct request *req,
911+ struct bio *bio,
912+ unsigned int nr_segs)
913+{
914+ const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
915+
916+ if (!ll_back_merge_fn(req, bio, nr_segs))
917+ return BIO_MERGE_FAILED;
918+
919+ trace_block_bio_backmerge(req->q, req, bio);
920+ rq_qos_merge(req->q, req, bio);
921+
922+ if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
923+ blk_rq_set_mixed_merge(req);
924+
925+ req->biotail->bi_next = bio;
926+ req->biotail = bio;
927+ req->__data_len += bio->bi_iter.bi_size;
928+
929+ bio_crypt_free_ctx(bio);
930+
931+ blk_account_io_merge_bio(req);
932+ return BIO_MERGE_OK;
933+}
934+
935+enum bio_merge_status bio_attempt_front_merge(struct request *req,
936+ struct bio *bio,
937+ unsigned int nr_segs)
938+{
939+ const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
940+
941+ if (!ll_front_merge_fn(req, bio, nr_segs))
942+ return BIO_MERGE_FAILED;
943+
944+ trace_block_bio_frontmerge(req->q, req, bio);
945+ rq_qos_merge(req->q, req, bio);
946+
947+ if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
948+ blk_rq_set_mixed_merge(req);
949+
950+ bio->bi_next = req->bio;
951+ req->bio = bio;
952+
953+ req->__sector = bio->bi_iter.bi_sector;
954+ req->__data_len += bio->bi_iter.bi_size;
955+
956+ bio_crypt_do_front_merge(req, bio);
957+
958+ blk_account_io_merge_bio(req);
959+ return BIO_MERGE_OK;
960+}
961+
962+enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
963+ struct request *req,
964+ struct bio *bio)
965+{
966+ unsigned short segments = blk_rq_nr_discard_segments(req);
967+
968+ if (segments >= queue_max_discard_segments(q))
969+ goto no_merge;
970+ if (blk_rq_sectors(req) + bio_sectors(bio) >
971+ blk_rq_get_max_sectors(req, blk_rq_pos(req)))
972+ goto no_merge;
973+
974+ rq_qos_merge(q, req, bio);
975+
976+ req->biotail->bi_next = bio;
977+ req->biotail = bio;
978+ req->__data_len += bio->bi_iter.bi_size;
979+ req->nr_phys_segments = segments + 1;
980+
981+ blk_account_io_merge_bio(req);
982+ return BIO_MERGE_OK;
983+no_merge:
984+ req_set_nomerge(q, req);
985+ return BIO_MERGE_FAILED;
986+}
987+
988+static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
989+ struct request *rq,
990+ struct bio *bio,
991+ unsigned int nr_segs,
992+ bool sched_allow_merge)
993+{
994+ if (!blk_rq_merge_ok(rq, bio))
995+ return BIO_MERGE_NONE;
996+
997+ switch (blk_try_merge(rq, bio)) {
998+ case ELEVATOR_BACK_MERGE:
999+ if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
1000+ return bio_attempt_back_merge(rq, bio, nr_segs);
1001+ break;
1002+ case ELEVATOR_FRONT_MERGE:
1003+ if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
1004+ return bio_attempt_front_merge(rq, bio, nr_segs);
1005+ break;
1006+ case ELEVATOR_DISCARD_MERGE:
1007+ return bio_attempt_discard_merge(q, rq, bio);
1008+ default:
1009+ return BIO_MERGE_NONE;
1010+ }
1011+
1012+ return BIO_MERGE_FAILED;
1013+}
1014+
1015+/**
1016+ * blk_attempt_plug_merge - try to merge with %current's plugged list
1017+ * @q: request_queue new bio is being queued at
1018+ * @bio: new bio being queued
1019+ * @nr_segs: number of segments in @bio
1020+ * @same_queue_rq: pointer to &struct request that gets filled in when
1021+ * another request associated with @q is found on the plug list
1022+ * (optional, may be %NULL)
1023+ *
1024+ * Determine whether @bio being queued on @q can be merged with a request
1025+ * on %current's plugged list. Returns %true if merge was successful,
1026+ * otherwise %false.
1027+ *
1028+ * Plugging coalesces IOs from the same issuer for the same purpose without
1029+ * going through @q->queue_lock. As such it's more of an issuing mechanism
1030+ * than scheduling, and the request, while may have elvpriv data, is not
1031+ * added on the elevator at this point. In addition, we don't have
1032+ * reliable access to the elevator outside queue lock. Only check basic
1033+ * merging parameters without querying the elevator.
1034+ *
1035+ * Caller must ensure !blk_queue_nomerges(q) beforehand.
1036+ */
1037+bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1038+ unsigned int nr_segs, struct request **same_queue_rq)
1039+{
1040+ struct blk_plug *plug;
1041+ struct request *rq;
1042+ struct list_head *plug_list;
1043+
1044+ plug = blk_mq_plug(q, bio);
1045+ if (!plug)
1046+ return false;
1047+
1048+ plug_list = &plug->mq_list;
1049+
1050+ list_for_each_entry_reverse(rq, plug_list, queuelist) {
1051+ if (rq->q == q && same_queue_rq) {
1052+ /*
1053+ * Only blk-mq multiple hardware queues case checks the
1054+ * rq in the same queue, there should be only one such
1055+ * rq in a queue
1056+ **/
1057+ *same_queue_rq = rq;
1058+ }
1059+
1060+ if (rq->q != q)
1061+ continue;
1062+
1063+ if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
1064+ BIO_MERGE_OK)
1065+ return true;
1066+ }
1067+
1068+ return false;
1069+}
1070+
1071+/*
1072+ * Iterate list of requests and see if we can merge this bio with any
1073+ * of them.
1074+ */
1075+bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
1076+ struct bio *bio, unsigned int nr_segs)
1077+{
1078+ struct request *rq;
1079+ int checked = 8;
1080+
1081+ list_for_each_entry_reverse(rq, list, queuelist) {
1082+ if (!checked--)
1083+ break;
1084+
1085+ switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) {
1086+ case BIO_MERGE_NONE:
1087+ continue;
1088+ case BIO_MERGE_OK:
1089+ return true;
1090+ case BIO_MERGE_FAILED:
1091+ return false;
1092+ }
1093+
1094+ }
1095+
1096+ return false;
1097+}
1098+EXPORT_SYMBOL_GPL(blk_bio_list_merge);
block/blk-mq-debugfs.c+6−5
@@ -116,6 +116,7 @@ static const char *const blk_queue_flag_name[] = { · 115 unmodified lines
116116 QUEUE_FLAG_NAME(SAME_FORCE),
117117 QUEUE_FLAG_NAME(DEAD),
118118 QUEUE_FLAG_NAME(INIT_DONE),
119+ QUEUE_FLAG_NAME(STABLE_WRITES),
119120 QUEUE_FLAG_NAME(POLL),
120121 QUEUE_FLAG_NAME(WC),
121122 QUEUE_FLAG_NAME(FUA),
@@ -240,7 +241,7 @@ static const char *const alloc_policy_name[] = { · 118 unmodified lines
240241 #define HCTX_FLAG_NAME(name) [ilog2(BLK_MQ_F_##name)] = #name
241242 static const char *const hctx_flag_name[] = {
242243 HCTX_FLAG_NAME(SHOULD_MERGE),
243− HCTX_FLAG_NAME(TAG_SHARED),
244+ HCTX_FLAG_NAME(TAG_QUEUE_SHARED),
244245 HCTX_FLAG_NAME(BLOCKING),
245246 HCTX_FLAG_NAME(NO_SCHED),
246247 HCTX_FLAG_NAME(STACKING),
@@ -452,11 +453,11 @@ static void blk_mq_debugfs_tags_show(struct seq_file *m, · 205 unmodified lines
452453 atomic_read(&tags->active_queues));
453454
454455 seq_puts(m, "\nbitmap_tags:\n");
455− sbitmap_queue_show(&tags->bitmap_tags, m);
456+ sbitmap_queue_show(tags->bitmap_tags, m);
456457
457458 if (tags->nr_reserved_tags) {
458459 seq_puts(m, "\nbreserved_tags:\n");
459− sbitmap_queue_show(&tags->breserved_tags, m);
460+ sbitmap_queue_show(tags->breserved_tags, m);
460461 }
461462 }
462463
@@ -487,7 +488,7 @@ static int hctx_tags_bitmap_show(void *data, struct seq_file *m) · 24 unmodified lines
487488 if (res)
488489 goto out;
489490 if (hctx->tags)
490− sbitmap_bitmap_show(&hctx->tags->bitmap_tags.sb, m);
491+ sbitmap_bitmap_show(&hctx->tags->bitmap_tags->sb, m);
491492 mutex_unlock(&q->sysfs_lock);
492493
493494 out:
@@ -521,7 +522,7 @@ static int hctx_sched_tags_bitmap_show(void *data, struct seq_file *m) · 27 unmodified lines
521522 if (res)
522523 goto out;
523524 if (hctx->sched_tags)
524− sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags.sb, m);
525+ sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags->sb, m);
525526 mutex_unlock(&q->sysfs_lock);
526527
527528 out:
block/blk-mq-sched.c+30−94
@@ -18,21 +18,6 @@ · 17 unmodified lines
1818 #include "blk-mq-tag.h"
1919 #include "blk-wbt.h"
2020
21−void blk_mq_sched_free_hctx_data(struct request_queue *q,
22− void (*exit)(struct blk_mq_hw_ctx *))
23−{
24− struct blk_mq_hw_ctx *hctx;
25− int i;
26−
27− queue_for_each_hw_ctx(q, hctx, i) {
28− if (exit && hctx->sched_data)
29− exit(hctx);
30− kfree(hctx->sched_data);
31− hctx->sched_data = NULL;
32− }
33−}
34−EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);
35−
3621 void blk_mq_sched_assign_ioc(struct request *rq)
3722 {
3823 struct request_queue *q = rq->q;
@@ -368,7 +353,7 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio, · 329 unmodified lines
368353 case ELEVATOR_BACK_MERGE:
369354 if (!blk_mq_sched_allow_merge(q, rq, bio))
370355 return false;
371− if (!bio_attempt_back_merge(rq, bio, nr_segs))
356+ if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
372357 return false;
373358 *merged_request = attempt_back_merge(q, rq);
374359 if (!*merged_request)
@@ -377,86 +362,20 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio, · 2 unmodified lines
377362 case ELEVATOR_FRONT_MERGE:
378363 if (!blk_mq_sched_allow_merge(q, rq, bio))
379364 return false;
380− if (!bio_attempt_front_merge(rq, bio, nr_segs))
365+ if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
381366 return false;
382367 *merged_request = attempt_front_merge(q, rq);
383368 if (!*merged_request)
384369 elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
385370 return true;
386371 case ELEVATOR_DISCARD_MERGE:
387− return bio_attempt_discard_merge(q, rq, bio);
372+ return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
388373 default:
389374 return false;
390375 }
391376 }
392377 EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
393378
394−/*
395− * Iterate list of requests and see if we can merge this bio with any
396− * of them.
397− */
398−bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
399− struct bio *bio, unsigned int nr_segs)
400−{
401− struct request *rq;
402− int checked = 8;
403−
404− list_for_each_entry_reverse(rq, list, queuelist) {
405− bool merged = false;
406−
407− if (!checked--)
408− break;
409−
410− if (!blk_rq_merge_ok(rq, bio))
411− continue;
412−
413− switch (blk_try_merge(rq, bio)) {
414− case ELEVATOR_BACK_MERGE:
415− if (blk_mq_sched_allow_merge(q, rq, bio))
416− merged = bio_attempt_back_merge(rq, bio,
417− nr_segs);
418− break;
419− case ELEVATOR_FRONT_MERGE:
420− if (blk_mq_sched_allow_merge(q, rq, bio))
421− merged = bio_attempt_front_merge(rq, bio,
422− nr_segs);
423− break;
424− case ELEVATOR_DISCARD_MERGE:
425− merged = bio_attempt_discard_merge(q, rq, bio);
426− break;
427− default:
428− continue;
429− }
430−
431− return merged;
432− }
433−
434− return false;
435−}
436−EXPORT_SYMBOL_GPL(blk_mq_bio_list_merge);
437−
438−/*
439− * Reverse check our software queue for entries that we could potentially
440− * merge with. Currently includes a hand-wavy stop count of 8, to not spend
441− * too much time checking for merges.
442− */
443−static bool blk_mq_attempt_merge(struct request_queue *q,
444− struct blk_mq_hw_ctx *hctx,
445− struct blk_mq_ctx *ctx, struct bio *bio,
446− unsigned int nr_segs)
447−{
448− enum hctx_type type = hctx->type;
449−
450− lockdep_assert_held(&ctx->lock);
451−
452− if (blk_mq_bio_list_merge(q, &ctx->rq_lists[type], bio, nr_segs)) {
453− ctx->rq_merged++;
454− return true;
455− }
456−
457− return false;
458−}
459−
460379 bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio,
461380 unsigned int nr_segs)
462381 {
@@ -470,14 +389,24 @@ bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio, · 7 unmodified lines
470389 return e->type->ops.bio_merge(hctx, bio, nr_segs);
471390
472391 type = hctx->type;
473− if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
474− !list_empty_careful(&ctx->rq_lists[type])) {
475− /* default per sw-queue merge */
476− spin_lock(&ctx->lock);
477− ret = blk_mq_attempt_merge(q, hctx, ctx, bio, nr_segs);
478− spin_unlock(&ctx->lock);
392+ if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE) ||
393+ list_empty_careful(&ctx->rq_lists[type]))
394+ return false;
395+
396+ /* default per sw-queue merge */
397+ spin_lock(&ctx->lock);
398+ /*
399+ * Reverse check our software queue for entries that we could
400+ * potentially merge with. Currently includes a hand-wavy stop
401+ * count of 8, to not spend too much time checking for merges.
402+ */
403+ if (blk_bio_list_merge(q, &ctx->rq_lists[type], bio, nr_segs)) {
404+ ctx->rq_merged++;
405+ ret = true;
479406 }
480407
408+ spin_unlock(&ctx->lock);
409+
481410 return ret;
482411 }
483412
@@ -531,7 +460,7 @@ void blk_mq_sched_insert_request(struct request *rq, bool at_head, · 47 unmodified lines
531460 goto run;
532461 }
533462
534− WARN_ON(e && (rq->tag != -1));
463+ WARN_ON(e && (rq->tag != BLK_MQ_NO_TAG));
535464
536465 if (blk_mq_sched_bypass_insert(hctx, !!e, rq)) {
537466 /*
@@ -616,9 +545,11 @@ static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set, · 78 unmodified lines
616545 struct blk_mq_hw_ctx *hctx,
617546 unsigned int hctx_idx)
618547 {
548+ unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;
549+
619550 if (hctx->sched_tags) {
620551 blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
621− blk_mq_free_rq_map(hctx->sched_tags);
552+ blk_mq_free_rq_map(hctx->sched_tags, flags);
622553 hctx->sched_tags = NULL;
623554 }
624555 }
@@ -628,10 +559,12 @@ static int blk_mq_sched_alloc_tags(struct request_queue *q, · 3 unmodified lines
628559 unsigned int hctx_idx)
629560 {
630561 struct blk_mq_tag_set *set = q->tag_set;
562+ /* Clear HCTX_SHARED so tags are init'ed */
563+ unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;
631564 int ret;
632565
633566 hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
634− set->reserved_tags);
567+ set->reserved_tags, flags);
635568 if (!hctx->sched_tags)
636569 return -ENOMEM;
637570
@@ -649,8 +582,11 @@ static void blk_mq_sched_tags_teardown(struct request_queue *q) · 11 unmodified lines
649582 int i;
650583
651584 queue_for_each_hw_ctx(q, hctx, i) {
585+ /* Clear HCTX_SHARED so tags are freed */
586+ unsigned int flags = hctx->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;
587+
652588 if (hctx->sched_tags) {
653− blk_mq_free_rq_map(hctx->sched_tags);
589+ blk_mq_free_rq_map(hctx->sched_tags, flags);
654590 hctx->sched_tags = NULL;
655591 }
656592 }
block/blk-mq-sched.h+0−3
@@ -5,9 +5,6 @@ · 4 unmodified lines
55 #include "blk-mq.h"
66 #include "blk-mq-tag.h"
77
8−void blk_mq_sched_free_hctx_data(struct request_queue *q,
9− void (*exit)(struct blk_mq_hw_ctx *));
10−
118 void blk_mq_sched_assign_ioc(struct request *rq);
129
1310 void blk_mq_sched_request_inserted(struct request *rq);
block/blk-mq-tag.c+110−42
@@ -23,9 +23,18 @@ · 22 unmodified lines
2323 */
2424 bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
2525 {
26− if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&
27− !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
28− atomic_inc(&hctx->tags->active_queues);
26+ if (blk_mq_is_sbitmap_shared(hctx->flags)) {
27+ struct request_queue *q = hctx->queue;
28+ struct blk_mq_tag_set *set = q->tag_set;
29+
30+ if (!test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags) &&
31+ !test_and_set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags))
32+ atomic_inc(&set->active_queues_shared_sbitmap);
33+ } else {
34+ if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&
35+ !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
36+ atomic_inc(&hctx->tags->active_queues);
37+ }
2938
3039 return true;
3140 }
@@ -35,9 +44,9 @@ bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) · 3 unmodified lines
3544 */
3645 void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
3746 {
38− sbitmap_queue_wake_all(&tags->bitmap_tags);
47+ sbitmap_queue_wake_all(tags->bitmap_tags);
3948 if (include_reserve)
40− sbitmap_queue_wake_all(&tags->breserved_tags);
49+ sbitmap_queue_wake_all(tags->breserved_tags);
4150 }
4251
4352 /*
@@ -47,11 +56,19 @@ void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve) · 3 unmodified lines
4756 void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
4857 {
4958 struct blk_mq_tags *tags = hctx->tags;
50−
51− if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
52− return;
53−
54− atomic_dec(&tags->active_queues);
59+ struct request_queue *q = hctx->queue;
60+ struct blk_mq_tag_set *set = q->tag_set;
61+
62+ if (blk_mq_is_sbitmap_shared(hctx->flags)) {
63+ if (!test_and_clear_bit(QUEUE_FLAG_HCTX_ACTIVE,
64+ &q->queue_flags))
65+ return;
66+ atomic_dec(&set->active_queues_shared_sbitmap);
67+ } else {
68+ if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
69+ return;
70+ atomic_dec(&tags->active_queues);
71+ }
5572
5673 blk_mq_tag_wakeup_all(tags, false);
5774 }
@@ -59,7 +76,8 @@ void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) · 1 unmodified line
5976 static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
6077 struct sbitmap_queue *bt)
6178 {
62− if (!data->q->elevator && !hctx_may_queue(data->hctx, bt))
79+ if (!data->q->elevator && !(data->flags & BLK_MQ_REQ_RESERVED) &&
80+ !hctx_may_queue(data->hctx, bt))
6381 return BLK_MQ_NO_TAG;
6482
6583 if (data->shallow_depth)
@@ -82,10 +100,10 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) · 16 unmodified lines
82100 WARN_ON_ONCE(1);
83101 return BLK_MQ_NO_TAG;
84102 }
85− bt = &tags->breserved_tags;
103+ bt = tags->breserved_tags;
86104 tag_offset = 0;
87105 } else {
88− bt = &tags->bitmap_tags;
106+ bt = tags->bitmap_tags;
89107 tag_offset = tags->nr_reserved_tags;
90108 }
91109
@@ -131,9 +149,9 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) · 39 unmodified lines
131149 data->ctx);
132150 tags = blk_mq_tags_from_data(data);
133151 if (data->flags & BLK_MQ_REQ_RESERVED)
134− bt = &tags->breserved_tags;
152+ bt = tags->breserved_tags;
135153 else
136− bt = &tags->bitmap_tags;
154+ bt = tags->bitmap_tags;
137155
138156 /*
139157 * If destination hw queue is changed, fake wake up on
@@ -167,10 +185,10 @@ void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, · 27 unmodified lines
167185 const int real_tag = tag - tags->nr_reserved_tags;
168186
169187 BUG_ON(real_tag >= tags->nr_tags);
170− sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);
188+ sbitmap_queue_clear(tags->bitmap_tags, real_tag, ctx->cpu);
171189 } else {
172190 BUG_ON(tag >= tags->nr_reserved_tags);
173− sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);
191+ sbitmap_queue_clear(tags->breserved_tags, tag, ctx->cpu);
174192 }
175193 }
176194
@@ -197,7 +215,7 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) · 20 unmodified lines
197215 * We can hit rq == NULL here, because the tagging functions
198216 * test and set the bit before assigning ->rqs[].
199217 */
200− if (rq && rq->q == hctx->queue)
218+ if (rq && rq->q == hctx->queue && rq->mq_hctx == hctx)
201219 return iter_data->fn(hctx, rq, iter_data->data, reserved);
202220 return true;
203221 }
@@ -298,9 +316,9 @@ static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags, · 94 unmodified lines
298316 WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED);
299317
300318 if (tags->nr_reserved_tags)
301− bt_tags_for_each(tags, &tags->breserved_tags, fn, priv,
319+ bt_tags_for_each(tags, tags->breserved_tags, fn, priv,
302320 flags | BT_TAG_ITER_RESERVED);
303− bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, flags);
321+ bt_tags_for_each(tags, tags->bitmap_tags, fn, priv, flags);
304322 }
305323
306324 /**
@@ -416,8 +434,8 @@ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, · 109 unmodified lines
416434 continue;
417435
418436 if (tags->nr_reserved_tags)
419− bt_for_each(hctx, &tags->breserved_tags, fn, priv, true);
420− bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false);
437+ bt_for_each(hctx, tags->breserved_tags, fn, priv, true);
438+ bt_for_each(hctx, tags->bitmap_tags, fn, priv, false);
421439 }
422440 blk_queue_exit(q);
423441 }
@@ -429,30 +447,64 @@ static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth, · 5 unmodified lines
429447 node);
430448 }
431449
432−static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags,
433− int node, int alloc_policy)
450+static int blk_mq_init_bitmap_tags(struct blk_mq_tags *tags,
451+ int node, int alloc_policy)
434452 {
435453 unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
436454 bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
437455
438− if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node))
439− goto free_tags;
440− if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin,
441− node))
456+ if (bt_alloc(&tags->__bitmap_tags, depth, round_robin, node))
457+ return -ENOMEM;
458+ if (bt_alloc(&tags->__breserved_tags, tags->nr_reserved_tags,
459+ round_robin, node))
442460 goto free_bitmap_tags;
443461
444− return tags;
462+ tags->bitmap_tags = &tags->__bitmap_tags;
463+ tags->breserved_tags = &tags->__breserved_tags;
464+
465+ return 0;
445466 free_bitmap_tags:
446− sbitmap_queue_free(&tags->bitmap_tags);
447−free_tags:
448− kfree(tags);
449− return NULL;
467+ sbitmap_queue_free(&tags->__bitmap_tags);
468+ return -ENOMEM;
469+}
470+
471+int blk_mq_init_shared_sbitmap(struct blk_mq_tag_set *set, unsigned int flags)
472+{
473+ unsigned int depth = set->queue_depth - set->reserved_tags;
474+ int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags);
475+ bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
476+ int i, node = set->numa_node;
477+
478+ if (bt_alloc(&set->__bitmap_tags, depth, round_robin, node))
479+ return -ENOMEM;
480+ if (bt_alloc(&set->__breserved_tags, set->reserved_tags,
481+ round_robin, node))
482+ goto free_bitmap_tags;
483+
484+ for (i = 0; i < set->nr_hw_queues; i++) {
485+ struct blk_mq_tags *tags = set->tags[i];
486+
487+ tags->bitmap_tags = &set->__bitmap_tags;
488+ tags->breserved_tags = &set->__breserved_tags;
489+ }
490+
491+ return 0;
492+free_bitmap_tags:
493+ sbitmap_queue_free(&set->__bitmap_tags);
494+ return -ENOMEM;
495+}
496+
497+void blk_mq_exit_shared_sbitmap(struct blk_mq_tag_set *set)
498+{
499+ sbitmap_queue_free(&set->__bitmap_tags);
500+ sbitmap_queue_free(&set->__breserved_tags);
450501 }
451502
452503 struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
453504 unsigned int reserved_tags,
454− int node, int alloc_policy)
505+ int node, unsigned int flags)
455506 {
507+ int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(flags);
456508 struct blk_mq_tags *tags;
457509
458510 if (total_tags > BLK_MQ_TAG_MAX) {
@@ -467,13 +519,22 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, · 8 unmodified lines
467519 tags->nr_tags = total_tags;
468520 tags->nr_reserved_tags = reserved_tags;
469521
470− return blk_mq_init_bitmap_tags(tags, node, alloc_policy);
522+ if (flags & BLK_MQ_F_TAG_HCTX_SHARED)
523+ return tags;
524+
525+ if (blk_mq_init_bitmap_tags(tags, node, alloc_policy) < 0) {
526+ kfree(tags);
527+ return NULL;
528+ }
529+ return tags;
471530 }
472531
473−void blk_mq_free_tags(struct blk_mq_tags *tags)
532+void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags)
474533 {
475− sbitmap_queue_free(&tags->bitmap_tags);
476− sbitmap_queue_free(&tags->breserved_tags);
534+ if (!(flags & BLK_MQ_F_TAG_HCTX_SHARED)) {
535+ sbitmap_queue_free(tags->bitmap_tags);
536+ sbitmap_queue_free(tags->breserved_tags);
537+ }
477538 kfree(tags);
478539 }
479540
@@ -492,6 +553,8 @@ int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, · 12 unmodified lines
492553 */
493554 if (tdepth > tags->nr_tags) {
494555 struct blk_mq_tag_set *set = hctx->queue->tag_set;
556+ /* Only sched tags can grow, so clear HCTX_SHARED flag */
557+ unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;
495558 struct blk_mq_tags *new;
496559 bool ret;
497560
@@ -506,30 +569,35 @@ int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, · 8 unmodified lines
506569 return -EINVAL;
507570
508571 new = blk_mq_alloc_rq_map(set, hctx->queue_num, tdepth,
509− tags->nr_reserved_tags);
572+ tags->nr_reserved_tags, flags);
510573 if (!new)
511574 return -ENOMEM;
512575 ret = blk_mq_alloc_rqs(set, new, hctx->queue_num, tdepth);
513576 if (ret) {
514− blk_mq_free_rq_map(new);
577+ blk_mq_free_rq_map(new, flags);
515578 return -ENOMEM;
516579 }
517580
518581 blk_mq_free_rqs(set, *tagsptr, hctx->queue_num);
519− blk_mq_free_rq_map(*tagsptr);
582+ blk_mq_free_rq_map(*tagsptr, flags);
520583 *tagsptr = new;
521584 } else {
522585 /*
523586 * Don't need (or can't) update reserved tags here, they
524587 * remain static and should never need resizing.
525588 */
526− sbitmap_queue_resize(&tags->bitmap_tags,
589+ sbitmap_queue_resize(tags->bitmap_tags,
527590 tdepth - tags->nr_reserved_tags);
528591 }
529592
530593 return 0;
531594 }
532595
596+void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set, unsigned int size)
597+{
598+ sbitmap_queue_resize(&set->__bitmap_tags, size - set->reserved_tags);
599+}
600+
533601 /**
534602 * blk_mq_unique_tag() - return a tag that is unique queue-wide
535603 * @rq: request for which to compute a unique tag
block/blk-mq-tag.h+17−39
@@ -2,8 +2,6 @@ · 1 unmodified line
22 #ifndef INT_BLK_MQ_TAG_H
33 #define INT_BLK_MQ_TAG_H
44
5−#include "blk-mq.h"
6−
75 /*
86 * Tag address space map.
97 */
@@ -13,17 +11,25 @@ struct blk_mq_tags { · 3 unmodified lines
1311
1412 atomic_t active_queues;
1513
16− struct sbitmap_queue bitmap_tags;
17− struct sbitmap_queue breserved_tags;
14+ struct sbitmap_queue *bitmap_tags;
15+ struct sbitmap_queue *breserved_tags;
16+
17+ struct sbitmap_queue __bitmap_tags;
18+ struct sbitmap_queue __breserved_tags;
1819
1920 struct request **rqs;
2021 struct request **static_rqs;
2122 struct list_head page_list;
2223 };
2324
25+extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags,
26+ unsigned int reserved_tags,
27+ int node, unsigned int flags);
28+extern void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags);
2429
25−extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node, int alloc_policy);
26−extern void blk_mq_free_tags(struct blk_mq_tags *tags);
30+extern int blk_mq_init_shared_sbitmap(struct blk_mq_tag_set *set,
31+ unsigned int flags);
32+extern void blk_mq_exit_shared_sbitmap(struct blk_mq_tag_set *set);
2733
2834 extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);
2935 extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
@@ -31,6 +37,9 @@ extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, · 1 unmodified line
3137 extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
3238 struct blk_mq_tags **tags,
3339 unsigned int depth, bool can_grow);
40+extern void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set,
41+ unsigned int size);
42+
3443 extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
3544 void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
3645 void *priv);
@@ -56,7 +65,7 @@ extern void __blk_mq_tag_idle(struct blk_mq_hw_ctx *); · 19 unmodified lines
5665
5766 static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
5867 {
59− if (!(hctx->flags & BLK_MQ_F_TAG_SHARED))
68+ if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
6069 return false;
6170
6271 return __blk_mq_tag_busy(hctx);
@@ -64,43 +73,12 @@ static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) · 1 unmodified line
6473
6574 static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
6675 {
67− if (!(hctx->flags & BLK_MQ_F_TAG_SHARED))
76+ if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
6877 return;
6978
7079 __blk_mq_tag_idle(hctx);
7180 }
7281
73−/*
74− * For shared tag users, we track the number of currently active users
75− * and attempt to provide a fair share of the tag depth for each of them.
76− */
77−static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
78− struct sbitmap_queue *bt)
79−{
80− unsigned int depth, users;
81−
82− if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))
83− return true;
84− if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
85− return true;
86−
87− /*
88− * Don't try dividing an ant
89− */
90− if (bt->sb.depth == 1)
91− return true;
92−
93− users = atomic_read(&hctx->tags->active_queues);
94− if (!users)
95− return true;
96−
97− /*
98− * Allow at least some tags
99− */
100− depth = max((bt->sb.depth + users - 1) / users, 4U);
101− return atomic_read(&hctx->nr_active) < depth;
102−}
103−
10482 static inline bool blk_mq_tag_is_reserved(struct blk_mq_tags *tags,
10583 unsigned int tag)
10684 {
block/blk-mq.c+54−32
@@ -519,7 +519,7 @@ void blk_mq_free_request(struct request *rq) · 518 unmodified lines
519519
520520 ctx->rq_completed[rq_is_sync(rq)]++;
521521 if (rq->rq_flags & RQF_MQ_INFLIGHT)
522− atomic_dec(&hctx->nr_active);
522+ __blk_mq_dec_active_requests(hctx);
523523
524524 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
525525 laptop_io_completion(q->backing_dev_info);
@@ -1096,19 +1096,20 @@ static inline unsigned int queued_to_index(unsigned int queued) · 570 unmodified lines
10961096
10971097 static bool __blk_mq_get_driver_tag(struct request *rq)
10981098 {
1099− struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
1099+ struct sbitmap_queue *bt = rq->mq_hctx->tags->bitmap_tags;
11001100 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
11011101 int tag;
11021102
11031103 blk_mq_tag_busy(rq->mq_hctx);
11041104
11051105 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1106− bt = &rq->mq_hctx->tags->breserved_tags;
1106+ bt = rq->mq_hctx->tags->breserved_tags;
11071107 tag_offset = 0;
1108+ } else {
1109+ if (!hctx_may_queue(rq->mq_hctx, bt))
1110+ return false;
11081111 }
11091112
1110− if (!hctx_may_queue(rq->mq_hctx, bt))
1111− return false;
11121113 tag = __sbitmap_queue_get(bt);
11131114 if (tag == BLK_MQ_NO_TAG)
11141115 return false;
@@ -1124,10 +1125,10 @@ static bool blk_mq_get_driver_tag(struct request *rq) · 9 unmodified lines
11241125 if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
11251126 return false;
11261127
1127− if ((hctx->flags & BLK_MQ_F_TAG_SHARED) &&
1128+ if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
11281129 !(rq->rq_flags & RQF_MQ_INFLIGHT)) {
11291130 rq->rq_flags |= RQF_MQ_INFLIGHT;
1130− atomic_inc(&hctx->nr_active);
1131+ __blk_mq_inc_active_requests(hctx);
11311132 }
11321133 hctx->tags->rqs[rq->tag] = rq;
11331134 return true;
@@ -1145,7 +1146,7 @@ static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, · 11 unmodified lines
11451146 struct sbitmap_queue *sbq;
11461147
11471148 list_del_init(&wait->entry);
1148− sbq = &hctx->tags->bitmap_tags;
1149+ sbq = hctx->tags->bitmap_tags;
11491150 atomic_dec(&sbq->ws_active);
11501151 }
11511152 spin_unlock(&hctx->dispatch_wait_lock);
@@ -1163,12 +1164,12 @@ static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, · 11 unmodified lines
11631164 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
11641165 struct request *rq)
11651166 {
1166− struct sbitmap_queue *sbq = &hctx->tags->bitmap_tags;
1167+ struct sbitmap_queue *sbq = hctx->tags->bitmap_tags;
11671168 struct wait_queue_head *wq;
11681169 wait_queue_entry_t *wait;
11691170 bool ret;
11701171
1171− if (!(hctx->flags & BLK_MQ_F_TAG_SHARED)) {
1172+ if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
11721173 blk_mq_sched_mark_restart_hctx(hctx);
11731174
11741175 /*
@@ -1420,7 +1421,7 @@ out: · 245 unmodified lines
14201421 bool needs_restart;
14211422 /* For non-shared tags, the RESTART check will suffice */
14221423 bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
1423− (hctx->flags & BLK_MQ_F_TAG_SHARED);
1424+ (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
14241425 bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
14251426
14261427 blk_mq_release_budgets(q, nr_budgets);
@@ -2296,20 +2297,21 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, · 869 unmodified lines
22962297 }
22972298 }
22982299
2299−void blk_mq_free_rq_map(struct blk_mq_tags *tags)
2300+void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags)
23002301 {
23012302 kfree(tags->rqs);
23022303 tags->rqs = NULL;
23032304 kfree(tags->static_rqs);
23042305 tags->static_rqs = NULL;
23052306
2306− blk_mq_free_tags(tags);
2307+ blk_mq_free_tags(tags, flags);
23072308 }
23082309
23092310 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
23102311 unsigned int hctx_idx,
23112312 unsigned int nr_tags,
2312− unsigned int reserved_tags)
2313+ unsigned int reserved_tags,
2314+ unsigned int flags)
23132315 {
23142316 struct blk_mq_tags *tags;
23152317 int node;
@@ -2318,8 +2320,7 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, · 2 unmodified lines
23182320 if (node == NUMA_NO_NODE)
23192321 node = set->numa_node;
23202322
2321− tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
2322− BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
2323+ tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags);
23232324 if (!tags)
23242325 return NULL;
23252326
@@ -2327,7 +2328,7 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, · 1 unmodified line
23272328 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
23282329 node);
23292330 if (!tags->rqs) {
2330− blk_mq_free_tags(tags);
2331+ blk_mq_free_tags(tags, flags);
23312332 return NULL;
23322333 }
23332334
@@ -2336,7 +2337,7 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, · 2 unmodified lines
23362337 node);
23372338 if (!tags->static_rqs) {
23382339 kfree(tags->rqs);
2339− blk_mq_free_tags(tags);
2340+ blk_mq_free_tags(tags, flags);
23402341 return NULL;
23412342 }
23422343
@@ -2660,6 +2661,7 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, · 317 unmodified lines
26602661 goto free_hctx;
26612662
26622663 atomic_set(&hctx->nr_active, 0);
2664+ atomic_set(&hctx->elevator_queued, 0);
26632665 if (node == NUMA_NO_NODE)
26642666 node = set->numa_node;
26652667 hctx->numa_node = node;
@@ -2668,7 +2670,7 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, · 2 unmodified lines
26682670 spin_lock_init(&hctx->lock);
26692671 INIT_LIST_HEAD(&hctx->dispatch);
26702672 hctx->queue = q;
2671− hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
2673+ hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
26722674
26732675 INIT_LIST_HEAD(&hctx->hctx_list);
26742676
@@ -2745,10 +2747,11 @@ static void blk_mq_init_cpu_queues(struct request_queue *q, · 70 unmodified lines
27452747 static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set,
27462748 int hctx_idx)
27472749 {
2750+ unsigned int flags = set->flags;
27482751 int ret = 0;
27492752
27502753 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
2751− set->queue_depth, set->reserved_tags);
2754+ set->queue_depth, set->reserved_tags, flags);
27522755 if (!set->tags[hctx_idx])
27532756 return false;
27542757
@@ -2757,7 +2760,7 @@ static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set, · 2 unmodified lines
27572760 if (!ret)
27582761 return true;
27592762
2760− blk_mq_free_rq_map(set->tags[hctx_idx]);
2763+ blk_mq_free_rq_map(set->tags[hctx_idx], flags);
27612764 set->tags[hctx_idx] = NULL;
27622765 return false;
27632766 }
@@ -2765,9 +2768,11 @@ static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set, · 1 unmodified line
27652768 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
27662769 unsigned int hctx_idx)
27672770 {
2771+ unsigned int flags = set->flags;
2772+
27682773 if (set->tags && set->tags[hctx_idx]) {
27692774 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2770− blk_mq_free_rq_map(set->tags[hctx_idx]);
2775+ blk_mq_free_rq_map(set->tags[hctx_idx], flags);
27712776 set->tags[hctx_idx] = NULL;
27722777 }
27732778 }
@@ -2885,14 +2890,14 @@ static void queue_set_hctx_shared(struct request_queue *q, bool shared) · 111 unmodified lines
28852890
28862891 queue_for_each_hw_ctx(q, hctx, i) {
28872892 if (shared)
2888− hctx->flags |= BLK_MQ_F_TAG_SHARED;
2893+ hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
28892894 else
2890− hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
2895+ hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
28912896 }
28922897 }
28932898
2894−static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set,
2895− bool shared)
2899+static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
2900+ bool shared)
28962901 {
28972902 struct request_queue *q;
28982903
@@ -2913,9 +2918,9 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q) · 14 unmodified lines
29132918 list_del(&q->tag_set_list);
29142919 if (list_is_singular(&set->tag_list)) {
29152920 /* just transitioned to unshared */
2916− set->flags &= ~BLK_MQ_F_TAG_SHARED;
2921+ set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
29172922 /* update existing queue */
2918− blk_mq_update_tag_set_depth(set, false);
2923+ blk_mq_update_tag_set_shared(set, false);
29192924 }
29202925 mutex_unlock(&set->tag_list_lock);
29212926 INIT_LIST_HEAD(&q->tag_set_list);
@@ -2930,12 +2935,12 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, · 8 unmodified lines
29302935 * Check to see if we're transitioning to shared (from 1 to 2 queues).
29312936 */
29322937 if (!list_empty(&set->tag_list) &&
2933− !(set->flags & BLK_MQ_F_TAG_SHARED)) {
2934− set->flags |= BLK_MQ_F_TAG_SHARED;
2938+ !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
2939+ set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
29352940 /* update existing queue */
2936− blk_mq_update_tag_set_depth(set, true);
2941+ blk_mq_update_tag_set_shared(set, true);
29372942 }
2938− if (set->flags & BLK_MQ_F_TAG_SHARED)
2943+ if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
29392944 queue_set_hctx_shared(q, true);
29402945 list_add_tail(&q->tag_set_list, &set->tag_list);
29412946
@@ -3438,11 +3443,23 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) · 496 unmodified lines
34383443 if (ret)
34393444 goto out_free_mq_map;
34403445
3446+ if (blk_mq_is_sbitmap_shared(set->flags)) {
3447+ atomic_set(&set->active_queues_shared_sbitmap, 0);
3448+
3449+ if (blk_mq_init_shared_sbitmap(set, set->flags)) {
3450+ ret = -ENOMEM;
3451+ goto out_free_mq_rq_maps;
3452+ }
3453+ }
3454+
34413455 mutex_init(&set->tag_list_lock);
34423456 INIT_LIST_HEAD(&set->tag_list);
34433457
34443458 return 0;
34453459
3460+out_free_mq_rq_maps:
3461+ for (i = 0; i < set->nr_hw_queues; i++)
3462+ blk_mq_free_map_and_requests(set, i);
34463463 out_free_mq_map:
34473464 for (i = 0; i < set->nr_maps; i++) {
34483465 kfree(set->map[i].mq_map);
@@ -3461,6 +3478,9 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set) · 12 unmodified lines
34613478 for (i = 0; i < set->nr_hw_queues; i++)
34623479 blk_mq_free_map_and_requests(set, i);
34633480
3481+ if (blk_mq_is_sbitmap_shared(set->flags))
3482+ blk_mq_exit_shared_sbitmap(set);
3483+
34643484 for (j = 0; j < set->nr_maps; j++) {
34653485 kfree(set->map[j].mq_map);
34663486 set->map[j].mq_map = NULL;
@@ -3497,6 +3517,8 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) · 30 unmodified lines
34973517 if (!hctx->sched_tags) {
34983518 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
34993519 false);
3520+ if (!ret && blk_mq_is_sbitmap_shared(set->flags))
3521+ blk_mq_tag_resize_shared_sbitmap(set, nr);
35003522 } else {
35013523 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
35023524 nr, true);
block/blk-mq.h+73−3
@@ -53,11 +53,12 @@ struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, · 52 unmodified lines
5353 */
5454 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
5555 unsigned int hctx_idx);
56−void blk_mq_free_rq_map(struct blk_mq_tags *tags);
56+void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags);
5757 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
5858 unsigned int hctx_idx,
5959 unsigned int nr_tags,
60− unsigned int reserved_tags);
60+ unsigned int reserved_tags,
61+ unsigned int flags);
6162 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
6263 unsigned int hctx_idx, unsigned int depth);
6364
@@ -158,6 +159,11 @@ struct blk_mq_alloc_data { · 94 unmodified lines
158159 struct blk_mq_hw_ctx *hctx;
159160 };
160161
162+static inline bool blk_mq_is_sbitmap_shared(unsigned int flags)
163+{
164+ return flags & BLK_MQ_F_TAG_HCTX_SHARED;
165+}
166+
161167 static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data)
162168 {
163169 if (data->q->elevator)
@@ -193,6 +199,28 @@ static inline bool blk_mq_get_dispatch_budget(struct request_queue *q) · 29 unmodified lines
193199 return true;
194200 }
195201
202+static inline void __blk_mq_inc_active_requests(struct blk_mq_hw_ctx *hctx)
203+{
204+ if (blk_mq_is_sbitmap_shared(hctx->flags))
205+ atomic_inc(&hctx->queue->nr_active_requests_shared_sbitmap);
206+ else
207+ atomic_inc(&hctx->nr_active);
208+}
209+
210+static inline void __blk_mq_dec_active_requests(struct blk_mq_hw_ctx *hctx)
211+{
212+ if (blk_mq_is_sbitmap_shared(hctx->flags))
213+ atomic_dec(&hctx->queue->nr_active_requests_shared_sbitmap);
214+ else
215+ atomic_dec(&hctx->nr_active);
216+}
217+
218+static inline int __blk_mq_active_requests(struct blk_mq_hw_ctx *hctx)
219+{
220+ if (blk_mq_is_sbitmap_shared(hctx->flags))
221+ return atomic_read(&hctx->queue->nr_active_requests_shared_sbitmap);
222+ return atomic_read(&hctx->nr_active);
223+}
196224 static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
197225 struct request *rq)
198226 {
@@ -201,7 +229,7 @@ static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx, · 2 unmodified lines
201229
202230 if (rq->rq_flags & RQF_MQ_INFLIGHT) {
203231 rq->rq_flags &= ~RQF_MQ_INFLIGHT;
204− atomic_dec(&hctx->nr_active);
232+ __blk_mq_dec_active_requests(hctx);
205233 }
206234 }
207235
@@ -253,4 +281,46 @@ static inline struct blk_plug *blk_mq_plug(struct request_queue *q, · 45 unmodified lines
253281 return NULL;
254282 }
255283
284+/*
285+ * For shared tag users, we track the number of currently active users
286+ * and attempt to provide a fair share of the tag depth for each of them.
287+ */
288+static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
289+ struct sbitmap_queue *bt)
290+{
291+ unsigned int depth, users;
292+
293+ if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
294+ return true;
295+
296+ /*
297+ * Don't try dividing an ant
298+ */
299+ if (bt->sb.depth == 1)
300+ return true;
301+
302+ if (blk_mq_is_sbitmap_shared(hctx->flags)) {
303+ struct request_queue *q = hctx->queue;
304+ struct blk_mq_tag_set *set = q->tag_set;
305+
306+ if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &q->queue_flags))
307+ return true;
308+ users = atomic_read(&set->active_queues_shared_sbitmap);
309+ } else {
310+ if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
311+ return true;
312+ users = atomic_read(&hctx->tags->active_queues);
313+ }
314+
315+ if (!users)
316+ return true;
317+
318+ /*
319+ * Allow at least some tags
320+ */
321+ depth = max((bt->sb.depth + users - 1) / users, 4U);
322+ return __blk_mq_active_requests(hctx) < depth;
323+}
324+
325+
256326 #endif
block/blk-settings.c+28−12
@@ -172,15 +172,13 @@ EXPORT_SYMBOL(blk_queue_max_hw_sectors); · 171 unmodified lines
172172 *
173173 * Description:
174174 * If a driver doesn't want IOs to cross a given chunk size, it can set
175− * this limit and prevent merging across chunks. Note that the chunk size
176− * must currently be a power-of-2 in sectors. Also note that the block
177− * layer must accept a page worth of data at any offset. So if the
178− * crossing of chunks is a hard limitation in the driver, it must still be
179− * prepared to split single page bios.
175+ * this limit and prevent merging across chunks. Note that the block layer
176+ * must accept a page worth of data at any offset. So if the crossing of
177+ * chunks is a hard limitation in the driver, it must still be prepared
178+ * to split single page bios.
180179 **/
181180 void blk_queue_chunk_sectors(struct request_queue *q, unsigned int chunk_sectors)
182181 {
183− BUG_ON(!is_power_of_2(chunk_sectors));
184182 q->limits.chunk_sectors = chunk_sectors;
185183 }
186184 EXPORT_SYMBOL(blk_queue_chunk_sectors);
@@ -374,6 +372,19 @@ void blk_queue_alignment_offset(struct request_queue *q, unsigned int offset) · 187 unmodified lines
374372 }
375373 EXPORT_SYMBOL(blk_queue_alignment_offset);
376374
375+void blk_queue_update_readahead(struct request_queue *q)
376+{
377+ /*
378+ * For read-ahead of large files to be effective, we need to read ahead
379+ * at least twice the optimal I/O size.
380+ */
381+ q->backing_dev_info->ra_pages =
382+ max(queue_io_opt(q) * 2 / PAGE_SIZE, VM_READAHEAD_PAGES);
383+ q->backing_dev_info->io_pages =
384+ queue_max_sectors(q) >> (PAGE_SHIFT - 9);
385+}
386+EXPORT_SYMBOL_GPL(blk_queue_update_readahead);
387+
377388 /**
378389 * blk_limits_io_min - set minimum request size for a device
379390 * @limits: the queue limits
@@ -452,6 +463,8 @@ EXPORT_SYMBOL(blk_limits_io_opt); · 72 unmodified lines
452463 void blk_queue_io_opt(struct request_queue *q, unsigned int opt)
453464 {
454465 blk_limits_io_opt(&q->limits, opt);
466+ q->backing_dev_info->ra_pages =
467+ max(queue_io_opt(q) * 2 / PAGE_SIZE, VM_READAHEAD_PAGES);
455468 }
456469 EXPORT_SYMBOL(blk_queue_io_opt);
457470
@@ -534,6 +547,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, · 76 unmodified lines
534547
535548 t->io_min = max(t->io_min, b->io_min);
536549 t->io_opt = lcm_not_zero(t->io_opt, b->io_opt);
550+ t->chunk_sectors = lcm_not_zero(t->chunk_sectors, b->chunk_sectors);
537551
538552 /* Physical block size a multiple of the logical block size? */
539553 if (t->physical_block_size & (t->logical_block_size - 1)) {
@@ -556,6 +570,13 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, · 16 unmodified lines
556570 ret = -1;
557571 }
558572
573+ /* chunk_sectors a multiple of the physical block size? */
574+ if ((t->chunk_sectors << 9) & (t->physical_block_size - 1)) {
575+ t->chunk_sectors = 0;
576+ t->misaligned = 1;
577+ ret = -1;
578+ }
579+
559580 t->raid_partial_stripes_expensive =
560581 max(t->raid_partial_stripes_expensive,
561582 b->raid_partial_stripes_expensive);
@@ -594,10 +615,6 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, · 32 unmodified lines
594615 t->discard_granularity;
595616 }
596617
597− if (b->chunk_sectors)
598− t->chunk_sectors = min_not_zero(t->chunk_sectors,
599− b->chunk_sectors);
600−
601618 t->zoned = max(t->zoned, b->zoned);
602619 return ret;
603620 }
@@ -629,8 +646,7 @@ void disk_stack_limits(struct gendisk *disk, struct block_device *bdev, · 25 unmodified lines
629646 top, bottom);
630647 }
631648
632− t->backing_dev_info->io_pages =
633− t->limits.max_sectors >> (PAGE_SHIFT - 9);
649+ blk_queue_update_readahead(disk->queue);
634650 }
635651 EXPORT_SYMBOL(disk_stack_limits);
636652
block/blk-sysfs.c+68−209
@@ -260,14 +260,14 @@ static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page) · 259 unmodified lines
260260
261261 #define QUEUE_SYSFS_BIT_FNS(name, flag, neg) \
262262 static ssize_t \
263−queue_show_##name(struct request_queue *q, char *page) \
263+queue_##name##_show(struct request_queue *q, char *page) \
264264 { \
265265 int bit; \
266266 bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags); \
267267 return queue_var_show(neg ? !bit : bit, page); \
268268 } \
269269 static ssize_t \
270−queue_store_##name(struct request_queue *q, const char *page, size_t count) \
270+queue_##name##_store(struct request_queue *q, const char *page, size_t count) \
271271 { \
272272 unsigned long val; \
273273 ssize_t ret; \
@@ -287,6 +287,7 @@ queue_store_##name(struct request_queue *q, const char *page, size_t count) \ · 13 unmodified lines
287287 QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
288288 QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
289289 QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
290+QUEUE_SYSFS_BIT_FNS(stable_writes, STABLE_WRITES, 0);
290291 #undef QUEUE_SYSFS_BIT_FNS
291292
292293 static ssize_t queue_zoned_show(struct request_queue *q, char *page)
@@ -547,218 +548,73 @@ static ssize_t queue_dax_show(struct request_queue *q, char *page) · 254 unmodified lines
547548 return queue_var_show(blk_queue_dax(q), page);
548549 }
549550
550−static struct queue_sysfs_entry queue_requests_entry = {
551− .attr = {.name = "nr_requests", .mode = 0644 },
552− .show = queue_requests_show,
553− .store = queue_requests_store,
554−};
555−
556−static struct queue_sysfs_entry queue_ra_entry = {
557− .attr = {.name = "read_ahead_kb", .mode = 0644 },
558− .show = queue_ra_show,
559− .store = queue_ra_store,
560−};
561−
562−static struct queue_sysfs_entry queue_max_sectors_entry = {
563− .attr = {.name = "max_sectors_kb", .mode = 0644 },
564− .show = queue_max_sectors_show,
565− .store = queue_max_sectors_store,
566−};
567−
568−static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
569− .attr = {.name = "max_hw_sectors_kb", .mode = 0444 },
570− .show = queue_max_hw_sectors_show,
571−};
572−
573−static struct queue_sysfs_entry queue_max_segments_entry = {
574− .attr = {.name = "max_segments", .mode = 0444 },
575− .show = queue_max_segments_show,
576−};
577−
578−static struct queue_sysfs_entry queue_max_discard_segments_entry = {
579− .attr = {.name = "max_discard_segments", .mode = 0444 },
580− .show = queue_max_discard_segments_show,
581−};
582−
583−static struct queue_sysfs_entry queue_max_integrity_segments_entry = {
584− .attr = {.name = "max_integrity_segments", .mode = 0444 },
585− .show = queue_max_integrity_segments_show,
586−};
587−
588−static struct queue_sysfs_entry queue_max_segment_size_entry = {
589− .attr = {.name = "max_segment_size", .mode = 0444 },
590− .show = queue_max_segment_size_show,
591−};
551+#define QUEUE_RO_ENTRY(_prefix, _name) \
552+static struct queue_sysfs_entry _prefix##_entry = { \
553+ .attr = { .name = _name, .mode = 0444 }, \
554+ .show = _prefix##_show, \
555+};
556+
557+#define QUEUE_RW_ENTRY(_prefix, _name) \
558+static struct queue_sysfs_entry _prefix##_entry = { \
559+ .attr = { .name = _name, .mode = 0644 }, \
560+ .show = _prefix##_show, \
561+ .store = _prefix##_store, \
562+};
563+
564+QUEUE_RW_ENTRY(queue_requests, "nr_requests");
565+QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
566+QUEUE_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
567+QUEUE_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
568+QUEUE_RO_ENTRY(queue_max_segments, "max_segments");
569+QUEUE_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
570+QUEUE_RO_ENTRY(queue_max_segment_size, "max_segment_size");
571+QUEUE_RW_ENTRY(elv_iosched, "scheduler");
572+
573+QUEUE_RO_ENTRY(queue_logical_block_size, "logical_block_size");
574+QUEUE_RO_ENTRY(queue_physical_block_size, "physical_block_size");
575+QUEUE_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
576+QUEUE_RO_ENTRY(queue_io_min, "minimum_io_size");
577+QUEUE_RO_ENTRY(queue_io_opt, "optimal_io_size");
578+
579+QUEUE_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
580+QUEUE_RO_ENTRY(queue_discard_granularity, "discard_granularity");
581+QUEUE_RO_ENTRY(queue_discard_max_hw, "discard_max_hw_bytes");
582+QUEUE_RW_ENTRY(queue_discard_max, "discard_max_bytes");
583+QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");
584+
585+QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
586+QUEUE_RO_ENTRY(queue_write_zeroes_max, "write_zeroes_max_bytes");
587+QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");
588+
589+QUEUE_RO_ENTRY(queue_zoned, "zoned");
590+QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
591+QUEUE_RO_ENTRY(queue_max_open_zones, "max_open_zones");
592+QUEUE_RO_ENTRY(queue_max_active_zones, "max_active_zones");
593+
594+QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
595+QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
596+QUEUE_RW_ENTRY(queue_poll, "io_poll");
597+QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
598+QUEUE_RW_ENTRY(queue_wc, "write_cache");
599+QUEUE_RO_ENTRY(queue_fua, "fua");
600+QUEUE_RO_ENTRY(queue_dax, "dax");
601+QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
602+QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
592603
593−static struct queue_sysfs_entry queue_iosched_entry = {
594− .attr = {.name = "scheduler", .mode = 0644 },
595− .show = elv_iosched_show,
596− .store = elv_iosched_store,
597−};
604+#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
605+QUEUE_RW_ENTRY(blk_throtl_sample_time, "throttle_sample_time");
606+#endif
598607
608+/* legacy alias for logical_block_size: */
599609 static struct queue_sysfs_entry queue_hw_sector_size_entry = {
600610 .attr = {.name = "hw_sector_size", .mode = 0444 },
601611 .show = queue_logical_block_size_show,
602612 };
603613
604−static struct queue_sysfs_entry queue_logical_block_size_entry = {
605− .attr = {.name = "logical_block_size", .mode = 0444 },
606− .show = queue_logical_block_size_show,
607−};
608−
609−static struct queue_sysfs_entry queue_physical_block_size_entry = {
610− .attr = {.name = "physical_block_size", .mode = 0444 },
611− .show = queue_physical_block_size_show,
612−};
613−
614−static struct queue_sysfs_entry queue_chunk_sectors_entry = {
615− .attr = {.name = "chunk_sectors", .mode = 0444 },
616− .show = queue_chunk_sectors_show,
617−};
618−
619−static struct queue_sysfs_entry queue_io_min_entry = {
620− .attr = {.name = "minimum_io_size", .mode = 0444 },
621− .show = queue_io_min_show,
622−};
623−
624−static struct queue_sysfs_entry queue_io_opt_entry = {
625− .attr = {.name = "optimal_io_size", .mode = 0444 },
626− .show = queue_io_opt_show,
627−};
628−
629−static struct queue_sysfs_entry queue_discard_granularity_entry = {
630− .attr = {.name = "discard_granularity", .mode = 0444 },
631− .show = queue_discard_granularity_show,
632−};
633−
634−static struct queue_sysfs_entry queue_discard_max_hw_entry = {
635− .attr = {.name = "discard_max_hw_bytes", .mode = 0444 },
636− .show = queue_discard_max_hw_show,
637−};
638−
639−static struct queue_sysfs_entry queue_discard_max_entry = {
640− .attr = {.name = "discard_max_bytes", .mode = 0644 },
641− .show = queue_discard_max_show,
642− .store = queue_discard_max_store,
643−};
644−
645−static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
646− .attr = {.name = "discard_zeroes_data", .mode = 0444 },
647− .show = queue_discard_zeroes_data_show,
648−};
649−
650−static struct queue_sysfs_entry queue_write_same_max_entry = {
651− .attr = {.name = "write_same_max_bytes", .mode = 0444 },
652− .show = queue_write_same_max_show,
653−};
654−
655−static struct queue_sysfs_entry queue_write_zeroes_max_entry = {
656− .attr = {.name = "write_zeroes_max_bytes", .mode = 0444 },
657− .show = queue_write_zeroes_max_show,
658−};
659−
660−static struct queue_sysfs_entry queue_zone_append_max_entry = {
661− .attr = {.name = "zone_append_max_bytes", .mode = 0444 },
662− .show = queue_zone_append_max_show,
663−};
664−
665−static struct queue_sysfs_entry queue_nonrot_entry = {
666− .attr = {.name = "rotational", .mode = 0644 },
667− .show = queue_show_nonrot,
668− .store = queue_store_nonrot,
669−};
670−
671−static struct queue_sysfs_entry queue_zoned_entry = {
672− .attr = {.name = "zoned", .mode = 0444 },
673− .show = queue_zoned_show,
674−};
675−
676−static struct queue_sysfs_entry queue_nr_zones_entry = {
677− .attr = {.name = "nr_zones", .mode = 0444 },
678− .show = queue_nr_zones_show,
679−};
680−
681−static struct queue_sysfs_entry queue_max_open_zones_entry = {
682− .attr = {.name = "max_open_zones", .mode = 0444 },
683− .show = queue_max_open_zones_show,
684−};
685−
686−static struct queue_sysfs_entry queue_max_active_zones_entry = {
687− .attr = {.name = "max_active_zones", .mode = 0444 },
688− .show = queue_max_active_zones_show,
689−};
690−
691−static struct queue_sysfs_entry queue_nomerges_entry = {
692− .attr = {.name = "nomerges", .mode = 0644 },
693− .show = queue_nomerges_show,
694− .store = queue_nomerges_store,
695−};
696−
697−static struct queue_sysfs_entry queue_rq_affinity_entry = {
698− .attr = {.name = "rq_affinity", .mode = 0644 },
699− .show = queue_rq_affinity_show,
700− .store = queue_rq_affinity_store,
701−};
702−
703−static struct queue_sysfs_entry queue_iostats_entry = {
704− .attr = {.name = "iostats", .mode = 0644 },
705− .show = queue_show_iostats,
706− .store = queue_store_iostats,
707−};
708−
709−static struct queue_sysfs_entry queue_random_entry = {
710− .attr = {.name = "add_random", .mode = 0644 },
711− .show = queue_show_random,
712− .store = queue_store_random,
713−};
714−
715−static struct queue_sysfs_entry queue_poll_entry = {
716− .attr = {.name = "io_poll", .mode = 0644 },
717− .show = queue_poll_show,
718− .store = queue_poll_store,
719−};
720−
721−static struct queue_sysfs_entry queue_poll_delay_entry = {
722− .attr = {.name = "io_poll_delay", .mode = 0644 },
723− .show = queue_poll_delay_show,
724− .store = queue_poll_delay_store,
725−};
726−
727−static struct queue_sysfs_entry queue_wc_entry = {
728− .attr = {.name = "write_cache", .mode = 0644 },
729− .show = queue_wc_show,
730− .store = queue_wc_store,
731−};
732−
733−static struct queue_sysfs_entry queue_fua_entry = {
734− .attr = {.name = "fua", .mode = 0444 },
735− .show = queue_fua_show,
736−};
737−
738−static struct queue_sysfs_entry queue_dax_entry = {
739− .attr = {.name = "dax", .mode = 0444 },
740− .show = queue_dax_show,
741−};
742−
743−static struct queue_sysfs_entry queue_io_timeout_entry = {
744− .attr = {.name = "io_timeout", .mode = 0644 },
745− .show = queue_io_timeout_show,
746− .store = queue_io_timeout_store,
747−};
748−
749−static struct queue_sysfs_entry queue_wb_lat_entry = {
750− .attr = {.name = "wbt_lat_usec", .mode = 0644 },
751− .show = queue_wb_lat_show,
752− .store = queue_wb_lat_store,
753−};
754−
755−#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
756−static struct queue_sysfs_entry throtl_sample_time_entry = {
757− .attr = {.name = "throttle_sample_time", .mode = 0644 },
758− .show = blk_throtl_sample_time_show,
759− .store = blk_throtl_sample_time_store,
760−};
761−#endif
614+QUEUE_RW_ENTRY(queue_nonrot, "rotational");
615+QUEUE_RW_ENTRY(queue_iostats, "iostats");
616+QUEUE_RW_ENTRY(queue_random, "add_random");
617+QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");
762618
763619 static struct attribute *queue_attrs[] = {
764620 &queue_requests_entry.attr,
@@ -769,7 +625,7 @@ static struct attribute *queue_attrs[] = { · 4 unmodified lines
769625 &queue_max_discard_segments_entry.attr,
770626 &queue_max_integrity_segments_entry.attr,
771627 &queue_max_segment_size_entry.attr,
772− &queue_iosched_entry.attr,
628+ &elv_iosched_entry.attr,
773629 &queue_hw_sector_size_entry.attr,
774630 &queue_logical_block_size_entry.attr,
775631 &queue_physical_block_size_entry.attr,
@@ -791,6 +647,7 @@ static struct attribute *queue_attrs[] = { · 15 unmodified lines
791647 &queue_nomerges_entry.attr,
792648 &queue_rq_affinity_entry.attr,
793649 &queue_iostats_entry.attr,
650+ &queue_stable_writes_entry.attr,
794651 &queue_random_entry.attr,
795652 &queue_poll_entry.attr,
796653 &queue_wc_entry.attr,
@@ -800,7 +657,7 @@ static struct attribute *queue_attrs[] = { · 3 unmodified lines
800657 &queue_poll_delay_entry.attr,
801658 &queue_io_timeout_entry.attr,
802659 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
803− &throtl_sample_time_entry.attr,
660+ &blk_throtl_sample_time_entry.attr,
804661 #endif
805662 NULL,
806663 };
@@ -1000,6 +857,8 @@ int blk_register_queue(struct gendisk *disk) · 193 unmodified lines
1000857 percpu_ref_switch_to_percpu(&q->q_usage_counter);
1001858 }
1002859
860+ blk_queue_update_readahead(q);
861+
1003862 ret = blk_trace_init_sysfs(dev);
1004863 if (ret)
1005864 return ret;
block/blk-throttle.c+36−23
@@ -15,10 +15,10 @@ · 14 unmodified lines
1515 #include "blk-cgroup-rwstat.h"
1616
1717 /* Max dispatch from a group in 1 round */
18−static int throtl_grp_quantum = 8;
18+#define THROTL_GRP_QUANTUM 8
1919
2020 /* Total max dispatch from all groups in one round */
21−static int throtl_quantum = 32;
21+#define THROTL_QUANTUM 32
2222
2323 /* Throttling is performed over a slice and after that slice is renewed */
2424 #define DFL_THROTL_SLICE_HD (HZ / 10)
@@ -150,7 +150,7 @@ struct throtl_grp { · 125 unmodified lines
150150 /* user configured IOPS limits */
151151 unsigned int iops_conf[2][LIMIT_CNT];
152152
153− /* Number of bytes disptached in current slice */
153+ /* Number of bytes dispatched in current slice */
154154 uint64_t bytes_disp[2];
155155 /* Number of bio's dispatched in current slice */
156156 unsigned int io_disp[2];
@@ -852,7 +852,7 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) · 695 unmodified lines
852852 /*
853853 * A bio has been dispatched. Also adjust slice_end. It might happen
854854 * that initially cgroup limit was very low resulting in high
855− * slice_end, but later limit was bumped up and bio was dispached
855+ * slice_end, but later limit was bumped up and bio was dispatched
856856 * sooner, then we need to reduce slice_end. A high bogus slice_end
857857 * is bad because it does not allow new slice to start.
858858 */
@@ -894,13 +894,19 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) · 35 unmodified lines
894894 }
895895
896896 static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,
897− unsigned long *wait)
897+ u32 iops_limit, unsigned long *wait)
898898 {
899899 bool rw = bio_data_dir(bio);
900900 unsigned int io_allowed;
901901 unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
902902 u64 tmp;
903903
904+ if (iops_limit == UINT_MAX) {
905+ if (wait)
906+ *wait = 0;
907+ return true;
908+ }
909+
904910 jiffy_elapsed = jiffies - tg->slice_start[rw];
905911
906912 /* Round up to the next throttle slice, wait time must be nonzero */
@@ -913,7 +919,7 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio, · 6 unmodified lines
913919 * have been trimmed.
914920 */
915921
916− tmp = (u64)tg_iops_limit(tg, rw) * jiffy_elapsed_rnd;
922+ tmp = (u64)iops_limit * jiffy_elapsed_rnd;
917923 do_div(tmp, HZ);
918924
919925 if (tmp > UINT_MAX)
@@ -936,13 +942,19 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio, · 16 unmodified lines
936942 }
937943
938944 static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
939− unsigned long *wait)
945+ u64 bps_limit, unsigned long *wait)
940946 {
941947 bool rw = bio_data_dir(bio);
942948 u64 bytes_allowed, extra_bytes, tmp;
943949 unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
944950 unsigned int bio_size = throtl_bio_data_size(bio);
945951
952+ if (bps_limit == U64_MAX) {
953+ if (wait)
954+ *wait = 0;
955+ return true;
956+ }
957+
946958 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];
947959
948960 /* Slice has just started. Consider one slice interval */
@@ -951,7 +963,7 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio, · 2 unmodified lines
951963
952964 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice);
953965
954− tmp = tg_bps_limit(tg, rw) * jiffy_elapsed_rnd;
966+ tmp = bps_limit * jiffy_elapsed_rnd;
955967 do_div(tmp, HZ);
956968 bytes_allowed = tmp;
957969
@@ -963,7 +975,7 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio, · 5 unmodified lines
963975
964976 /* Calc approx time to dispatch */
965977 extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed;
966− jiffy_wait = div64_u64(extra_bytes * HZ, tg_bps_limit(tg, rw));
978+ jiffy_wait = div64_u64(extra_bytes * HZ, bps_limit);
967979
968980 if (!jiffy_wait)
969981 jiffy_wait = 1;
@@ -987,6 +999,8 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, · 17 unmodified lines
987999 {
9881000 bool rw = bio_data_dir(bio);
9891001 unsigned long bps_wait = 0, iops_wait = 0, max_wait = 0;
1002+ u64 bps_limit = tg_bps_limit(tg, rw);
1003+ u32 iops_limit = tg_iops_limit(tg, rw);
9901004
9911005 /*
9921006 * Currently whole state machine of group depends on first bio
@@ -998,8 +1012,7 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, · 5 unmodified lines
9981012 bio != throtl_peek_queued(&tg->service_queue.queued[rw]));
9991013
10001014 /* If tg->bps = -1, then BW is unlimited */
1001− if (tg_bps_limit(tg, rw) == U64_MAX &&
1002− tg_iops_limit(tg, rw) == UINT_MAX) {
1015+ if (bps_limit == U64_MAX && iops_limit == UINT_MAX) {
10031016 if (wait)
10041017 *wait = 0;
10051018 return true;
@@ -1021,8 +1034,8 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, · 15 unmodified lines
10211034 jiffies + tg->td->throtl_slice);
10221035 }
10231036
1024− if (tg_with_in_bps_limit(tg, bio, &bps_wait) &&
1025− tg_with_in_iops_limit(tg, bio, &iops_wait)) {
1037+ if (tg_with_in_bps_limit(tg, bio, bps_limit, &bps_wait) &&
1038+ tg_with_in_iops_limit(tg, bio, iops_limit, &iops_wait)) {
10261039 if (wait)
10271040 *wait = 0;
10281041 return true;
@@ -1082,7 +1095,7 @@ static void throtl_add_bio_tg(struct bio *bio, struct throtl_qnode *qn, · 53 unmodified lines
10821095 * If @tg doesn't currently have any bios queued in the same
10831096 * direction, queueing @bio can change when @tg should be
10841097 * dispatched. Mark that @tg was empty. This is automatically
1085− * cleaered on the next tg_update_disptime().
1098+ * cleared on the next tg_update_disptime().
10861099 */
10871100 if (!sq->nr_queued[rw])
10881101 tg->flags |= THROTL_TG_WAS_EMPTY;
@@ -1175,8 +1188,8 @@ static int throtl_dispatch_tg(struct throtl_grp *tg) · 86 unmodified lines
11751188 {
11761189 struct throtl_service_queue *sq = &tg->service_queue;
11771190 unsigned int nr_reads = 0, nr_writes = 0;
1178− unsigned int max_nr_reads = throtl_grp_quantum*3/4;
1179− unsigned int max_nr_writes = throtl_grp_quantum - max_nr_reads;
1191+ unsigned int max_nr_reads = THROTL_GRP_QUANTUM * 3 / 4;
1192+ unsigned int max_nr_writes = THROTL_GRP_QUANTUM - max_nr_reads;
11801193 struct bio *bio;
11811194
11821195 /* Try to dispatch 75% READS and 25% WRITES */
@@ -1226,7 +1239,7 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq) · 43 unmodified lines
12261239 if (sq->nr_queued[0] || sq->nr_queued[1])
12271240 tg_update_disptime(tg);
12281241
1229− if (nr_disp >= throtl_quantum)
1242+ if (nr_disp >= THROTL_QUANTUM)
12301243 break;
12311244 }
12321245
@@ -1303,7 +1316,7 @@ again: · 70 unmodified lines
13031316 }
13041317 }
13051318 } else {
1306− /* reached the top-level, queue issueing */
1319+ /* reached the top-level, queue issuing */
13071320 queue_work(kthrotld_workqueue, &td->dispatch_work);
13081321 }
13091322 out_unlock:
@@ -1314,8 +1327,8 @@ out_unlock: · 4 unmodified lines
13141327 * blk_throtl_dispatch_work_fn - work function for throtl_data->dispatch_work
13151328 * @work: work item being executed
13161329 *
1317− * This function is queued for execution when bio's reach the bio_lists[]
1318− * of throtl_data->service_queue. Those bio's are ready and issued by this
1330+ * This function is queued for execution when bios reach the bio_lists[]
1331+ * of throtl_data->service_queue. Those bios are ready and issued by this
13191332 * function.
13201333 */
13211334 static void blk_throtl_dispatch_work_fn(struct work_struct *work)
@@ -1428,8 +1441,8 @@ static void tg_conf_updated(struct throtl_grp *tg, bool global) · 106 unmodified lines
14281441 * that a group's limit are dropped suddenly and we don't want to
14291442 * account recently dispatched IO with new low rate.
14301443 */
1431− throtl_start_new_slice(tg, 0);
1432− throtl_start_new_slice(tg, 1);
1444+ throtl_start_new_slice(tg, READ);
1445+ throtl_start_new_slice(tg, WRITE);
14331446
14341447 if (tg->flags & THROTL_TG_PENDING) {
14351448 tg_update_disptime(tg);
@@ -2230,7 +2243,7 @@ again: · 794 unmodified lines
22302243
22312244 /*
22322245 * @bio passed through this layer without being throttled.
2233− * Climb up the ladder. If we''re already at the top, it
2246+ * Climb up the ladder. If we're already at the top, it
22342247 * can be executed directly.
22352248 */
22362249 qn = &tg->qnode_on_parent[rw];
block/blk.h+18−7
@@ -29,6 +29,12 @@ struct blk_flush_queue { · 28 unmodified lines
2929 spinlock_t mq_flush_lock;
3030 };
3131
32+enum bio_merge_status {
33+ BIO_MERGE_OK,
34+ BIO_MERGE_NONE,
35+ BIO_MERGE_FAILED,
36+};
37+
3238 extern struct kmem_cache *blk_requestq_cachep;
3339 extern struct kobj_type blk_queue_ktype;
3440 extern struct ida blk_queue_ida;
@@ -169,14 +175,19 @@ static inline void blk_integrity_del(struct gendisk *disk) · 134 unmodified lines
169175 unsigned long blk_rq_timeout(unsigned long timeout);
170176 void blk_add_timer(struct request *req);
171177
172−bool bio_attempt_front_merge(struct request *req, struct bio *bio,
173− unsigned int nr_segs);
174−bool bio_attempt_back_merge(struct request *req, struct bio *bio,
175− unsigned int nr_segs);
176−bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
177− struct bio *bio);
178+enum bio_merge_status bio_attempt_front_merge(struct request *req,
179+ struct bio *bio,
180+ unsigned int nr_segs);
181+enum bio_merge_status bio_attempt_back_merge(struct request *req,
182+ struct bio *bio,
183+ unsigned int nr_segs);
184+enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
185+ struct request *req,
186+ struct bio *bio);
178187 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
179188 unsigned int nr_segs, struct request **same_queue_rq);
189+bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
190+ struct bio *bio, unsigned int nr_segs);
180191
181192 void blk_account_io_start(struct request *req);
182193 void blk_account_io_done(struct request *req, u64 now);
@@ -350,7 +361,7 @@ char *disk_name(struct gendisk *hd, int partno, char *buf); · 167 unmodified lines
350361 #define ADDPART_FLAG_NONE 0
351362 #define ADDPART_FLAG_RAID 1
352363 #define ADDPART_FLAG_WHOLEDISK 2
353−void delete_partition(struct gendisk *disk, struct hd_struct *part);
364+void delete_partition(struct hd_struct *part);
354365 int bdev_add_partition(struct block_device *bdev, int partno,
355366 sector_t start, sector_t length);
356367 int bdev_del_partition(struct block_device *bdev, int partno);
block/bsg-lib.c+1−1
@@ -207,7 +207,7 @@ static int bsg_map_buffer(struct bsg_buffer *buf, struct request *req) · 206 unmodified lines
207207
208208 BUG_ON(!req->nr_phys_segments);
209209
210− buf->sg_list = kzalloc(sz, GFP_KERNEL);
210+ buf->sg_list = kmalloc(sz, GFP_KERNEL);
211211 if (!buf->sg_list)
212212 return -ENOMEM;
213213 sg_init_table(buf->sg_list, req->nr_phys_segments);
block/genhd.c+89−67
@@ -50,14 +50,13 @@ static void disk_release_events(struct gendisk *disk); · 49 unmodified lines
5050 * zero and will not be set to zero
5151 */
5252 void set_capacity_revalidate_and_notify(struct gendisk *disk, sector_t size,
53− bool revalidate)
53+ bool update_bdev)
5454 {
5555 sector_t capacity = get_capacity(disk);
5656
5757 set_capacity(disk, size);
58−
59− if (revalidate)
60− revalidate_disk(disk);
58+ if (update_bdev)
59+ revalidate_disk_size(disk, true);
6160
6261 if (capacity != size && capacity != 0 && size != 0) {
6362 char *envp[] = { "RESIZE=1", NULL };
@@ -110,8 +109,7 @@ static void part_stat_read_all(struct hd_struct *part, struct disk_stats *stat) · 46 unmodified lines
110109 }
111110 }
112111
113−static unsigned int part_in_flight(struct request_queue *q,
114− struct hd_struct *part)
112+static unsigned int part_in_flight(struct hd_struct *part)
115113 {
116114 unsigned int inflight = 0;
117115 int cpu;
@@ -126,8 +124,7 @@ static unsigned int part_in_flight(struct request_queue *q, · 8 unmodified lines
126124 return inflight;
127125 }
128126
129−static void part_in_flight_rw(struct request_queue *q, struct hd_struct *part,
130− unsigned int inflight[2])
127+static void part_in_flight_rw(struct hd_struct *part, unsigned int inflight[2])
131128 {
132129 int cpu;
133130
@@ -676,11 +673,23 @@ static int exact_lock(dev_t devt, void *data) · 542 unmodified lines
676673 return 0;
677674 }
678675
676+static void disk_scan_partitions(struct gendisk *disk)
677+{
678+ struct block_device *bdev;
679+
680+ if (!get_capacity(disk) || !disk_part_scan_enabled(disk))
681+ return;
682+
683+ set_bit(GD_NEED_PART_SCAN, &disk->state);
684+ bdev = blkdev_get_by_dev(disk_devt(disk), FMODE_READ, NULL);
685+ if (!IS_ERR(bdev))
686+ blkdev_put(bdev, FMODE_READ);
687+}
688+
679689 static void register_disk(struct device *parent, struct gendisk *disk,
680690 const struct attribute_group **groups)
681691 {
682692 struct device *ddev = disk_to_dev(disk);
683− struct block_device *bdev;
684693 struct disk_part_iter piter;
685694 struct hd_struct *part;
686695 int err;
@@ -722,25 +731,8 @@ static void register_disk(struct device *parent, struct gendisk *disk, · 35 unmodified lines
722731 return;
723732 }
724733
725− /* No minors to use for partitions */
726− if (!disk_part_scan_enabled(disk))
727− goto exit;
734+ disk_scan_partitions(disk);
728735
729− /* No such device (e.g., media were just removed) */
730− if (!get_capacity(disk))
731− goto exit;
732−
733− bdev = bdget_disk(disk, 0);
734− if (!bdev)
735− goto exit;
736−
737− bdev->bd_invalidated = 1;
738− err = blkdev_get(bdev, FMODE_READ, NULL);
739− if (err < 0)
740− goto exit;
741− blkdev_put(bdev, FMODE_READ);
742−
743−exit:
744736 /* announce disk after possible partitions are created */
745737 dev_set_uevent_suppress(ddev, 0);
746738 kobject_uevent(&ddev->kobj, KOBJ_ADD);
@@ -913,7 +905,7 @@ void del_gendisk(struct gendisk *disk) · 166 unmodified lines
913905 DISK_PITER_INCL_EMPTY | DISK_PITER_REVERSE);
914906 while ((part = disk_part_iter_next(&piter))) {
915907 invalidate_partition(disk, part->partno);
916− delete_partition(disk, part);
908+ delete_partition(part);
917909 }
918910 disk_part_iter_exit(&piter);
919911
@@ -1301,7 +1293,7 @@ ssize_t part_stat_show(struct device *dev, · 381 unmodified lines
13011293 if (queue_is_mq(q))
13021294 inflight = blk_mq_in_flight(q, p);
13031295 else
1304− inflight = part_in_flight(q, p);
1296+ inflight = part_in_flight(p);
13051297
13061298 return sprintf(buf,
13071299 "%8lu %8lu %8llu %8u "
@@ -1343,7 +1335,7 @@ ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr, · 35 unmodified lines
13431335 if (queue_is_mq(q))
13441336 blk_mq_in_flight_rw(q, p, inflight);
13451337 else
1346− part_in_flight_rw(q, p, inflight);
1338+ part_in_flight_rw(p, inflight);
13471339
13481340 return sprintf(buf, "%8u %8u\n", inflight[0], inflight[1]);
13491341 }
@@ -1623,7 +1615,7 @@ static int diskstats_show(struct seq_file *seqf, void *v) · 273 unmodified lines
16231615 if (queue_is_mq(gp->queue))
16241616 inflight = blk_mq_in_flight(gp->queue, hd);
16251617 else
1626− inflight = part_in_flight(gp->queue, hd);
1618+ inflight = part_in_flight(hd);
16271619
16281620 seq_printf(seqf, "%4d %7d %s "
16291621 "%lu %lu %lu %u "
@@ -1729,45 +1721,48 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) · 99 unmodified lines
17291721 }
17301722
17311723 disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
1732− if (disk) {
1733− disk->part0.dkstats = alloc_percpu(struct disk_stats);
1734− if (!disk->part0.dkstats) {
1735− kfree(disk);
1736− return NULL;
1737− }
1738− init_rwsem(&disk->lookup_sem);
1739− disk->node_id = node_id;
1740− if (disk_expand_part_tbl(disk, 0)) {
1741− free_percpu(disk->part0.dkstats);
1742− kfree(disk);
1743− return NULL;
1744− }
1745− ptbl = rcu_dereference_protected(disk->part_tbl, 1);
1746− rcu_assign_pointer(ptbl->part[0], &disk->part0);
1724+ if (!disk)
1725+ return NULL;
17471726
1748− /*
1749− * set_capacity() and get_capacity() currently don't use
1750− * seqcounter to read/update the part0->nr_sects. Still init
1751− * the counter as we can read the sectors in IO submission
1752− * patch using seqence counters.
1753− *
1754− * TODO: Ideally set_capacity() and get_capacity() should be
1755− * converted to make use of bd_mutex and sequence counters.
1756− */
1757− hd_sects_seq_init(&disk->part0);
1758− if (hd_ref_init(&disk->part0)) {
1759− hd_free_part(&disk->part0);
1760− kfree(disk);
1761− return NULL;
1762− }
1727+ disk->part0.dkstats = alloc_percpu(struct disk_stats);
1728+ if (!disk->part0.dkstats)
1729+ goto out_free_disk;
17631730
1764− disk->minors = minors;
1765− rand_initialize_disk(disk);
1766− disk_to_dev(disk)->class = &block_class;
1767− disk_to_dev(disk)->type = &disk_type;
1768− device_initialize(disk_to_dev(disk));
1731+ init_rwsem(&disk->lookup_sem);
1732+ disk->node_id = node_id;
1733+ if (disk_expand_part_tbl(disk, 0)) {
1734+ free_percpu(disk->part0.dkstats);
1735+ goto out_free_disk;
17691736 }
1737+
1738+ ptbl = rcu_dereference_protected(disk->part_tbl, 1);
1739+ rcu_assign_pointer(ptbl->part[0], &disk->part0);
1740+
1741+ /*
1742+ * set_capacity() and get_capacity() currently don't use
1743+ * seqcounter to read/update the part0->nr_sects. Still init
1744+ * the counter as we can read the sectors in IO submission
1745+ * patch using seqence counters.
1746+ *
1747+ * TODO: Ideally set_capacity() and get_capacity() should be
1748+ * converted to make use of bd_mutex and sequence counters.
1749+ */
1750+ hd_sects_seq_init(&disk->part0);
1751+ if (hd_ref_init(&disk->part0))
1752+ goto out_free_part0;
1753+
1754+ disk->minors = minors;
1755+ rand_initialize_disk(disk);
1756+ disk_to_dev(disk)->class = &block_class;
1757+ disk_to_dev(disk)->type = &disk_type;
1758+ device_initialize(disk_to_dev(disk));
17701759 return disk;
1760+
1761+out_free_part0:
1762+ hd_free_part(&disk->part0);
1763+out_free_disk:
1764+ kfree(disk);
1765+ return NULL;
17711766 }
17721767 EXPORT_SYMBOL(__alloc_disk_node);
17731768
@@ -2052,7 +2047,7 @@ void disk_flush_events(struct gendisk *disk, unsigned int mask) · 278 unmodified lines
20522047 * CONTEXT:
20532048 * Might sleep.
20542049 */
2055−unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask)
2050+static unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask)
20562051 {
20572052 struct disk_events *ev = disk->ev;
20582053 unsigned int pending;
@@ -2090,6 +2085,33 @@ unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask) · 31 unmodified lines
20902085 return pending;
20912086 }
20922087
2088+/**
2089+ * bdev_check_media_change - check if a removable media has been changed
2090+ * @bdev: block device to check
2091+ *
2092+ * Check whether a removable media has been changed, and attempt to free all
2093+ * dentries and inodes and invalidates all block device page cache entries in
2094+ * that case.
2095+ *
2096+ * Returns %true if the block device changed, or %false if not.
2097+ */
2098+bool bdev_check_media_change(struct block_device *bdev)
2099+{
2100+ unsigned int events;
2101+
2102+ events = disk_clear_events(bdev->bd_disk, DISK_EVENT_MEDIA_CHANGE |
2103+ DISK_EVENT_EJECT_REQUEST);
2104+ if (!(events & DISK_EVENT_MEDIA_CHANGE))
2105+ return false;
2106+
2107+ if (__invalidate_device(bdev, true))
2108+ pr_warn("VFS: busy inodes on changed media %s\n",
2109+ bdev->bd_disk->disk_name);
2110+ set_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state);
2111+ return true;
2112+}
2113+EXPORT_SYMBOL(bdev_check_media_change);
2114+
20932115 /*
20942116 * Separate this part out so that a different pointer for clearing_ptr can be
20952117 * passed in for disk_clear_events.
block/ioctl.c+16−13
@@ -112,8 +112,7 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, · 111 unmodified lines
112112 uint64_t range[2];
113113 uint64_t start, len;
114114 struct request_queue *q = bdev_get_queue(bdev);
115− struct address_space *mapping = bdev->bd_inode->i_mapping;
116−
115+ int err;
117116
118117 if (!(mode & FMODE_WRITE))
119118 return -EBADF;
@@ -134,7 +133,11 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, · 14 unmodified lines
134133
135134 if (start + len > i_size_read(bdev->bd_inode))
136135 return -EINVAL;
137− truncate_inode_pages_range(mapping, start, start + len - 1);
136+
137+ err = truncate_bdev_range(bdev, mode, start, start + len - 1);
138+ if (err)
139+ return err;
140+
138141 return blkdev_issue_discard(bdev, start >> 9, len >> 9,
139142 GFP_KERNEL, flags);
140143 }
@@ -143,8 +146,8 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode, · 2 unmodified lines
143146 unsigned long arg)
144147 {
145148 uint64_t range[2];
146− struct address_space *mapping;
147149 uint64_t start, end, len;
150+ int err;
148151
149152 if (!(mode & FMODE_WRITE))
150153 return -EBADF;
@@ -166,8 +169,9 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode, · 15 unmodified lines
166169 return -EINVAL;
167170
168171 /* Invalidate the page cache, including dirty pages */
169− mapping = bdev->bd_inode->i_mapping;
170− truncate_inode_pages_range(mapping, start, end);
172+ err = truncate_bdev_range(bdev, mode, start, end);
173+ if (err)
174+ return err;
171175
172176 return blkdev_issue_zeroout(bdev, start >> 9, len >> 9, GFP_KERNEL,
173177 BLKDEV_ZERO_NOUNMAP);
@@ -474,15 +478,14 @@ static int blkdev_bszset(struct block_device *bdev, fmode_t mode, · 300 unmodified lines
474478 if (get_user(n, argp))
475479 return -EFAULT;
476480
477− if (!(mode & FMODE_EXCL)) {
478− bdgrab(bdev);
479− if (blkdev_get(bdev, mode | FMODE_EXCL, &bdev) < 0)
480− return -EBUSY;
481− }
481+ if (mode & FMODE_EXCL)
482+ return set_blocksize(bdev, n);
482483
484+ if (IS_ERR(blkdev_get_by_dev(bdev->bd_dev, mode | FMODE_EXCL, &bdev)))
485+ return -EBUSY;
483486 ret = set_blocksize(bdev, n);
484− if (!(mode & FMODE_EXCL))
485− blkdev_put(bdev, mode | FMODE_EXCL);
487+ blkdev_put(bdev, mode | FMODE_EXCL);
488+
486489 return ret;
487490 }
488491
block/ioprio.c+1−1
@@ -69,7 +69,7 @@ int ioprio_check_cap(int ioprio) · 68 unmodified lines
6969
7070 switch (class) {
7171 case IOPRIO_CLASS_RT:
72− if (!capable(CAP_SYS_ADMIN))
72+ if (!capable(CAP_SYS_NICE) && !capable(CAP_SYS_ADMIN))
7373 return -EPERM;
7474 fallthrough;
7575 /* rt has prio field too */
block/kyber-iosched.c+3−3
@@ -359,7 +359,7 @@ static unsigned int kyber_sched_tags_shift(struct request_queue *q) · 358 unmodified lines
359359 * All of the hardware queues have the same depth, so we can just grab
360360 * the shift of the first one.
361361 */
362− return q->queue_hw_ctx[0]->sched_tags->bitmap_tags.sb.shift;
362+ return q->queue_hw_ctx[0]->sched_tags->bitmap_tags->sb.shift;
363363 }
364364
365365 static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q)
@@ -502,7 +502,7 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) · 136 unmodified lines
502502 khd->batching = 0;
503503
504504 hctx->sched_data = khd;
505− sbitmap_queue_min_shallow_depth(&hctx->sched_tags->bitmap_tags,
505+ sbitmap_queue_min_shallow_depth(hctx->sched_tags->bitmap_tags,
506506 kqd->async_depth);
507507
508508 return 0;
@@ -573,7 +573,7 @@ static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio, · 64 unmodified lines
573573 bool merged;
574574
575575 spin_lock(&kcq->lock);
576− merged = blk_mq_bio_list_merge(hctx->queue, rq_list, bio, nr_segs);
576+ merged = blk_bio_list_merge(hctx->queue, rq_list, bio, nr_segs);
577577 spin_unlock(&kcq->lock);
578578
579579 return merged;
block/mq-deadline.c+6−0
@@ -386,6 +386,8 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) · 385 unmodified lines
386386 spin_lock(&dd->lock);
387387 rq = __dd_dispatch_request(dd);
388388 spin_unlock(&dd->lock);
389+ if (rq)
390+ atomic_dec(&rq->mq_hctx->elevator_queued);
389391
390392 return rq;
391393 }
@@ -533,6 +535,7 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, · 141 unmodified lines
533535 rq = list_first_entry(list, struct request, queuelist);
534536 list_del_init(&rq->queuelist);
535537 dd_insert_request(hctx, rq, at_head);
538+ atomic_inc(&hctx->elevator_queued);
536539 }
537540 spin_unlock(&dd->lock);
538541 }
@@ -579,6 +582,9 @@ static bool dd_has_work(struct blk_mq_hw_ctx *hctx) · 40 unmodified lines
579582 {
580583 struct deadline_data *dd = hctx->queue->elevator->elevator_data;
581584
585+ if (!atomic_read(&hctx->elevator_queued))
586+ return false;
587+
582588 return !list_empty_careful(&dd->dispatch) ||
583589 !list_empty_careful(&dd->fifo_list[0]) ||
584590 !list_empty_careful(&dd->fifo_list[1]);
block/partitions/core.c+15−12
@@ -199,14 +199,20 @@ static ssize_t part_alignment_offset_show(struct device *dev, · 198 unmodified lines
199199 struct device_attribute *attr, char *buf)
200200 {
201201 struct hd_struct *p = dev_to_part(dev);
202− return sprintf(buf, "%llu\n", (unsigned long long)p->alignment_offset);
202+
203+ return sprintf(buf, "%u\n",
204+ queue_limit_alignment_offset(&part_to_disk(p)->queue->limits,
205+ p->start_sect));
203206 }
204207
205208 static ssize_t part_discard_alignment_show(struct device *dev,
206209 struct device_attribute *attr, char *buf)
207210 {
208211 struct hd_struct *p = dev_to_part(dev);
209− return sprintf(buf, "%u\n", p->discard_alignment);
212+
213+ return sprintf(buf, "%u\n",
214+ queue_limit_discard_alignment(&part_to_disk(p)->queue->limits,
215+ p->start_sect));
210216 }
211217
212218 static DEVICE_ATTR(partition, 0444, part_partition_show, NULL);
@@ -318,8 +324,9 @@ int hd_ref_init(struct hd_struct *part) · 105 unmodified lines
318324 * Must be called either with bd_mutex held, before a disk can be opened or
319325 * after all disk users are gone.
320326 */
321−void delete_partition(struct gendisk *disk, struct hd_struct *part)
327+void delete_partition(struct hd_struct *part)
322328 {
329+ struct gendisk *disk = part_to_disk(part);
323330 struct disk_part_tbl *ptbl =
324331 rcu_dereference_protected(disk->part_tbl, 1);
325332
@@ -327,7 +334,7 @@ void delete_partition(struct gendisk *disk, struct hd_struct *part) · 1 unmodified line
327334 * ->part_tbl is referenced in this part's release handler, so
328335 * we have to hold the disk device
329336 */
330− get_device(disk_to_dev(part_to_disk(part)));
337+ get_device(disk_to_dev(disk));
331338 rcu_assign_pointer(ptbl->part[part->partno], NULL);
332339 kobject_put(part->holder_dir);
333340 device_del(part_to_dev(part));
@@ -405,10 +412,6 @@ static struct hd_struct *add_partition(struct gendisk *disk, int partno, · 71 unmodified lines
405412 pdev = part_to_dev(p);
406413
407414 p->start_sect = start;
408− p->alignment_offset =
409− queue_limit_alignment_offset(&disk->queue->limits, start);
410− p->discard_alignment =
411− queue_limit_discard_alignment(&disk->queue->limits, start);
412415 p->nr_sects = len;
413416 p->partno = partno;
414417 p->policy = get_disk_ro(disk);
@@ -554,7 +557,7 @@ int bdev_del_partition(struct block_device *bdev, int partno) · 139 unmodified lines
554557 sync_blockdev(bdevp);
555558 invalidate_bdev(bdevp);
556559
557− delete_partition(bdev->bd_disk, part);
560+ delete_partition(part);
558561 ret = 0;
559562 out_unlock:
560563 mutex_unlock(&bdev->bd_mutex);
@@ -592,8 +595,8 @@ int bdev_resize_partition(struct block_device *bdev, int partno, · 31 unmodified lines
592595 if (partition_overlaps(bdev->bd_disk, start, length, partno))
593596 goto out_unlock;
594597
595− part_nr_sects_write(part, (sector_t)length);
596− i_size_write(bdevp->bd_inode, length << SECTOR_SHIFT);
598+ part_nr_sects_write(part, length);
599+ bd_set_nr_sectors(bdevp, length);
597600
598601 ret = 0;
599602 out_unlock:
@@ -634,7 +637,7 @@ int blk_drop_partitions(struct block_device *bdev) · 34 unmodified lines
634637
635638 disk_part_iter_init(&piter, bdev->bd_disk, DISK_PITER_INCL_EMPTY);
636639 while ((part = disk_part_iter_next(&piter)))
637− delete_partition(bdev->bd_disk, part);
640+ delete_partition(part);
638641 disk_part_iter_exit(&piter);
639642
640643 return 0;
block/scsi_ioctl.c+0−2
@@ -37,8 +37,6 @@ const unsigned char scsi_command_size_tbl[8] = · 36 unmodified lines
3737 };
3838 EXPORT_SYMBOL(scsi_command_size_tbl);
3939
40−#include <scsi/sg.h>
41−
4240 static int sg_get_version(int __user *p)
4341 {
4442 static const int sg_version_num = 30527;
drivers/block/amiflop.c+1−1
@@ -1670,7 +1670,7 @@ static int floppy_open(struct block_device *bdev, fmode_t mode) · 1669 unmodified lines
16701670 }
16711671
16721672 if (mode & (FMODE_READ|FMODE_WRITE)) {
1673− check_disk_change(bdev);
1673+ bdev_check_media_change(bdev);
16741674 if (mode & FMODE_WRITE) {
16751675 int wrprot;
16761676
drivers/block/aoe/aoeblk.c+1−2
@@ -347,7 +347,6 @@ aoeblk_gdalloc(void *vp) · 346 unmodified lines
347347 mempool_t *mp;
348348 struct request_queue *q;
349349 struct blk_mq_tag_set *set;
350− enum { KB = 1024, MB = KB * KB, READ_AHEAD = 2 * MB, };
351350 ulong flags;
352351 int late = 0;
353352 int err;
@@ -407,7 +406,7 @@ aoeblk_gdalloc(void *vp) · 53 unmodified lines
407406 WARN_ON(d->gd);
408407 WARN_ON(d->flags & DEVFL_UP);
409408 blk_queue_max_hw_sectors(q, BLK_DEF_MAX_SECTORS);
410− q->backing_dev_info->ra_pages = READ_AHEAD / PAGE_SIZE;
409+ blk_queue_io_opt(q, SZ_2M);
411410 d->bufpool = mp;
412411 d->blkq = gd->queue = q;
413412 q->queuedata = d;
drivers/block/aoe/aoecmd.c+1−3
@@ -900,9 +900,7 @@ aoecmd_sleepwork(struct work_struct *work) · 899 unmodified lines
900900 ssize = get_capacity(d->gd);
901901 bd = bdget_disk(d->gd, 0);
902902 if (bd) {
903− inode_lock(bd->bd_inode);
904− i_size_write(bd->bd_inode, (loff_t)ssize<<9);
905− inode_unlock(bd->bd_inode);
903+ bd_set_nr_sectors(bd, ssize);
906904 bdput(bd);
907905 }
908906 spin_lock_irq(&d->lock);
drivers/block/ataflop.c+4−3
@@ -1732,7 +1732,8 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, · 1731 unmodified lines
17321732 /* invalidate the buffer track to force a reread */
17331733 BufferDrive = -1;
17341734 set_bit(drive, &fake_change);
1735− check_disk_change(bdev);
1735+ if (bdev_check_media_change(bdev))
1736+ floppy_revalidate(bdev->bd_disk);
17361737 return 0;
17371738 default:
17381739 return -EINVAL;
@@ -1909,7 +1910,8 @@ static int floppy_open(struct block_device *bdev, fmode_t mode) · 170 unmodified lines
19091910 return 0;
19101911
19111912 if (mode & (FMODE_READ|FMODE_WRITE)) {
1912− check_disk_change(bdev);
1913+ if (bdev_check_media_change(bdev))
1914+ floppy_revalidate(bdev->bd_disk);
19131915 if (mode & FMODE_WRITE) {
19141916 if (p->wpstat) {
19151917 if (p->ref < 0)
@@ -1953,7 +1955,6 @@ static const struct block_device_operations floppy_fops = { · 37 unmodified lines
19531955 .release = floppy_release,
19541956 .ioctl = fd_ioctl,
19551957 .check_events = floppy_check_events,
1956− .revalidate_disk= floppy_revalidate,
19571958 };
19581959
19591960 static const struct blk_mq_ops ataflop_mq_ops = {
drivers/block/brd.c+0−1
@@ -403,7 +403,6 @@ static struct brd_device *brd_alloc(int i) · 402 unmodified lines
403403 disk->flags = GENHD_FL_EXT_DEVT;
404404 sprintf(disk->disk_name, "ram%d", i);
405405 set_capacity(disk, rd_size * 2);
406− brd->brd_queue->backing_dev_info->capabilities |= BDI_CAP_SYNCHRONOUS_IO;
407406
408407 /* Tell the block layer that this is not a rotational device */
409408 blk_queue_flag_set(QUEUE_FLAG_NONROT, brd->brd_queue);
drivers/block/drbd/drbd_nl.c+1−15
@@ -1362,15 +1362,7 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi · 1361 unmodified lines
13621362
13631363 if (b) {
13641364 blk_stack_limits(&q->limits, &b->limits, 0);
1365−
1366− if (q->backing_dev_info->ra_pages !=
1367− b->backing_dev_info->ra_pages) {
1368− drbd_info(device, "Adjusting my ra_pages to backing device's (%lu -> %lu)\n",
1369− q->backing_dev_info->ra_pages,
1370− b->backing_dev_info->ra_pages);
1371− q->backing_dev_info->ra_pages =
1372− b->backing_dev_info->ra_pages;
1373− }
1365+ blk_queue_update_readahead(q);
13741366 }
13751367 fixup_discard_if_not_supported(q);
13761368 fixup_write_zeroes(device, q);
@@ -3370,7 +3362,6 @@ static void device_to_statistics(struct device_statistics *s, · 1993 unmodified lines
33703362 if (get_ldev(device)) {
33713363 struct drbd_md *md = &device->ldev->md;
33723364 u64 *history_uuids = (u64 *)s->history_uuids;
3373− struct request_queue *q;
33743365 int n;
33753366
33763367 spin_lock_irq(&md->uuid_lock);
@@ -3384,11 +3375,6 @@ static void device_to_statistics(struct device_statistics *s, · 7 unmodified lines
33843375 spin_unlock_irq(&md->uuid_lock);
33853376
33863377 s->dev_disk_flags = md->flags;
3387− q = bdev_get_queue(device->ldev->backing_bdev);
3388− s->dev_lower_blocked =
3389− bdi_congested(q->backing_dev_info,
3390− (1 << WB_async_congested) |
3391− (1 << WB_sync_congested));
33923378 put_ldev(device);
33933379 }
33943380 s->dev_size = drbd_get_capacity(device->this_bdev);
drivers/block/floppy.c+5−3
@@ -561,6 +561,7 @@ static void floppy_release_irq_and_dma(void); · 560 unmodified lines
561561 * output_byte is automatically disabled when reset is set.
562562 */
563563 static void reset_fdc(void);
564+static int floppy_revalidate(struct gendisk *disk);
564565
565566 /*
566567 * These are global variables, as that's the easiest way to give
@@ -3275,7 +3276,8 @@ static int invalidate_drive(struct block_device *bdev) · 2708 unmodified lines
32753276 /* invalidate the buffer track to force a reread */
32763277 set_bit((long)bdev->bd_disk->private_data, &fake_change);
32773278 process_fd_request();
3278− check_disk_change(bdev);
3279+ if (bdev_check_media_change(bdev))
3280+ floppy_revalidate(bdev->bd_disk);
32793281 return 0;
32803282 }
32813283
@@ -4123,7 +4125,8 @@ static int floppy_open(struct block_device *bdev, fmode_t mode) · 841 unmodified lines
41234125 drive_state[drive].last_checked = 0;
41244126 clear_bit(FD_OPEN_SHOULD_FAIL_BIT,
41254127 &drive_state[drive].flags);
4126− check_disk_change(bdev);
4128+ if (bdev_check_media_change(bdev))
4129+ floppy_revalidate(bdev->bd_disk);
41274130 if (test_bit(FD_DISK_CHANGED_BIT, &drive_state[drive].flags))
41284131 goto out;
41294132 if (test_bit(FD_OPEN_SHOULD_FAIL_BIT, &drive_state[drive].flags))
@@ -4291,7 +4294,6 @@ static const struct block_device_operations floppy_fops = { · 161 unmodified lines
42914294 .ioctl = fd_ioctl,
42924295 .getgeo = fd_getgeo,
42934296 .check_events = floppy_check_events,
4294− .revalidate_disk = floppy_revalidate,
42954297 #ifdef CONFIG_COMPAT
42964298 .compat_ioctl = fd_compat_ioctl,
42974299 #endif
drivers/block/loop.c+2−2
@@ -253,7 +253,7 @@ static void loop_set_size(struct loop_device *lo, loff_t size) · 252 unmodified lines
253253 {
254254 struct block_device *bdev = lo->lo_device;
255255
256− bd_set_size(bdev, size << SECTOR_SHIFT);
256+ bd_set_nr_sectors(bdev, size);
257257
258258 set_capacity_revalidate_and_notify(lo->lo_disk, size, false);
259259 }
@@ -1251,7 +1251,7 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) · 991 unmodified lines
12511251 set_capacity(lo->lo_disk, 0);
12521252 loop_sysfs_exit(lo);
12531253 if (bdev) {
1254− bd_set_size(bdev, 0);
1254+ bd_set_nr_sectors(bdev, 0);
12551255 /* let user-space know about this change */
12561256 kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
12571257 }
drivers/block/nbd.c+8−7
@@ -300,6 +300,7 @@ static void nbd_size_update(struct nbd_device *nbd) · 299 unmodified lines
300300 {
301301 struct nbd_config *config = nbd->config;
302302 struct block_device *bdev = bdget_disk(nbd->disk, 0);
303+ sector_t nr_sectors = config->bytesize >> 9;
303304
304305 if (config->flags & NBD_FLAG_SEND_TRIM) {
305306 nbd->disk->queue->limits.discard_granularity = config->blksize;
@@ -308,13 +309,13 @@ static void nbd_size_update(struct nbd_device *nbd) · 2 unmodified lines
308309 }
309310 blk_queue_logical_block_size(nbd->disk->queue, config->blksize);
310311 blk_queue_physical_block_size(nbd->disk->queue, config->blksize);
311− set_capacity(nbd->disk, config->bytesize >> 9);
312+ set_capacity(nbd->disk, nr_sectors);
312313 if (bdev) {
313314 if (bdev->bd_disk) {
314− bd_set_size(bdev, config->bytesize);
315+ bd_set_nr_sectors(bdev, nr_sectors);
315316 set_blocksize(bdev, config->blksize);
316317 } else
317− bdev->bd_invalidated = 1;
318+ set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);
318319 bdput(bdev);
319320 }
320321 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);
@@ -1138,7 +1139,7 @@ static void nbd_bdev_reset(struct block_device *bdev) · 817 unmodified lines
11381139 {
11391140 if (bdev->bd_openers > 1)
11401141 return;
1141− bd_set_size(bdev, 0);
1142+ bd_set_nr_sectors(bdev, 0);
11421143 }
11431144
11441145 static void nbd_parse_flags(struct nbd_device *nbd)
@@ -1321,7 +1322,7 @@ static int nbd_start_device_ioctl(struct nbd_device *nbd, struct block_device *b · 176 unmodified lines
13211322 return ret;
13221323
13231324 if (max_part)
1324− bdev->bd_invalidated = 1;
1325+ set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);
13251326 mutex_unlock(&nbd->config_lock);
13261327 ret = wait_event_interruptible(config->recv_wq,
13271328 atomic_read(&config->recv_threads) == 0);
@@ -1499,9 +1500,9 @@ static int nbd_open(struct block_device *bdev, fmode_t mode) · 171 unmodified lines
14991500 refcount_set(&nbd->config_refs, 1);
15001501 refcount_inc(&nbd->refs);
15011502 mutex_unlock(&nbd->config_lock);
1502− bdev->bd_invalidated = 1;
1503+ set_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state);
15031504 } else if (nbd_disconnected(nbd->config)) {
1504− bdev->bd_invalidated = 1;
1505+ set_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state);
15051506 }
15061507 out:
15071508 mutex_unlock(&nbd_index_mutex);
drivers/block/paride/pcd.c+1−1
@@ -233,7 +233,7 @@ static int pcd_block_open(struct block_device *bdev, fmode_t mode) · 232 unmodified lines
233233 struct pcd_unit *cd = bdev->bd_disk->private_data;
234234 int ret;
235235
236− check_disk_change(bdev);
236+ bdev_check_media_change(bdev);
237237
238238 mutex_lock(&pcd_mutex);
239239 ret = cdrom_open(&cd->info, bdev, mode);
drivers/block/pktcdvd.c+14−80
@@ -1082,65 +1082,6 @@ static void pkt_put_packet_data(struct pktcdvd_device *pd, struct packet_data *p · 1081 unmodified lines
10821082 }
10831083 }
10841084
1085−/*
1086− * recover a failed write, query for relocation if possible
1087− *
1088− * returns 1 if recovery is possible, or 0 if not
1089− *
1090− */
1091−static int pkt_start_recovery(struct packet_data *pkt)
1092−{
1093− /*
1094− * FIXME. We need help from the file system to implement
1095− * recovery handling.
1096− */
1097− return 0;
1098−#if 0
1099− struct request *rq = pkt->rq;
1100− struct pktcdvd_device *pd = rq->rq_disk->private_data;
1101− struct block_device *pkt_bdev;
1102− struct super_block *sb = NULL;
1103− unsigned long old_block, new_block;
1104− sector_t new_sector;
1105−
1106− pkt_bdev = bdget(kdev_t_to_nr(pd->pkt_dev));
1107− if (pkt_bdev) {
1108− sb = get_super(pkt_bdev);
1109− bdput(pkt_bdev);
1110− }
1111−
1112− if (!sb)
1113− return 0;
1114−
1115− if (!sb->s_op->relocate_blocks)
1116− goto out;
1117−
1118− old_block = pkt->sector / (CD_FRAMESIZE >> 9);
1119− if (sb->s_op->relocate_blocks(sb, old_block, &new_block))
1120− goto out;
1121−
1122− new_sector = new_block * (CD_FRAMESIZE >> 9);
1123− pkt->sector = new_sector;
1124−
1125− bio_reset(pkt->bio);
1126− bio_set_dev(pkt->bio, pd->bdev);
1127− bio_set_op_attrs(pkt->bio, REQ_OP_WRITE, 0);
1128− pkt->bio->bi_iter.bi_sector = new_sector;
1129− pkt->bio->bi_iter.bi_size = pkt->frames * CD_FRAMESIZE;
1130− pkt->bio->bi_vcnt = pkt->frames;
1131−
1132− pkt->bio->bi_end_io = pkt_end_io_packet_write;
1133− pkt->bio->bi_private = pkt;
1134−
1135− drop_super(sb);
1136− return 1;
1137−
1138−out:
1139− drop_super(sb);
1140− return 0;
1141−#endif
1142−}
1143−
11441085 static inline void pkt_set_state(struct packet_data *pkt, enum packet_data_state state)
11451086 {
11461087 #if PACKET_DEBUG > 1
@@ -1357,12 +1298,8 @@ static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data · 210 unmodified lines
13571298 break;
13581299
13591300 case PACKET_RECOVERY_STATE:
1360− if (pkt_start_recovery(pkt)) {
1361− pkt_start_write(pd, pkt);
1362− } else {
1363− pkt_dbg(2, pd, "No recovery possible\n");
1364− pkt_set_state(pkt, PACKET_FINISHED_STATE);
1365− }
1301+ pkt_dbg(2, pd, "No recovery possible\n");
1302+ pkt_set_state(pkt, PACKET_FINISHED_STATE);
13661303 break;
13671304
13681305 case PACKET_FINISHED_STATE:
@@ -2173,16 +2110,18 @@ static int pkt_open_dev(struct pktcdvd_device *pd, fmode_t write) · 804 unmodified lines
21732110 int ret;
21742111 long lba;
21752112 struct request_queue *q;
2113+ struct block_device *bdev;
21762114
21772115 /*
21782116 * We need to re-open the cdrom device without O_NONBLOCK to be able
21792117 * to read/write from/to it. It is already opened in O_NONBLOCK mode
2180− * so bdget() can't fail.
2118+ * so open should not fail.
21812119 */
2182− bdget(pd->bdev->bd_dev);
2183− ret = blkdev_get(pd->bdev, FMODE_READ | FMODE_EXCL, pd);
2184− if (ret)
2120+ bdev = blkdev_get_by_dev(pd->bdev->bd_dev, FMODE_READ | FMODE_EXCL, pd);
2121+ if (IS_ERR(bdev)) {
2122+ ret = PTR_ERR(bdev);
21852123 goto out;
2124+ }
21862125
21872126 ret = pkt_get_last_written(pd, &lba);
21882127 if (ret) {
@@ -2192,7 +2131,7 @@ static int pkt_open_dev(struct pktcdvd_device *pd, fmode_t write) · 3 unmodified lines
21922131
21932132 set_capacity(pd->disk, lba << 2);
21942133 set_capacity(pd->bdev->bd_disk, lba << 2);
2195− bd_set_size(pd->bdev, (loff_t)lba << 11);
2134+ bd_set_nr_sectors(pd->bdev, lba << 2);
21962135
21972136 q = bdev_get_queue(pd->bdev);
21982137 if (write) {
@@ -2226,7 +2165,7 @@ static int pkt_open_dev(struct pktcdvd_device *pd, fmode_t write) · 27 unmodified lines
22262165 return 0;
22272166
22282167 out_putdev:
2229− blkdev_put(pd->bdev, FMODE_READ | FMODE_EXCL);
2168+ blkdev_put(bdev, FMODE_READ | FMODE_EXCL);
22302169 out:
22312170 return ret;
22322171 }
@@ -2563,7 +2502,6 @@ static int pkt_seq_show(struct seq_file *m, void *p) · 330 unmodified lines
25632502 static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev)
25642503 {
25652504 int i;
2566− int ret = 0;
25672505 char b[BDEVNAME_SIZE];
25682506 struct block_device *bdev;
25692507
@@ -2586,12 +2524,9 @@ static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev) · 16 unmodified lines
25862524 }
25872525 }
25882526
2589− bdev = bdget(dev);
2590− if (!bdev)
2591− return -ENOMEM;
2592− ret = blkdev_get(bdev, FMODE_READ | FMODE_NDELAY, NULL);
2593− if (ret)
2594− return ret;
2527+ bdev = blkdev_get_by_dev(dev, FMODE_READ | FMODE_NDELAY, NULL);
2528+ if (IS_ERR(bdev))
2529+ return PTR_ERR(bdev);
25952530 if (!blk_queue_scsi_passthrough(bdev_get_queue(bdev))) {
25962531 blkdev_put(bdev, FMODE_READ | FMODE_NDELAY);
25972532 return -EINVAL;
@@ -2609,7 +2544,6 @@ static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev) · 11 unmodified lines
26092544 pd->cdrw.thread = kthread_run(kcdrwd, pd, "%s", pd->name);
26102545 if (IS_ERR(pd->cdrw.thread)) {
26112546 pkt_err(pd, "can't start kernel thread\n");
2612− ret = -ENOMEM;
26132547 goto out_mem;
26142548 }
26152549
@@ -2621,7 +2555,7 @@ out_mem: · 5 unmodified lines
26212555 blkdev_put(bdev, FMODE_READ | FMODE_NDELAY);
26222556 /* This is safe: open() is still holding a reference. */
26232557 module_put(THIS_MODULE);
2624− return ret;
2558+ return -ENOMEM;
26252559 }
26262560
26272561 static int pkt_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
drivers/block/rbd.c+2−2
@@ -4921,7 +4921,7 @@ static void rbd_dev_update_size(struct rbd_device *rbd_dev) · 4920 unmodified lines
49214921 size = (sector_t)rbd_dev->mapping.size / SECTOR_SIZE;
49224922 dout("setting size to %llu sectors", (unsigned long long)size);
49234923 set_capacity(rbd_dev->disk, size);
4924− revalidate_disk(rbd_dev->disk);
4924+ revalidate_disk_size(rbd_dev->disk, true);
49254925 }
49264926 }
49274927
@@ -5022,7 +5022,7 @@ static int rbd_init_disk(struct rbd_device *rbd_dev) · 94 unmodified lines
50225022 }
50235023
50245024 if (!ceph_test_opt(rbd_dev->rbd_client->client, NOCRC))
5025− q->backing_dev_info->capabilities |= BDI_CAP_STABLE_WRITES;
5025+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
50265026
50275027 /*
50285028 * disk_release() expects a queue ref from add_disk() and will
drivers/block/rnbd/rnbd-clt.c+3−9
@@ -102,18 +102,12 @@ static int rnbd_clt_set_dev_attr(struct rnbd_clt_dev *dev, · 101 unmodified lines
102102 static int rnbd_clt_change_capacity(struct rnbd_clt_dev *dev,
103103 size_t new_nsectors)
104104 {
105− int err = 0;
106−
107105 rnbd_clt_info(dev, "Device size changed from %zu to %zu sectors\n",
108106 dev->nsectors, new_nsectors);
109107 dev->nsectors = new_nsectors;
110108 set_capacity(dev->gd, dev->nsectors);
111− err = revalidate_disk(dev->gd);
112− if (err)
113− rnbd_clt_err(dev,
114− "Failed to change device size from %zu to %zu, err: %d\n",
115− dev->nsectors, new_nsectors, err);
116− return err;
109+ revalidate_disk_size(dev->gd, true);
110+ return 0;
117111 }
118112
119113 static int process_msg_open_rsp(struct rnbd_clt_dev *dev,
@@ -1180,7 +1174,7 @@ static int setup_mq_tags(struct rnbd_clt_session *sess) · 1060 unmodified lines
11801174 tag_set->queue_depth = sess->queue_depth;
11811175 tag_set->numa_node = NUMA_NO_NODE;
11821176 tag_set->flags = BLK_MQ_F_SHOULD_MERGE |
1183− BLK_MQ_F_TAG_SHARED;
1177+ BLK_MQ_F_TAG_QUEUE_SHARED;
11841178 tag_set->cmd_size = sizeof(struct rnbd_iu);
11851179 tag_set->nr_hw_queues = num_online_cpus();
11861180
drivers/block/swim.c+2−20
@@ -638,7 +638,8 @@ static int floppy_open(struct block_device *bdev, fmode_t mode) · 637 unmodified lines
638638 return 0;
639639
640640 if (mode & (FMODE_READ|FMODE_WRITE)) {
641− check_disk_change(bdev);
641+ if (bdev_check_media_change(bdev) && fs->disk_in)
642+ fs->ejected = 0;
642643 if ((mode & FMODE_WRITE) && fs->write_protected) {
643644 err = -EROFS;
644645 goto out;
@@ -735,24 +736,6 @@ static unsigned int floppy_check_events(struct gendisk *disk, · 90 unmodified lines
735736 return fs->ejected ? DISK_EVENT_MEDIA_CHANGE : 0;
736737 }
737738
738−static int floppy_revalidate(struct gendisk *disk)
739−{
740− struct floppy_state *fs = disk->private_data;
741− struct swim __iomem *base = fs->swd->base;
742−
743− swim_drive(base, fs->location);
744−
745− if (fs->ejected)
746− setup_medium(fs);
747−
748− if (!fs->disk_in)
749− swim_motor(base, OFF);
750− else
751− fs->ejected = 0;
752−
753− return !fs->disk_in;
754−}
755−
756739 static const struct block_device_operations floppy_fops = {
757740 .owner = THIS_MODULE,
758741 .open = floppy_unlocked_open,
@@ -760,7 +743,6 @@ static const struct block_device_operations floppy_fops = { · 1 unmodified line
760743 .ioctl = floppy_ioctl,
761744 .getgeo = floppy_getgeo,
762745 .check_events = floppy_check_events,
763− .revalidate_disk = floppy_revalidate,
764746 };
765747
766748 static struct kobject *floppy_find(dev_t dev, int *part, void *data)
drivers/block/swim3.c+2−2
@@ -945,7 +945,8 @@ static int floppy_open(struct block_device *bdev, fmode_t mode) · 944 unmodified lines
945945
946946 if (err == 0 && (mode & FMODE_NDELAY) == 0
947947 && (mode & (FMODE_READ|FMODE_WRITE))) {
948− check_disk_change(bdev);
948+ if (bdev_check_media_change(bdev))
949+ floppy_revalidate(bdev->bd_disk);
949950 if (fs->ejected)
950951 err = -ENXIO;
951952 }
@@ -1055,7 +1056,6 @@ static const struct block_device_operations floppy_fops = { · 103 unmodified lines
10551056 .release = floppy_release,
10561057 .ioctl = floppy_ioctl,
10571058 .check_events = floppy_check_events,
1058− .revalidate_disk= floppy_revalidate,
10591059 };
10601060
10611061 static const struct blk_mq_ops swim3_mq_ops = {
drivers/block/virtio_blk.c+2−2
@@ -598,7 +598,7 @@ static void virtblk_update_cache_mode(struct virtio_device *vdev) · 597 unmodified lines
598598 struct virtio_blk *vblk = vdev->priv;
599599
600600 blk_queue_write_cache(vblk->disk->queue, writeback, false);
601− revalidate_disk(vblk->disk);
601+ revalidate_disk_size(vblk->disk, true);
602602 }
603603
604604 static const char *const virtblk_cache_types[] = {
@@ -646,7 +646,7 @@ static struct attribute *virtblk_attrs[] = { · 41 unmodified lines
646646 static umode_t virtblk_attrs_are_visible(struct kobject *kobj,
647647 struct attribute *a, int n)
648648 {
649− struct device *dev = container_of(kobj, struct device, kobj);
649+ struct device *dev = kobj_to_dev(kobj);
650650 struct gendisk *disk = dev_to_disk(dev);
651651 struct virtio_blk *vblk = disk->private_data;
652652 struct virtio_device *vdev = vblk->vdev;
drivers/block/xsysace.c+10−16
@@ -888,26 +888,20 @@ static unsigned int ace_check_events(struct gendisk *gd, unsigned int clearing) · 887 unmodified lines
888888 return ace->media_change ? DISK_EVENT_MEDIA_CHANGE : 0;
889889 }
890890
891−static int ace_revalidate_disk(struct gendisk *gd)
891+static void ace_media_changed(struct ace_device *ace)
892892 {
893− struct ace_device *ace = gd->private_data;
894893 unsigned long flags;
895894
896− dev_dbg(ace->dev, "ace_revalidate_disk()\n");
897−
898− if (ace->media_change) {
899− dev_dbg(ace->dev, "requesting cf id and scheduling tasklet\n");
895+ dev_dbg(ace->dev, "requesting cf id and scheduling tasklet\n");
900896
901− spin_lock_irqsave(&ace->lock, flags);
902− ace->id_req_count++;
903− spin_unlock_irqrestore(&ace->lock, flags);
897+ spin_lock_irqsave(&ace->lock, flags);
898+ ace->id_req_count++;
899+ spin_unlock_irqrestore(&ace->lock, flags);
904900
905− tasklet_schedule(&ace->fsm_tasklet);
906− wait_for_completion(&ace->id_completion);
907− }
901+ tasklet_schedule(&ace->fsm_tasklet);
902+ wait_for_completion(&ace->id_completion);
908903
909904 dev_dbg(ace->dev, "revalidate complete\n");
910− return ace->id_result;
911905 }
912906
913907 static int ace_open(struct block_device *bdev, fmode_t mode)
@@ -922,7 +916,8 @@ static int ace_open(struct block_device *bdev, fmode_t mode) · 8 unmodified lines
922916 ace->users++;
923917 spin_unlock_irqrestore(&ace->lock, flags);
924918
925− check_disk_change(bdev);
919+ if (bdev_check_media_change(bdev) && ace->media_change)
920+ ace_media_changed(ace);
926921 mutex_unlock(&xsysace_mutex);
927922
928923 return 0;
@@ -966,7 +961,6 @@ static const struct block_device_operations ace_fops = { · 37 unmodified lines
966961 .open = ace_open,
967962 .release = ace_release,
968963 .check_events = ace_check_events,
969− .revalidate_disk = ace_revalidate_disk,
970964 .getgeo = ace_getgeo,
971965 };
972966
@@ -1080,7 +1074,7 @@ static int ace_setup(struct ace_device *ace) · 107 unmodified lines
10801074 (unsigned long long) ace->physaddr, ace->baseaddr, ace->irq);
10811075
10821076 ace->media_change = 1;
1083− ace_revalidate_disk(ace->gd);
1077+ ace_media_changed(ace);
10841078
10851079 /* Make the sysace device 'live' */
10861080 add_disk(ace->gd);
drivers/block/zram/zram_drv.c+19−11
@@ -52,6 +52,9 @@ static unsigned int num_devices = 1; · 51 unmodified lines
5252 */
5353 static size_t huge_class_size;
5454
55+static const struct block_device_operations zram_devops;
56+static const struct block_device_operations zram_wb_devops;
57+
5558 static void zram_free_page(struct zram *zram, size_t index);
5659 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
5760 u32 index, int offset, struct bio *bio);
@@ -408,8 +411,7 @@ static void reset_bdev(struct zram *zram) · 350 unmodified lines
408411 zram->backing_dev = NULL;
409412 zram->old_block_size = 0;
410413 zram->bdev = NULL;
411− zram->disk->queue->backing_dev_info->capabilities |=
412− BDI_CAP_SYNCHRONOUS_IO;
414+ zram->disk->fops = &zram_devops;
413415 kvfree(zram->bitmap);
414416 zram->bitmap = NULL;
415417 }
@@ -491,9 +493,10 @@ static ssize_t backing_dev_store(struct device *dev, · 75 unmodified lines
491493 goto out;
492494 }
493495
494− bdev = bdgrab(I_BDEV(inode));
495− err = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
496− if (err < 0) {
496+ bdev = blkdev_get_by_dev(inode->i_rdev,
497+ FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
498+ if (IS_ERR(bdev)) {
499+ err = PTR_ERR(bdev);
497500 bdev = NULL;
498501 goto out;
499502 }
@@ -528,8 +531,7 @@ static ssize_t backing_dev_store(struct device *dev, · 28 unmodified lines
528531 * freely but in fact, IO is going on so finally could cause
529532 * use-after-free when the IO is really done.
530533 */
531− zram->disk->queue->backing_dev_info->capabilities &=
532− ~BDI_CAP_SYNCHRONOUS_IO;
534+ zram->disk->fops = &zram_wb_devops;
533535 up_write(&zram->init_lock);
534536
535537 pr_info("setup backing device %s\n", file_name);
@@ -1739,7 +1741,7 @@ static ssize_t disksize_store(struct device *dev, · 1203 unmodified lines
17391741 zram->disksize = disksize;
17401742 set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
17411743
1742− revalidate_disk(zram->disk);
1744+ revalidate_disk_size(zram->disk, true);
17431745 up_write(&zram->init_lock);
17441746
17451747 return len;
@@ -1786,7 +1788,7 @@ static ssize_t reset_store(struct device *dev, · 40 unmodified lines
17861788 /* Make sure all the pending I/O are finished */
17871789 fsync_bdev(bdev);
17881790 zram_reset_device(zram);
1789− revalidate_disk(zram->disk);
1791+ revalidate_disk_size(zram->disk, true);
17901792 bdput(bdev);
17911793
17921794 mutex_lock(&bdev->bd_mutex);
@@ -1819,6 +1821,13 @@ static const struct block_device_operations zram_devops = { · 26 unmodified lines
18191821 .owner = THIS_MODULE
18201822 };
18211823
1824+static const struct block_device_operations zram_wb_devops = {
1825+ .open = zram_open,
1826+ .submit_bio = zram_submit_bio,
1827+ .swap_slot_free_notify = zram_slot_free_notify,
1828+ .owner = THIS_MODULE
1829+};
1830+
18221831 static DEVICE_ATTR_WO(compact);
18231832 static DEVICE_ATTR_RW(disksize);
18241833 static DEVICE_ATTR_RO(initstate);
@@ -1946,8 +1955,7 @@ static int zram_add(void) · 121 unmodified lines
19461955 if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
19471956 blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX);
19481957
1949− zram->disk->queue->backing_dev_info->capabilities |=
1950− (BDI_CAP_STABLE_WRITES | BDI_CAP_SYNCHRONOUS_IO);
1958+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, zram->disk->queue);
19511959 device_add_disk(NULL, zram->disk, zram_disk_attr_groups);
19521960
19531961 strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
drivers/cdrom/gdrom.c+1−1
@@ -479,7 +479,7 @@ static int gdrom_bdops_open(struct block_device *bdev, fmode_t mode) · 478 unmodified lines
479479 {
480480 int ret;
481481
482− check_disk_change(bdev);
482+ bdev_check_media_change(bdev);
483483
484484 mutex_lock(&gdrom_mutex);
485485 ret = cdrom_open(gd.cd_info, bdev, mode);
drivers/char/raw.c+24−32
@@ -28,7 +28,8 @@ · 27 unmodified lines
2828 #include <linux/uaccess.h>
2929
3030 struct raw_device_data {
31− struct block_device *binding;
31+ dev_t binding;
32+ struct block_device *bdev;
3233 int inuse;
3334 };
3435
@@ -63,19 +64,25 @@ static int raw_open(struct inode *inode, struct file *filp) · 28 unmodified lines
6364 return 0;
6465 }
6566
67+ pr_warn_ratelimited(
68+ "process %s (pid %d) is using the deprecated raw device\n"
69+ "support will be removed in Linux 5.14.\n",
70+ current->comm, current->pid);
71+
6672 mutex_lock(&raw_mutex);
6773
6874 /*
6975 * All we need to do on open is check that the device is bound.
7076 */
71− bdev = raw_devices[minor].binding;
7277 err = -ENODEV;
73− if (!bdev)
78+ if (!raw_devices[minor].binding)
7479 goto out;
75− bdgrab(bdev);
76− err = blkdev_get(bdev, filp->f_mode | FMODE_EXCL, raw_open);
77− if (err)
80+ bdev = blkdev_get_by_dev(raw_devices[minor].binding,
81+ filp->f_mode | FMODE_EXCL, raw_open);
82+ if (IS_ERR(bdev)) {
83+ err = PTR_ERR(bdev);
7884 goto out;
85+ }
7986 err = set_blocksize(bdev, bdev_logical_block_size(bdev));
8087 if (err)
8188 goto out1;
@@ -85,6 +92,7 @@ static int raw_open(struct inode *inode, struct file *filp) · 3 unmodified lines
8592 file_inode(filp)->i_mapping =
8693 bdev->bd_inode->i_mapping;
8794 filp->private_data = bdev;
95+ raw_devices[minor].bdev = bdev;
8896 mutex_unlock(&raw_mutex);
8997 return 0;
9098
@@ -105,7 +113,7 @@ static int raw_release(struct inode *inode, struct file *filp) · 14 unmodified lines
105113 struct block_device *bdev;
106114
107115 mutex_lock(&raw_mutex);
108− bdev = raw_devices[minor].binding;
116+ bdev = raw_devices[minor].bdev;
109117 if (--raw_devices[minor].inuse == 0)
110118 /* Here inode->i_mapping == bdev->bd_inode->i_mapping */
111119 inode->i_mapping = &inode->i_data;
@@ -128,6 +136,7 @@ raw_ioctl(struct file *filp, unsigned int command, unsigned long arg) · 16 unmodified lines
128136 static int bind_set(int number, u64 major, u64 minor)
129137 {
130138 dev_t dev = MKDEV(major, minor);
139+ dev_t raw = MKDEV(RAW_MAJOR, number);
131140 struct raw_device_data *rawdev;
132141 int err = 0;
133142
@@ -161,25 +170,17 @@ static int bind_set(int number, u64 major, u64 minor) · 27 unmodified lines
161170 mutex_unlock(&raw_mutex);
162171 return -EBUSY;
163172 }
164− if (rawdev->binding) {
165− bdput(rawdev->binding);
173+ if (rawdev->binding)
166174 module_put(THIS_MODULE);
167− }
175+
176+ rawdev->binding = dev;
168177 if (!dev) {
169178 /* unbind */
170− rawdev->binding = NULL;
171− device_destroy(raw_class, MKDEV(RAW_MAJOR, number));
179+ device_destroy(raw_class, raw);
172180 } else {
173− rawdev->binding = bdget(dev);
174− if (rawdev->binding == NULL) {
175− err = -ENOMEM;
176− } else {
177− dev_t raw = MKDEV(RAW_MAJOR, number);
178− __module_get(THIS_MODULE);
179− device_destroy(raw_class, raw);
180− device_create(raw_class, NULL, raw, NULL,
181− "raw%d", number);
182− }
181+ __module_get(THIS_MODULE);
182+ device_destroy(raw_class, raw);
183+ device_create(raw_class, NULL, raw, NULL, "raw%d", number);
183184 }
184185 mutex_unlock(&raw_mutex);
185186 return err;
@@ -187,18 +188,9 @@ static int bind_set(int number, u64 major, u64 minor) · 1 unmodified line
187188
188189 static int bind_get(int number, dev_t *dev)
189190 {
190− struct raw_device_data *rawdev;
191− struct block_device *bdev;
192−
193191 if (number <= 0 || number >= max_raw_minors)
194192 return -EINVAL;
195−
196− rawdev = &raw_devices[number];
197−
198− mutex_lock(&raw_mutex);
199− bdev = rawdev->binding;
200− *dev = bdev ? bdev->bd_dev : 0;
201− mutex_unlock(&raw_mutex);
193+ *dev = raw_devices[number].binding;
202194 return 0;
203195 }
204196
drivers/ide/ide-cd.c+5−11
@@ -1611,7 +1611,11 @@ static int idecd_open(struct block_device *bdev, fmode_t mode) · 1610 unmodified lines
16111611 struct cdrom_info *info;
16121612 int rc = -ENXIO;
16131613
1614− check_disk_change(bdev);
1614+ if (bdev_check_media_change(bdev)) {
1615+ info = ide_drv_g(bdev->bd_disk, cdrom_info);
1616+
1617+ ide_cd_read_toc(info->drive);
1618+ }
16151619
16161620 mutex_lock(&ide_cd_mutex);
16171621 info = ide_cd_get(bdev->bd_disk);
@@ -1753,15 +1757,6 @@ static unsigned int idecd_check_events(struct gendisk *disk, · 135 unmodified lines
17531757 return cdrom_check_events(&info->devinfo, clearing);
17541758 }
17551759
1756−static int idecd_revalidate_disk(struct gendisk *disk)
1757−{
1758− struct cdrom_info *info = ide_drv_g(disk, cdrom_info);
1759−
1760− ide_cd_read_toc(info->drive);
1761−
1762− return 0;
1763−}
1764−
17651760 static const struct block_device_operations idecd_ops = {
17661761 .owner = THIS_MODULE,
17671762 .open = idecd_open,
@@ -1770,7 +1765,6 @@ static const struct block_device_operations idecd_ops = { · 2 unmodified lines
17701765 .compat_ioctl = IS_ENABLED(CONFIG_COMPAT) ?
17711766 idecd_compat_ioctl : NULL,
17721767 .check_events = idecd_check_events,
1773− .revalidate_disk = idecd_revalidate_disk
17741768 };
17751769
17761770 /* module options */
drivers/ide/ide-disk.c+1−4
@@ -739,12 +739,9 @@ static void ide_disk_setup(ide_drive_t *drive) · 738 unmodified lines
739739 set_wcache(drive, 1);
740740
741741 if ((drive->dev_flags & IDE_DFLAG_LBA) == 0 &&
742− (drive->head == 0 || drive->head > 16)) {
742+ (drive->head == 0 || drive->head > 16))
743743 printk(KERN_ERR "%s: invalid geometry: %d physical heads?\n",
744744 drive->name, drive->head);
745− drive->dev_flags &= ~IDE_DFLAG_ATTACH;
746− } else
747− drive->dev_flags |= IDE_DFLAG_ATTACH;
748745 }
749746
750747 static void ide_disk_flush(ide_drive_t *drive)
drivers/ide/ide-floppy.c+0−2
@@ -516,8 +516,6 @@ static void ide_floppy_setup(ide_drive_t *drive) · 515 unmodified lines
516516 (void) ide_floppy_get_capacity(drive);
517517
518518 ide_proc_register_driver(drive, floppy->driver);
519−
520− drive->dev_flags |= IDE_DFLAG_ATTACH;
521519 }
522520
523521 static void ide_floppy_flush(ide_drive_t *drive)
drivers/ide/ide-gd.c+6−42
@@ -225,8 +225,12 @@ static int ide_gd_open(struct block_device *bdev, fmode_t mode) · 224 unmodified lines
225225 * and the door_lock is irrelevant at this point.
226226 */
227227 drive->disk_ops->set_doorlock(drive, disk, 1);
228− drive->dev_flags |= IDE_DFLAG_MEDIA_CHANGED;
229− check_disk_change(bdev);
228+ if (__invalidate_device(bdev, true))
229+ pr_warn("VFS: busy inodes on changed media %s\n",
230+ bdev->bd_disk->disk_name);
231+ drive->disk_ops->get_capacity(drive);
232+ set_capacity(disk, ide_gd_capacity(drive));
233+ set_bit(GD_NEED_PART_SCAN, &disk->state);
230234 } else if (drive->dev_flags & IDE_DFLAG_FORMAT_IN_PROGRESS) {
231235 ret = -EBUSY;
232236 goto out_put_idkp;
@@ -284,32 +288,6 @@ static int ide_gd_getgeo(struct block_device *bdev, struct hd_geometry *geo) · 51 unmodified lines
284288 return 0;
285289 }
286290
287−static unsigned int ide_gd_check_events(struct gendisk *disk,
288− unsigned int clearing)
289−{
290− struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj);
291− ide_drive_t *drive = idkp->drive;
292− bool ret;
293−
294− /* do not scan partitions twice if this is a removable device */
295− if (drive->dev_flags & IDE_DFLAG_ATTACH) {
296− drive->dev_flags &= ~IDE_DFLAG_ATTACH;
297− return 0;
298− }
299−
300− /*
301− * The following is used to force revalidation on the first open on
302− * removeable devices, and never gets reported to userland as
303− * DISK_EVENT_FLAG_UEVENT isn't set in genhd->event_flags.
304− * This is intended as removable ide disk can't really detect
305− * MEDIA_CHANGE events.
306− */
307− ret = drive->dev_flags & IDE_DFLAG_MEDIA_CHANGED;
308− drive->dev_flags &= ~IDE_DFLAG_MEDIA_CHANGED;
309−
310− return ret ? DISK_EVENT_MEDIA_CHANGE : 0;
311−}
312−
313291 static void ide_gd_unlock_native_capacity(struct gendisk *disk)
314292 {
315293 struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj);
@@ -320,18 +298,6 @@ static void ide_gd_unlock_native_capacity(struct gendisk *disk) · 4 unmodified lines
320298 disk_ops->unlock_native_capacity(drive);
321299 }
322300
323−static int ide_gd_revalidate_disk(struct gendisk *disk)
324−{
325− struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj);
326− ide_drive_t *drive = idkp->drive;
327−
328− if (ide_gd_check_events(disk, 0))
329− drive->disk_ops->get_capacity(drive);
330−
331− set_capacity(disk, ide_gd_capacity(drive));
332− return 0;
333−}
334−
335301 static int ide_gd_ioctl(struct block_device *bdev, fmode_t mode,
336302 unsigned int cmd, unsigned long arg)
337303 {
@@ -364,9 +330,7 @@ static const struct block_device_operations ide_gd_ops = { · 26 unmodified lines
364330 .compat_ioctl = ide_gd_compat_ioctl,
365331 #endif
366332 .getgeo = ide_gd_getgeo,
367− .check_events = ide_gd_check_events,
368333 .unlock_native_capacity = ide_gd_unlock_native_capacity,
369− .revalidate_disk = ide_gd_revalidate_disk
370334 };
371335
372336 static int ide_gd_probe(ide_drive_t *drive)
drivers/md/bcache/request.c+6−4
@@ -475,6 +475,7 @@ struct search { · 474 unmodified lines
475475 unsigned int read_dirty_data:1;
476476 unsigned int cache_missed:1;
477477
478+ struct hd_struct *part;
478479 unsigned long start_time;
479480
480481 struct btree_op op;
@@ -669,7 +670,7 @@ static void bio_complete(struct search *s) · 188 unmodified lines
669670 {
670671 if (s->orig_bio) {
671672 /* Count on bcache device */
672− disk_end_io_acct(s->d->disk, bio_op(s->orig_bio), s->start_time);
673+ part_end_io_acct(s->part, s->orig_bio, s->start_time);
673674
674675 trace_bcache_request_end(s->d, s->orig_bio);
675676 s->orig_bio->bi_status = s->iop.status;
@@ -731,7 +732,7 @@ static inline struct search *search_alloc(struct bio *bio, · 55 unmodified lines
731732 s->write = op_is_write(bio_op(bio));
732733 s->read_dirty_data = 0;
733734 /* Count on the bcache device */
734− s->start_time = disk_start_io_acct(d->disk, bio_sectors(bio), bio_op(bio));
735+ s->start_time = part_start_io_acct(d->disk, &s->part, bio);
735736 s->iop.c = d->c;
736737 s->iop.bio = NULL;
737738 s->iop.inode = d->id;
@@ -1072,6 +1073,7 @@ struct detached_dev_io_private { · 334 unmodified lines
10721073 unsigned long start_time;
10731074 bio_end_io_t *bi_end_io;
10741075 void *bi_private;
1076+ struct hd_struct *part;
10751077 };
10761078
10771079 static void detached_dev_end_io(struct bio *bio)
@@ -1083,7 +1085,7 @@ static void detached_dev_end_io(struct bio *bio) · 5 unmodified lines
10831085 bio->bi_private = ddip->bi_private;
10841086
10851087 /* Count on the bcache device */
1086− disk_end_io_acct(ddip->d->disk, bio_op(bio), ddip->start_time);
1088+ part_end_io_acct(ddip->part, bio, ddip->start_time);
10871089
10881090 if (bio->bi_status) {
10891091 struct cached_dev *dc = container_of(ddip->d,
@@ -1109,7 +1111,7 @@ static void detached_dev_do_request(struct bcache_device *d, struct bio *bio) · 19 unmodified lines
11091111 ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO);
11101112 ddip->d = d;
11111113 /* Count on the bcache device */
1112− ddip->start_time = disk_start_io_acct(d->disk, bio_sectors(bio), bio_op(bio));
1114+ ddip->start_time = part_start_io_acct(d->disk, &ddip->part, bio);
11131115 ddip->bi_end_io = bio->bi_end_io;
11141116 ddip->bi_private = bio->bi_private;
11151117 bio->bi_end_io = detached_dev_end_io;
drivers/md/bcache/super.c+2−3
@@ -1427,9 +1427,8 @@ static int cached_dev_init(struct cached_dev *dc, unsigned int block_size) · 1426 unmodified lines
14271427 if (ret)
14281428 return ret;
14291429
1430− dc->disk.disk->queue->backing_dev_info->ra_pages =
1431− max(dc->disk.disk->queue->backing_dev_info->ra_pages,
1432− q->backing_dev_info->ra_pages);
1430+ blk_queue_io_opt(dc->disk.disk->queue,
1431+ max(queue_io_opt(dc->disk.disk->queue), queue_io_opt(q)));
14331432
14341433 atomic_set(&dc->io_errors, 0);
14351434 dc->io_disable = false;
drivers/md/dm-raid.c+1−1
@@ -701,7 +701,7 @@ static void rs_set_capacity(struct raid_set *rs) · 700 unmodified lines
701701 struct gendisk *gendisk = dm_disk(dm_table_get_md(rs->ti->table));
702702
703703 set_capacity(gendisk, rs->md.array_sectors);
704− revalidate_disk(gendisk);
704+ revalidate_disk_size(gendisk, true);
705705 }
706706
707707 /*
drivers/md/dm-table.c+4−5
@@ -1819,7 +1819,7 @@ static int device_requires_stable_pages(struct dm_target *ti, · 1818 unmodified lines
18191819 {
18201820 struct request_queue *q = bdev_get_queue(dev->bdev);
18211821
1822− return q && bdi_cap_stable_pages_required(q->backing_dev_info);
1822+ return q && blk_queue_stable_writes(q);
18231823 }
18241824
18251825 /*
@@ -1904,9 +1904,9 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, · 78 unmodified lines
19041904 * because they do their own checksumming.
19051905 */
19061906 if (dm_table_requires_stable_pages(t))
1907− q->backing_dev_info->capabilities |= BDI_CAP_STABLE_WRITES;
1907+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
19081908 else
1909− q->backing_dev_info->capabilities &= ~BDI_CAP_STABLE_WRITES;
1909+ blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
19101910
19111911 /*
19121912 * Determine whether or not this queue's I/O timings contribute
@@ -1929,8 +1929,7 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, · 16 unmodified lines
19291929 }
19301930 #endif
19311931
1932− /* Allow reads to exceed readahead limits */
1933− q->backing_dev_info->io_pages = limits->max_sectors >> (PAGE_SHIFT - 9);
1932+ blk_queue_update_readahead(q);
19341933 }
19351934
19361935 unsigned int dm_table_get_num_targets(struct dm_table *t)
drivers/md/dm.c+2−13
@@ -2098,18 +2098,6 @@ static void event_callback(void *context) · 2097 unmodified lines
20982098 dm_issue_global_event();
20992099 }
21002100
2101−/*
2102− * Protected by md->suspend_lock obtained by dm_swap_table().
2103− */
2104−static void __set_size(struct mapped_device *md, sector_t size)
2105−{
2106− lockdep_assert_held(&md->suspend_lock);
2107−
2108− set_capacity(md->disk, size);
2109−
2110− i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
2111−}
2112−
21132101 /*
21142102 * Returns old map, which caller must destroy.
21152103 */
@@ -2132,7 +2120,8 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, · 16 unmodified lines
21322120 if (size != dm_get_size(md))
21332121 memset(&md->geometry, 0, sizeof(md->geometry));
21342122
2135− __set_size(md, size);
2123+ set_capacity(md->disk, size);
2124+ bd_set_nr_sectors(md->bdev, size);
21362125
21372126 dm_table_event_callback(t, event_callback, md);
21382127
drivers/md/md-cluster.c+3−3
@@ -582,7 +582,7 @@ static int process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg) · 581 unmodified lines
582582 break;
583583 case CHANGE_CAPACITY:
584584 set_capacity(mddev->gendisk, mddev->array_sectors);
585− revalidate_disk(mddev->gendisk);
585+ revalidate_disk_size(mddev->gendisk, true);
586586 break;
587587 case RESYNCING:
588588 set_bit(MD_RESYNCING_REMOTE, &mddev->recovery);
@@ -1296,12 +1296,12 @@ static void update_size(struct mddev *mddev, sector_t old_dev_sectors) · 707 unmodified lines
12961296 pr_err("%s:%d: failed to send CHANGE_CAPACITY msg\n",
12971297 __func__, __LINE__);
12981298 set_capacity(mddev->gendisk, mddev->array_sectors);
1299− revalidate_disk(mddev->gendisk);
1299+ revalidate_disk_size(mddev->gendisk, true);
13001300 } else {
13011301 /* revert to previous sectors */
13021302 ret = mddev->pers->resize(mddev, old_dev_sectors);
13031303 if (!ret)
1304− revalidate_disk(mddev->gendisk);
1304+ revalidate_disk_size(mddev->gendisk, true);
13051305 ret = __sendmsg(cinfo, &cmsg);
13061306 if (ret)
13071307 pr_err("%s:%d: failed to send METADATA_UPDATED msg\n",
drivers/md/md-linear.c+1−1
@@ -202,7 +202,7 @@ static int linear_add(struct mddev *mddev, struct md_rdev *rdev) · 201 unmodified lines
202202 md_set_array_sectors(mddev, linear_size(mddev, 0, 0));
203203 set_capacity(mddev->gendisk, mddev->array_sectors);
204204 mddev_resume(mddev);
205− revalidate_disk(mddev->gendisk);
205+ revalidate_disk_size(mddev->gendisk, true);
206206 kfree_rcu(oldconf, rcu);
207207 return 0;
208208 }
drivers/md/md.c+10−10
@@ -464,6 +464,7 @@ struct md_io { · 463 unmodified lines
464464 bio_end_io_t *orig_bi_end_io;
465465 void *orig_bi_private;
466466 unsigned long start_time;
467+ struct hd_struct *part;
467468 };
468469
469470 static void md_end_io(struct bio *bio)
@@ -471,7 +472,7 @@ static void md_end_io(struct bio *bio) · 1 unmodified line
471472 struct md_io *md_io = bio->bi_private;
472473 struct mddev *mddev = md_io->mddev;
473474
474− disk_end_io_acct(mddev->gendisk, bio_op(bio), md_io->start_time);
475+ part_end_io_acct(md_io->part, bio, md_io->start_time);
475476
476477 bio->bi_end_io = md_io->orig_bi_end_io;
477478 bio->bi_private = md_io->orig_bi_private;
@@ -517,9 +518,8 @@ static blk_qc_t md_submit_bio(struct bio *bio) · 39 unmodified lines
517518 bio->bi_end_io = md_end_io;
518519 bio->bi_private = md_io;
519520
520− md_io->start_time = disk_start_io_acct(mddev->gendisk,
521− bio_sectors(bio),
522− bio_op(bio));
521+ md_io->start_time = part_start_io_acct(mddev->gendisk,
522+ &md_io->part, bio);
523523 }
524524
525525 /* bio could be mergeable after passing to underlayer */
@@ -5358,7 +5358,7 @@ array_size_store(struct mddev *mddev, const char *buf, size_t len) · 4832 unmodified lines
53585358 mddev->array_sectors = sectors;
53595359 if (mddev->pers) {
53605360 set_capacity(mddev->gendisk, mddev->array_sectors);
5361− revalidate_disk(mddev->gendisk);
5361+ revalidate_disk_size(mddev->gendisk, true);
53625362 }
53635363 }
53645364 mddev_unlock(mddev);
@@ -6109,7 +6109,7 @@ int do_md_run(struct mddev *mddev) · 744 unmodified lines
61096109 md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
61106110
61116111 set_capacity(mddev->gendisk, mddev->array_sectors);
6112− revalidate_disk(mddev->gendisk);
6112+ revalidate_disk_size(mddev->gendisk, true);
61136113 clear_bit(MD_NOT_READY, &mddev->flags);
61146114 mddev->changed = 1;
61156115 kobject_uevent(&disk_to_dev(mddev->gendisk)->kobj, KOBJ_CHANGE);
@@ -6427,7 +6427,7 @@ static int do_md_stop(struct mddev *mddev, int mode, · 311 unmodified lines
64276427 set_capacity(disk, 0);
64286428 mutex_unlock(&mddev->open_mutex);
64296429 mddev->changed = 1;
6430− revalidate_disk(disk);
6430+ revalidate_disk_size(disk, true);
64316431
64326432 if (mddev->ro)
64336433 mddev->ro = 0;
@@ -7259,7 +7259,7 @@ static int update_size(struct mddev *mddev, sector_t num_sectors) · 825 unmodified lines
72597259 md_cluster_ops->update_size(mddev, old_dev_sectors);
72607260 else if (mddev->queue) {
72617261 set_capacity(mddev->gendisk, mddev->array_sectors);
7262− revalidate_disk(mddev->gendisk);
7262+ revalidate_disk_size(mddev->gendisk, true);
72637263 }
72647264 }
72657265 return rv;
@@ -7848,7 +7848,7 @@ static int md_open(struct block_device *bdev, fmode_t mode) · 582 unmodified lines
78487848 atomic_inc(&mddev->openers);
78497849 mutex_unlock(&mddev->open_mutex);
78507850
7851− check_disk_change(bdev);
7851+ bdev_check_media_change(bdev);
78527852 out:
78537853 if (err)
78547854 mddev_put(mddev);
@@ -9018,7 +9018,7 @@ void md_do_sync(struct md_thread *thread) · 1163 unmodified lines
90189018 mddev_unlock(mddev);
90199019 if (!mddev_is_clustered(mddev)) {
90209020 set_capacity(mddev->gendisk, mddev->array_sectors);
9021− revalidate_disk(mddev->gendisk);
9021+ revalidate_disk_size(mddev->gendisk, true);
90229022 }
90239023 }
90249024
drivers/md/md.h+1−1
@@ -397,7 +397,7 @@ struct mddev { · 396 unmodified lines
397397 * These locks are separate due to conflicting interactions
398398 * with bdev->bd_mutex.
399399 * Lock ordering is:
400− * reconfig_mutex -> bd_mutex : e.g. do_md_run -> revalidate_disk
400+ * reconfig_mutex -> bd_mutex
401401 * bd_mutex -> open_mutex: e.g. __blkdev_get -> md_open
402402 */
403403 struct mutex open_mutex;
drivers/md/raid0.c+0−16
@@ -410,22 +410,6 @@ static int raid0_run(struct mddev *mddev) · 409 unmodified lines
410410 mdname(mddev),
411411 (unsigned long long)mddev->array_sectors);
412412
413− if (mddev->queue) {
414− /* calculate the max read-ahead size.
415− * For read-ahead of large files to be effective, we need to
416− * readahead at least twice a whole stripe. i.e. number of devices
417− * multiplied by chunk size times 2.
418− * If an individual device has an ra_pages greater than the
419− * chunk size, then we will not drive that device as hard as it
420− * wants. We consider this a configuration error: a larger
421− * chunksize should be used in that case.
422− */
423− int stripe = mddev->raid_disks *
424− (mddev->chunk_sectors << 9) / PAGE_SIZE;
425− if (mddev->queue->backing_dev_info->ra_pages < 2* stripe)
426− mddev->queue->backing_dev_info->ra_pages = 2* stripe;
427− }
428−
429413 dump_zones(mddev);
430414
431415 ret = md_integrity_register(mddev);
drivers/md/raid10.c+15−31
@@ -3703,10 +3703,20 @@ static struct r10conf *setup_conf(struct mddev *mddev) · 3702 unmodified lines
37033703 return ERR_PTR(err);
37043704 }
37053705
3706+static void raid10_set_io_opt(struct r10conf *conf)
3707+{
3708+ int raid_disks = conf->geo.raid_disks;
3709+
3710+ if (!(conf->geo.raid_disks % conf->geo.near_copies))
3711+ raid_disks /= conf->geo.near_copies;
3712+ blk_queue_io_opt(conf->mddev->queue, (conf->mddev->chunk_sectors << 9) *
3713+ raid_disks);
3714+}
3715+
37063716 static int raid10_run(struct mddev *mddev)
37073717 {
37083718 struct r10conf *conf;
3709− int i, disk_idx, chunk_size;
3719+ int i, disk_idx;
37103720 struct raid10_info *disk;
37113721 struct md_rdev *rdev;
37123722 sector_t size;
@@ -3742,18 +3752,13 @@ static int raid10_run(struct mddev *mddev) · 29 unmodified lines
37423752 mddev->thread = conf->thread;
37433753 conf->thread = NULL;
37443754
3745− chunk_size = mddev->chunk_sectors << 9;
37463755 if (mddev->queue) {
37473756 blk_queue_max_discard_sectors(mddev->queue,
37483757 mddev->chunk_sectors);
37493758 blk_queue_max_write_same_sectors(mddev->queue, 0);
37503759 blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
3751− blk_queue_io_min(mddev->queue, chunk_size);
3752− if (conf->geo.raid_disks % conf->geo.near_copies)
3753− blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
3754− else
3755− blk_queue_io_opt(mddev->queue, chunk_size *
3756− (conf->geo.raid_disks / conf->geo.near_copies));
3760+ blk_queue_io_min(mddev->queue, mddev->chunk_sectors << 9);
3761+ raid10_set_io_opt(conf);
37573762 }
37583763
37593764 rdev_for_each(rdev, mddev) {
@@ -3868,19 +3873,6 @@ static int raid10_run(struct mddev *mddev) · 108 unmodified lines
38683873 mddev->resync_max_sectors = size;
38693874 set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
38703875
3871− if (mddev->queue) {
3872− int stripe = conf->geo.raid_disks *
3873− ((mddev->chunk_sectors << 9) / PAGE_SIZE);
3874−
3875− /* Calculate max read-ahead size.
3876− * We need to readahead at least twice a whole stripe....
3877− * maybe...
3878− */
3879− stripe /= conf->geo.near_copies;
3880− if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
3881− mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
3882− }
3883−
38843876 if (md_integrity_register(mddev))
38853877 goto out_free_conf;
38863878
@@ -4718,16 +4710,8 @@ static void end_reshape(struct r10conf *conf) · 831 unmodified lines
47184710 conf->reshape_safe = MaxSector;
47194711 spin_unlock_irq(&conf->device_lock);
47204712
4721− /* read-ahead size must cover two whole stripes, which is
4722− * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
4723− */
4724− if (conf->mddev->queue) {
4725− int stripe = conf->geo.raid_disks *
4726− ((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
4727− stripe /= conf->geo.near_copies;
4728− if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
4729− conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
4730− }
4713+ if (conf->mddev->queue)
4714+ raid10_set_io_opt(conf);
47314715 conf->fullsync = 0;
47324716 }
47334717
drivers/md/raid5.c+13−18
@@ -6638,14 +6638,14 @@ raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len) · 6637 unmodified lines
66386638 if (!conf)
66396639 err = -ENODEV;
66406640 else if (new != conf->skip_copy) {
6641+ struct request_queue *q = mddev->queue;
6642+
66416643 mddev_suspend(mddev);
66426644 conf->skip_copy = new;
66436645 if (new)
6644− mddev->queue->backing_dev_info->capabilities |=
6645− BDI_CAP_STABLE_WRITES;
6646+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
66466647 else
6647− mddev->queue->backing_dev_info->capabilities &=
6648− ~BDI_CAP_STABLE_WRITES;
6648+ blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
66496649 mddev_resume(mddev);
66506650 }
66516651 mddev_unlock(mddev);
@@ -7232,6 +7232,12 @@ static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded · 580 unmodified lines
72327232 return 0;
72337233 }
72347234
7235+static void raid5_set_io_opt(struct r5conf *conf)
7236+{
7237+ blk_queue_io_opt(conf->mddev->queue, (conf->chunk_sectors << 9) *
7238+ (conf->raid_disks - conf->max_degraded));
7239+}
7240+
72357241 static int raid5_run(struct mddev *mddev)
72367242 {
72377243 struct r5conf *conf;
@@ -7516,13 +7522,10 @@ static int raid5_run(struct mddev *mddev) · 278 unmodified lines
75167522 int data_disks = conf->previous_raid_disks - conf->max_degraded;
75177523 int stripe = data_disks *
75187524 ((mddev->chunk_sectors << 9) / PAGE_SIZE);
7519− if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
7520− mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
75217525
75227526 chunk_size = mddev->chunk_sectors << 9;
75237527 blk_queue_io_min(mddev->queue, chunk_size);
7524− blk_queue_io_opt(mddev->queue, chunk_size *
7525− (conf->raid_disks - conf->max_degraded));
7528+ raid5_set_io_opt(conf);
75267529 mddev->queue->limits.raid_partial_stripes_expensive = 1;
75277530 /*
75287531 * We can only discard a whole stripe. It doesn't make sense to
@@ -8106,16 +8109,8 @@ static void end_reshape(struct r5conf *conf) · 577 unmodified lines
81068109 spin_unlock_irq(&conf->device_lock);
81078110 wake_up(&conf->wait_for_overlap);
81088111
8109− /* read-ahead size must cover two whole stripes, which is
8110− * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
8111− */
8112− if (conf->mddev->queue) {
8113− int data_disks = conf->raid_disks - conf->max_degraded;
8114− int stripe = data_disks * ((conf->chunk_sectors << 9)
8115− / PAGE_SIZE);
8116− if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
8117− conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
8118− }
8112+ if (conf->mddev->queue)
8113+ raid5_set_io_opt(conf);
81198114 }
81208115 }
81218116
drivers/mmc/core/queue.c+1−2
@@ -472,8 +472,7 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card) · 471 unmodified lines
472472 }
473473
474474 if (mmc_host_is_spi(host) && host->use_spi_crc)
475− mq->queue->backing_dev_info->capabilities |=
476− BDI_CAP_STABLE_WRITES;
475+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, mq->queue);
477476
478477 mq->queue->queuedata = mq;
479478 blk_queue_rq_timeout(mq->queue, 60 * HZ);
drivers/mtd/mtdcore.c+2−0
@@ -2196,6 +2196,8 @@ static struct backing_dev_info * __init mtd_bdi_init(char *name) · 2195 unmodified lines
21962196 bdi = bdi_alloc(NUMA_NO_NODE);
21972197 if (!bdi)
21982198 return ERR_PTR(-ENOMEM);
2199+ bdi->ra_pages = 0;
2200+ bdi->io_pages = 0;
21992201
22002202 /*
22012203 * We put '-0' suffix to the name to get the same name format as we
drivers/nvdimm/blk.c+1−2
@@ -226,7 +226,6 @@ static int nsblk_rw_bytes(struct nd_namespace_common *ndns, · 225 unmodified lines
226226 static const struct block_device_operations nd_blk_fops = {
227227 .owner = THIS_MODULE,
228228 .submit_bio = nd_blk_submit_bio,
229− .revalidate_disk = nvdimm_revalidate_disk,
230229 };
231230
232231 static void nd_blk_release_queue(void *q)
@@ -284,7 +283,7 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) · 51 unmodified lines
284283
285284 set_capacity(disk, available_disk_size >> SECTOR_SHIFT);
286285 device_add_disk(dev, disk, NULL);
287− revalidate_disk(disk);
286+ nvdimm_check_and_set_ro(disk);
288287 return 0;
289288 }
290289
drivers/nvdimm/btt.c+1−4
@@ -1513,7 +1513,6 @@ static const struct block_device_operations btt_fops = { · 1512 unmodified lines
15131513 .submit_bio = btt_submit_bio,
15141514 .rw_page = btt_rw_page,
15151515 .getgeo = btt_getgeo,
1516− .revalidate_disk = nvdimm_revalidate_disk,
15171516 };
15181517
15191518 static int btt_blk_init(struct btt *btt)
@@ -1538,8 +1537,6 @@ static int btt_blk_init(struct btt *btt) · 18 unmodified lines
15381537 btt->btt_disk->private_data = btt;
15391538 btt->btt_disk->queue = btt->btt_queue;
15401539 btt->btt_disk->flags = GENHD_FL_EXT_DEVT;
1541− btt->btt_disk->queue->backing_dev_info->capabilities |=
1542− BDI_CAP_SYNCHRONOUS_IO;
15431540
15441541 blk_queue_logical_block_size(btt->btt_queue, btt->sector_size);
15451542 blk_queue_max_hw_sectors(btt->btt_queue, UINT_MAX);
@@ -1558,7 +1555,7 @@ static int btt_blk_init(struct btt *btt) · 12 unmodified lines
15581555 set_capacity(btt->btt_disk, btt->nlba * btt->sector_size >> 9);
15591556 device_add_disk(&btt->nd_btt->dev, btt->btt_disk, NULL);
15601557 btt->nd_btt->size = btt->nlba * (u64)btt->sector_size;
1561− revalidate_disk(btt->btt_disk);
1558+ nvdimm_check_and_set_ro(btt->btt_disk);
15621559
15631560 return 0;
15641561 }
drivers/nvdimm/bus.c+3−6
@@ -628,7 +628,7 @@ int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner, · 627 unmodified lines
628628 }
629629 EXPORT_SYMBOL(__nd_driver_register);
630630
631−int nvdimm_revalidate_disk(struct gendisk *disk)
631+void nvdimm_check_and_set_ro(struct gendisk *disk)
632632 {
633633 struct device *dev = disk_to_dev(disk)->parent;
634634 struct nd_region *nd_region = to_nd_region(dev->parent);
@@ -639,16 +639,13 @@ int nvdimm_revalidate_disk(struct gendisk *disk) · 4 unmodified lines
639639 * read-only if the disk is already read-only.
640640 */
641641 if (disk_ro || nd_region->ro == disk_ro)
642− return 0;
642+ return;
643643
644644 dev_info(dev, "%s read-only, marking %s read-only\n",
645645 dev_name(&nd_region->dev), disk->disk_name);
646646 set_disk_ro(disk, 1);
647−
648− return 0;
649−
650647 }
651−EXPORT_SYMBOL(nvdimm_revalidate_disk);
648+EXPORT_SYMBOL(nvdimm_check_and_set_ro);
652649
653650 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
654651 char *buf)
drivers/nvdimm/nd.h+1−1
@@ -361,7 +361,7 @@ u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region); · 360 unmodified lines
361361 void nvdimm_bus_lock(struct device *dev);
362362 void nvdimm_bus_unlock(struct device *dev);
363363 bool is_nvdimm_bus_locked(struct device *dev);
364−int nvdimm_revalidate_disk(struct gendisk *disk);
364+void nvdimm_check_and_set_ro(struct gendisk *disk);
365365 void nvdimm_drvdata_release(struct kref *kref);
366366 void put_ndd(struct nvdimm_drvdata *ndd);
367367 int nd_label_reserve_dpa(struct nvdimm_drvdata *ndd);
drivers/nvdimm/pmem.c+1−3
@@ -281,7 +281,6 @@ static const struct block_device_operations pmem_fops = { · 280 unmodified lines
281281 .owner = THIS_MODULE,
282282 .submit_bio = pmem_submit_bio,
283283 .rw_page = pmem_rw_page,
284− .revalidate_disk = nvdimm_revalidate_disk,
285284 };
286285
287286 static int pmem_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
@@ -476,7 +475,6 @@ static int pmem_attach_disk(struct device *dev, · 188 unmodified lines
476475 disk->queue = q;
477476 disk->flags = GENHD_FL_EXT_DEVT;
478477 disk->private_data = pmem;
479− disk->queue->backing_dev_info->capabilities |= BDI_CAP_SYNCHRONOUS_IO;
480478 nvdimm_namespace_disk_name(ndns, disk->disk_name);
481479 set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)
482480 / 512);
@@ -501,7 +499,7 @@ static int pmem_attach_disk(struct device *dev, · 18 unmodified lines
501499 if (devm_add_action_or_reset(dev, pmem_release_disk, pmem))
502500 return -ENOMEM;
503501
504− revalidate_disk(disk);
502+ nvdimm_check_and_set_ro(disk);
505503
506504 pmem->bb_state = sysfs_get_dirent(disk_to_dev(disk)->kobj.sd,
507505 "badblocks");
drivers/nvme/host/core.c+35−18
@@ -94,21 +94,34 @@ static void nvme_put_subsystem(struct nvme_subsystem *subsys); · 93 unmodified lines
9494 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
9595 unsigned nsid);
9696
97+static void nvme_update_bdev_size(struct gendisk *disk)
98+{
99+ struct block_device *bdev = bdget_disk(disk, 0);
100+
101+ if (bdev) {
102+ bd_set_nr_sectors(bdev, get_capacity(disk));
103+ bdput(bdev);
104+ }
105+}
106+
107+/*
108+ * Prepare a queue for teardown.
109+ *
110+ * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
111+ * the capacity to 0 after that to avoid blocking dispatchers that may be
112+ * holding bd_butex. This will end buffered writers dirtying pages that can't
113+ * be synced.
114+ */
97115 static void nvme_set_queue_dying(struct nvme_ns *ns)
98116 {
99− /*
100− * Revalidating a dead namespace sets capacity to 0. This will end
101− * buffered writers dirtying pages that can't be synced.
102− */
103117 if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
104118 return;
119+
105120 blk_set_queue_dying(ns->queue);
106− /* Forcibly unquiesce queues to avoid blocking dispatch */
107121 blk_mq_unquiesce_queue(ns->queue);
108− /*
109− * Revalidate after unblocking dispatchers that may be holding bd_butex
110− */
111− revalidate_disk(ns->disk);
122+
123+ set_capacity(ns->disk, 0);
124+ nvme_update_bdev_size(ns->disk);
112125 }
113126
114127 static void nvme_queue_scan(struct nvme_ctrl *ctrl)
@@ -2134,7 +2147,8 @@ static int __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) · 2019 unmodified lines
21342147 nvme_update_disk_info(ns->head->disk, ns, id);
21352148 blk_stack_limits(&ns->head->disk->queue->limits,
21362149 &ns->queue->limits, 0);
2137− nvme_mpath_update_disk_size(ns->head->disk);
2150+ blk_queue_update_readahead(ns->head->disk->queue);
2151+ nvme_update_bdev_size(ns->head->disk);
21382152 }
21392153 #endif
21402154 return 0;
@@ -2339,7 +2353,6 @@ static const struct block_device_operations nvme_fops = { · 198 unmodified lines
23392353 .open = nvme_open,
23402354 .release = nvme_release,
23412355 .getgeo = nvme_getgeo,
2342− .revalidate_disk= nvme_revalidate_disk,
23432356 .report_zones = nvme_report_zones,
23442357 .pr_ops = &nvme_pr_ops,
23452358 };
@@ -3924,8 +3937,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) · 1578 unmodified lines
39243937 goto out_free_ns;
39253938
39263939 if (ctrl->opts && ctrl->opts->data_digest)
3927− ns->queue->backing_dev_info->capabilities
3928− |= BDI_CAP_STABLE_WRITES;
3940+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
39293941
39303942 blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
39313943 if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
@@ -4051,14 +4063,19 @@ static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid) · 119 unmodified lines
40514063 static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
40524064 {
40534065 struct nvme_ns *ns;
4066+ int ret;
40544067
40554068 ns = nvme_find_get_ns(ctrl, nsid);
4056− if (ns) {
4057− if (revalidate_disk(ns->disk))
4058− nvme_ns_remove(ns);
4059− nvme_put_ns(ns);
4060− } else
4069+ if (!ns) {
40614070 nvme_alloc_ns(ctrl, nsid);
4071+ return;
4072+ }
4073+
4074+ ret = nvme_revalidate_disk(ns->disk);
4075+ revalidate_disk_size(ns->disk, ret == 0);
4076+ if (ret)
4077+ nvme_ns_remove(ns);
4078+ nvme_put_ns(ns);
40624079 }
40634080
40644081 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
drivers/nvme/host/multipath.c+3−7
@@ -673,13 +673,9 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) · 672 unmodified lines
673673 nvme_mpath_set_live(ns);
674674 }
675675
676− if (bdi_cap_stable_pages_required(ns->queue->backing_dev_info)) {
677− struct gendisk *disk = ns->head->disk;
678−
679− if (disk)
680− disk->queue->backing_dev_info->capabilities |=
681− BDI_CAP_STABLE_WRITES;
682− }
676+ if (blk_queue_stable_writes(ns->queue) && ns->head->disk)
677+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES,
678+ ns->head->disk->queue);
683679 }
684680
685681 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
drivers/nvme/host/nvme.h+0−13
@@ -682,16 +682,6 @@ static inline void nvme_trace_bio_complete(struct request *req, · 681 unmodified lines
682682 trace_block_bio_complete(ns->head->disk->queue, req->bio);
683683 }
684684
685−static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
686−{
687− struct block_device *bdev = bdget_disk(disk, 0);
688−
689− if (bdev) {
690− bd_set_size(bdev, get_capacity(disk) << SECTOR_SHIFT);
691− bdput(bdev);
692− }
693−}
694−
695685 extern struct device_attribute dev_attr_ana_grpid;
696686 extern struct device_attribute dev_attr_ana_state;
697687 extern struct device_attribute subsys_attr_iopolicy;
@@ -766,9 +756,6 @@ static inline void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) · 68 unmodified lines
766756 static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
767757 {
768758 }
769−static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
770−{
771−}
772759 #endif /* CONFIG_NVME_MULTIPATH */
773760
774761 #ifdef CONFIG_BLK_DEV_ZONED
drivers/s390/block/dasd_genhd.c+4−11
@@ -101,18 +101,11 @@ int dasd_scan_partitions(struct dasd_block *block) · 100 unmodified lines
101101 struct block_device *bdev;
102102 int rc;
103103
104− bdev = bdget_disk(block->gdp, 0);
105− if (!bdev) {
106− DBF_DEV_EVENT(DBF_ERR, block->base, "%s",
107− "scan partitions error, bdget returned NULL");
108− return -ENODEV;
109− }
110−
111− rc = blkdev_get(bdev, FMODE_READ, NULL);
112− if (rc < 0) {
104+ bdev = blkdev_get_by_dev(disk_devt(block->gdp), FMODE_READ, NULL);
105+ if (IS_ERR(bdev)) {
113106 DBF_DEV_EVENT(DBF_ERR, block->base,
114− "scan partitions error, blkdev_get returned %d",
115− rc);
107+ "scan partitions error, blkdev_get returned %ld",
108+ PTR_ERR(bdev));
116109 return -ENODEV;
117110 }
118111
drivers/s390/block/dasd_ioctl.c+2−7
@@ -55,10 +55,7 @@ dasd_ioctl_enable(struct block_device *bdev) · 54 unmodified lines
5555
5656 dasd_enable_device(base);
5757 /* Formatting the dasd device can change the capacity. */
58− mutex_lock(&bdev->bd_mutex);
59− i_size_write(bdev->bd_inode,
60− (loff_t)get_capacity(base->block->gdp) << 9);
61− mutex_unlock(&bdev->bd_mutex);
58+ bd_set_nr_sectors(bdev, get_capacity(base->block->gdp));
6259 dasd_put_device(base);
6360 return 0;
6461 }
@@ -91,9 +88,7 @@ dasd_ioctl_disable(struct block_device *bdev) · 26 unmodified lines
9188 * Set i_size to zero, since read, write, etc. check against this
9289 * value.
9390 */
94− mutex_lock(&bdev->bd_mutex);
95− i_size_write(bdev->bd_inode, 0);
96− mutex_unlock(&bdev->bd_mutex);
91+ bd_set_nr_sectors(bdev, 0);
9792 dasd_put_device(base);
9893 return 0;
9994 }
drivers/scsi/iscsi_tcp.c+2−2
@@ -962,8 +962,8 @@ static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev) · 961 unmodified lines
962962 struct iscsi_conn *conn = session->leadconn;
963963
964964 if (conn->datadgst_en)
965− sdev->request_queue->backing_dev_info->capabilities
966− |= BDI_CAP_STABLE_WRITES;
965+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES,
966+ sdev->request_queue);
967967 blk_queue_dma_alignment(sdev->request_queue, 0);
968968 return 0;
969969 }
drivers/scsi/sd.c+8−5
@@ -217,7 +217,7 @@ cache_type_store(struct device *dev, struct device_attribute *attr, · 216 unmodified lines
217217 sd_print_sense_hdr(sdkp, &sshdr);
218218 return -EINVAL;
219219 }
220− revalidate_disk(sdkp->disk);
220+ sd_revalidate_disk(sdkp->disk);
221221 return count;
222222 }
223223
@@ -1381,8 +1381,10 @@ static int sd_open(struct block_device *bdev, fmode_t mode) · 1157 unmodified lines
13811381 if (!scsi_block_when_processing_errors(sdev))
13821382 goto error_out;
13831383
1384− if (sdev->removable || sdkp->write_prot)
1385− check_disk_change(bdev);
1384+ if (sdev->removable || sdkp->write_prot) {
1385+ if (bdev_check_media_change(bdev))
1386+ sd_revalidate_disk(bdev->bd_disk);
1387+ }
13861388
13871389 /*
13881390 * If the drive is empty, just let the open fail.
@@ -1706,8 +1708,10 @@ static int sd_sync_cache(struct scsi_disk *sdkp, struct scsi_sense_hdr *sshdr) · 317 unmodified lines
17061708 static void sd_rescan(struct device *dev)
17071709 {
17081710 struct scsi_disk *sdkp = dev_get_drvdata(dev);
1711+ int ret;
17091712
1710− revalidate_disk(sdkp->disk);
1713+ ret = sd_revalidate_disk(sdkp->disk);
1714+ revalidate_disk_size(sdkp->disk, ret == 0);
17111715 }
17121716
17131717 static int sd_ioctl(struct block_device *bdev, fmode_t mode,
@@ -1841,7 +1845,6 @@ static const struct block_device_operations sd_fops = { · 127 unmodified lines
18411845 .compat_ioctl = sd_compat_ioctl,
18421846 #endif
18431847 .check_events = sd_check_events,
1844− .revalidate_disk = sd_revalidate_disk,
18451848 .unlock_native_capacity = sd_unlock_native_capacity,
18461849 .report_zones = sd_zbc_report_zones,
18471850 .pr_ops = &sd_pr_ops,
drivers/scsi/sr.c+14−22
@@ -517,6 +517,17 @@ static blk_status_t sr_init_command(struct scsi_cmnd *SCpnt) · 516 unmodified lines
517517 return ret;
518518 }
519519
520+static void sr_revalidate_disk(struct scsi_cd *cd)
521+{
522+ struct scsi_sense_hdr sshdr;
523+
524+ /* if the unit is not ready, nothing more to do */
525+ if (scsi_test_unit_ready(cd->device, SR_TIMEOUT, MAX_RETRIES, &sshdr))
526+ return;
527+ sr_cd_check(&cd->cdi);
528+ get_sectorsize(cd);
529+}
530+
520531 static int sr_block_open(struct block_device *bdev, fmode_t mode)
521532 {
522533 struct scsi_cd *cd;
@@ -529,7 +540,8 @@ static int sr_block_open(struct block_device *bdev, fmode_t mode) · 6 unmodified lines
529540
530541 sdev = cd->device;
531542 scsi_autopm_get_device(sdev);
532− check_disk_change(bdev);
543+ if (bdev_check_media_change(bdev))
544+ sr_revalidate_disk(cd);
533545
534546 mutex_lock(&cd->lock);
535547 ret = cdrom_open(&cd->cdi, bdev, mode);
@@ -658,26 +670,6 @@ static unsigned int sr_block_check_events(struct gendisk *disk, · 122 unmodified lines
658670 return ret;
659671 }
660672
661−static int sr_block_revalidate_disk(struct gendisk *disk)
662−{
663− struct scsi_sense_hdr sshdr;
664− struct scsi_cd *cd;
665−
666− cd = scsi_cd_get(disk);
667− if (!cd)
668− return -ENXIO;
669−
670− /* if the unit is not ready, nothing more to do */
671− if (scsi_test_unit_ready(cd->device, SR_TIMEOUT, MAX_RETRIES, &sshdr))
672− goto out;
673−
674− sr_cd_check(&cd->cdi);
675− get_sectorsize(cd);
676−out:
677− scsi_cd_put(cd);
678− return 0;
679−}
680−
681673 static const struct block_device_operations sr_bdops =
682674 {
683675 .owner = THIS_MODULE,
@@ -688,7 +680,6 @@ static const struct block_device_operations sr_bdops = · 4 unmodified lines
688680 .compat_ioctl = sr_block_compat_ioctl,
689681 #endif
690682 .check_events = sr_block_check_events,
691− .revalidate_disk = sr_block_revalidate_disk,
692683 };
693684
694685 static int sr_open(struct cdrom_device_info *cdi, int purpose)
@@ -802,6 +793,7 @@ static int sr_probe(struct device *dev) · 107 unmodified lines
802793
803794 dev_set_drvdata(dev, cd);
804795 disk->flags |= GENHD_FL_REMOVABLE;
796+ sr_revalidate_disk(cd);
805797 device_add_disk(&sdev->sdev_gendev, disk, NULL);
806798
807799 sdev_printk(KERN_DEBUG, sdev,
fs/9p/vfs_file.c+1−1
@@ -625,7 +625,7 @@ static void v9fs_mmap_vm_close(struct vm_area_struct *vma) · 624 unmodified lines
625625
626626 inode = file_inode(vma->vm_file);
627627
628− if (!mapping_cap_writeback_dirty(inode->i_mapping))
628+ if (!mapping_can_writeback(inode->i_mapping))
629629 wbc.nr_to_write = 0;
630630
631631 might_sleep();
fs/9p/vfs_super.c+4−2
@@ -80,8 +80,10 @@ v9fs_fill_super(struct super_block *sb, struct v9fs_session_info *v9ses, · 79 unmodified lines
8080 if (ret)
8181 return ret;
8282
83− if (v9ses->cache)
84− sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
83+ if (!v9ses->cache) {
84+ sb->s_bdi->ra_pages = 0;
85+ sb->s_bdi->io_pages = 0;
86+ }
8587
8688 sb->s_flags |= SB_ACTIVE | SB_DIRSYNC;
8789 if (!v9ses->cache)
fs/afs/super.c+0−1
@@ -456,7 +456,6 @@ static int afs_fill_super(struct super_block *sb, struct afs_fs_context *ctx) · 455 unmodified lines
456456 ret = super_setup_bdi(sb);
457457 if (ret)
458458 return ret;
459− sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
460459
461460 /* allocate the root inode and dentry */
462461 if (as->dyn_root) {
fs/block_dev.c+85−90
@@ -103,6 +103,35 @@ void invalidate_bdev(struct block_device *bdev) · 102 unmodified lines
103103 }
104104 EXPORT_SYMBOL(invalidate_bdev);
105105
106+/*
107+ * Drop all buffers & page cache for given bdev range. This function bails
108+ * with error if bdev has other exclusive owner (such as filesystem).
109+ */
110+int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
111+ loff_t lstart, loff_t lend)
112+{
113+ struct block_device *claimed_bdev = NULL;
114+ int err;
115+
116+ /*
117+ * If we don't hold exclusive handle for the device, upgrade to it
118+ * while we discard the buffer cache to avoid discarding buffers
119+ * under live filesystem.
120+ */
121+ if (!(mode & FMODE_EXCL)) {
122+ claimed_bdev = bdev->bd_contains;
123+ err = bd_prepare_to_claim(bdev, claimed_bdev,
124+ truncate_bdev_range);
125+ if (err)
126+ return err;
127+ }
128+ truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend);
129+ if (claimed_bdev)
130+ bd_abort_claiming(bdev, claimed_bdev, truncate_bdev_range);
131+ return 0;
132+}
133+EXPORT_SYMBOL(truncate_bdev_range);
134+
106135 static void set_init_blocksize(struct block_device *bdev)
107136 {
108137 bdev->bd_inode->i_blkbits = blksize_bits(bdev_logical_block_size(bdev));
@@ -876,11 +905,11 @@ struct block_device *bdget(dev_t dev) · 767 unmodified lines
876905 bdev = &BDEV_I(inode)->bdev;
877906
878907 if (inode->i_state & I_NEW) {
908+ spin_lock_init(&bdev->bd_size_lock);
879909 bdev->bd_contains = NULL;
880910 bdev->bd_super = NULL;
881911 bdev->bd_inode = inode;
882912 bdev->bd_part_count = 0;
883− bdev->bd_invalidated = 0;
884913 inode->i_mode = S_IFBLK;
885914 inode->i_rdev = dev;
886915 inode->i_bdev = bdev;
@@ -1290,6 +1319,7 @@ static void check_disk_size_change(struct gendisk *disk, · 403 unmodified lines
12901319 {
12911320 loff_t disk_size, bdev_size;
12921321
1322+ spin_lock(&bdev->bd_size_lock);
12931323 disk_size = (loff_t)get_capacity(disk) << 9;
12941324 bdev_size = i_size_read(bdev->bd_inode);
12951325 if (disk_size != bdev_size) {
@@ -1299,85 +1329,51 @@ static void check_disk_size_change(struct gendisk *disk, · 3 unmodified lines
12991329 disk->disk_name, bdev_size, disk_size);
13001330 }
13011331 i_size_write(bdev->bd_inode, disk_size);
1302− if (bdev_size > disk_size && __invalidate_device(bdev, false))
1332+ }
1333+ spin_unlock(&bdev->bd_size_lock);
1334+
1335+ if (bdev_size > disk_size) {
1336+ if (__invalidate_device(bdev, false))
13031337 pr_warn("VFS: busy inodes on resized disk %s\n",
13041338 disk->disk_name);
13051339 }
1306− bdev->bd_invalidated = 0;
13071340 }
13081341
13091342 /**
1310− * revalidate_disk - wrapper for lower-level driver's revalidate_disk call-back
1311− * @disk: struct gendisk to be revalidated
1343+ * revalidate_disk_size - checks for disk size change and adjusts bdev size.
1344+ * @disk: struct gendisk to check
1345+ * @verbose: if %true log a message about a size change if there is any
13121346 *
1313− * This routine is a wrapper for lower-level driver's revalidate_disk
1314− * call-backs. It is used to do common pre and post operations needed
1315− * for all revalidate_disk operations.
1347+ * This routine checks to see if the bdev size does not match the disk size
1348+ * and adjusts it if it differs. When shrinking the bdev size, its all caches
1349+ * are freed.
13161350 */
1317−int revalidate_disk(struct gendisk *disk)
1351+void revalidate_disk_size(struct gendisk *disk, bool verbose)
13181352 {
1319− int ret = 0;
1320−
1321− if (disk->fops->revalidate_disk)
1322− ret = disk->fops->revalidate_disk(disk);
1353+ struct block_device *bdev;
13231354
13241355 /*
13251356 * Hidden disks don't have associated bdev so there's no point in
1326− * revalidating it.
1357+ * revalidating them.
13271358 */
1328− if (!(disk->flags & GENHD_FL_HIDDEN)) {
1329− struct block_device *bdev = bdget_disk(disk, 0);
1330−
1331− if (!bdev)
1332− return ret;
1359+ if (disk->flags & GENHD_FL_HIDDEN)
1360+ return;
13331361
1334− mutex_lock(&bdev->bd_mutex);
1335− check_disk_size_change(disk, bdev, ret == 0);
1336− mutex_unlock(&bdev->bd_mutex);
1362+ bdev = bdget_disk(disk, 0);
1363+ if (bdev) {
1364+ check_disk_size_change(disk, bdev, verbose);
13371365 bdput(bdev);
13381366 }
1339− return ret;
13401367 }
1341−EXPORT_SYMBOL(revalidate_disk);
1368+EXPORT_SYMBOL(revalidate_disk_size);
13421369
1343−/*
1344− * This routine checks whether a removable media has been changed,
1345− * and invalidates all buffer-cache-entries in that case. This
1346− * is a relatively slow routine, so we have to try to minimize using
1347− * it. Thus it is called only upon a 'mount' or 'open'. This
1348− * is the best way of combining speed and utility, I think.
1349− * People changing diskettes in the middle of an operation deserve
1350− * to lose :-)
1351− */
1352−int check_disk_change(struct block_device *bdev)
1370+void bd_set_nr_sectors(struct block_device *bdev, sector_t sectors)
13531371 {
1354− struct gendisk *disk = bdev->bd_disk;
1355− const struct block_device_operations *bdops = disk->fops;
1356− unsigned int events;
1357−
1358− events = disk_clear_events(disk, DISK_EVENT_MEDIA_CHANGE |
1359− DISK_EVENT_EJECT_REQUEST);
1360− if (!(events & DISK_EVENT_MEDIA_CHANGE))
1361− return 0;
1362−
1363− if (__invalidate_device(bdev, true))
1364− pr_warn("VFS: busy inodes on changed media %s\n",
1365− disk->disk_name);
1366− bdev->bd_invalidated = 1;
1367− if (bdops->revalidate_disk)
1368− bdops->revalidate_disk(bdev->bd_disk);
1369− return 1;
1370−}
1371−
1372−EXPORT_SYMBOL(check_disk_change);
1373−
1374−void bd_set_size(struct block_device *bdev, loff_t size)
1375−{
1376− inode_lock(bdev->bd_inode);
1377− i_size_write(bdev->bd_inode, size);
1378− inode_unlock(bdev->bd_inode);
1372+ spin_lock(&bdev->bd_size_lock);
1373+ i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT);
1374+ spin_unlock(&bdev->bd_size_lock);
13791375 }
1380−EXPORT_SYMBOL(bd_set_size);
1376+EXPORT_SYMBOL(bd_set_nr_sectors);
13811377
13821378 static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
13831379
@@ -1388,6 +1384,8 @@ int bdev_disk_changed(struct block_device *bdev, bool invalidate) · 4 unmodified lines
13881384
13891385 lockdep_assert_held(&bdev->bd_mutex);
13901386
1387+ clear_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state);
1388+
13911389 rescan:
13921390 ret = blk_drop_partitions(bdev);
13931391 if (ret)
@@ -1446,22 +1444,8 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, void *holder, · 52 unmodified lines
14461444 struct gendisk *disk;
14471445 int ret;
14481446 int partno;
1449− int perm = 0;
14501447 bool first_open = false, unblock_events = true, need_restart;
14511448
1452− if (mode & FMODE_READ)
1453− perm |= MAY_READ;
1454− if (mode & FMODE_WRITE)
1455− perm |= MAY_WRITE;
1456− /*
1457− * hooks: /n/, see "layering violations".
1458− */
1459− if (!for_part) {
1460− ret = devcgroup_inode_permission(bdev->bd_inode, perm);
1461− if (ret != 0)
1462− return ret;
1463− }
1464−
14651449 restart:
14661450 need_restart = false;
14671451 ret = -ENXIO;
@@ -1514,7 +1498,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, void *holder, · 46 unmodified lines
15141498 }
15151499
15161500 if (!ret) {
1517− bd_set_size(bdev,(loff_t)get_capacity(disk)<<9);
1501+ bd_set_nr_sectors(bdev, get_capacity(disk));
15181502 set_init_blocksize(bdev);
15191503 }
15201504
@@ -1524,7 +1508,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, void *holder, · 3 unmodified lines
15241508 * The latter is necessary to prevent ghost
15251509 * partitions on a removed medium.
15261510 */
1527− if (bdev->bd_invalidated &&
1511+ if (test_bit(GD_NEED_PART_SCAN, &disk->state) &&
15281512 (!ret || ret == -ENOMEDIUM))
15291513 bdev_disk_changed(bdev, ret == -ENOMEDIUM);
15301514
@@ -1542,7 +1526,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, void *holder, · 11 unmodified lines
15421526 ret = -ENXIO;
15431527 goto out_clear;
15441528 }
1545− bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9);
1529+ bd_set_nr_sectors(bdev, bdev->bd_part->nr_sects);
15461530 set_init_blocksize(bdev);
15471531 }
15481532
@@ -1554,7 +1538,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, void *holder, · 5 unmodified lines
15541538 if (bdev->bd_disk->fops->open)
15551539 ret = bdev->bd_disk->fops->open(bdev, mode);
15561540 /* the same as first opener case, read comment there */
1557− if (bdev->bd_invalidated &&
1541+ if (test_bit(GD_NEED_PART_SCAN, &disk->state) &&
15581542 (!ret || ret == -ENOMEDIUM))
15591543 bdev_disk_changed(bdev, ret == -ENOMEDIUM);
15601544 if (ret)
@@ -1632,16 +1616,27 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, void *holder, · 71 unmodified lines
16321616 * RETURNS:
16331617 * 0 on success, -errno on failure.
16341618 */
1635−int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
1619+static int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
16361620 {
1637− int res;
1621+ int ret, perm = 0;
16381622
1639− res =__blkdev_get(bdev, mode, holder, 0);
1640− if (res)
1641− bdput(bdev);
1642− return res;
1623+ if (mode & FMODE_READ)
1624+ perm |= MAY_READ;
1625+ if (mode & FMODE_WRITE)
1626+ perm |= MAY_WRITE;
1627+ ret = devcgroup_inode_permission(bdev->bd_inode, perm);
1628+ if (ret)
1629+ goto bdput;
1630+
1631+ ret =__blkdev_get(bdev, mode, holder, 0);
1632+ if (ret)
1633+ goto bdput;
1634+ return 0;
1635+
1636+bdput:
1637+ bdput(bdev);
1638+ return ret;
16431639 }
1644−EXPORT_SYMBOL(blkdev_get);
16451640
16461641 /**
16471642 * blkdev_get_by_path - open a block device by name
@@ -1889,7 +1884,7 @@ ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) · 241 unmodified lines
18891884 if (bdev_read_only(I_BDEV(bd_inode)))
18901885 return -EPERM;
18911886
1892− if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode))
1887+ if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode->i_rdev))
18931888 return -ETXTBSY;
18941889
18951890 if (!iov_iter_count(from))
@@ -1969,7 +1964,6 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, · 73 unmodified lines
19691964 loff_t len)
19701965 {
19711966 struct block_device *bdev = I_BDEV(bdev_file_inode(file));
1972− struct address_space *mapping;
19731967 loff_t end = start + len - 1;
19741968 loff_t isize;
19751969 int error;
@@ -1997,8 +1991,9 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, · 21 unmodified lines
19971991 return -EINVAL;
19981992
19991993 /* Invalidate the page cache, including dirty pages. */
2000− mapping = bdev->bd_inode->i_mapping;
2001− truncate_inode_pages_range(mapping, start, end);
1994+ error = truncate_bdev_range(bdev, file->f_mode, start, end);
1995+ if (error)
1996+ return error;
20021997
20031998 switch (mode) {
20041999 case FALLOC_FL_ZERO_RANGE:
@@ -2025,7 +2020,7 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, · 20 unmodified lines
20252020 * the caller will be given -EBUSY. The third argument is
20262021 * inclusive, so the rounding here is safe.
20272022 */
2028− return invalidate_inode_pages2_range(mapping,
2023+ return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
20292024 start >> PAGE_SHIFT,
20302025 end >> PAGE_SHIFT);
20312026 }
fs/btrfs/disk-io.c+0−2
@@ -3091,8 +3091,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device · 3090 unmodified lines
30913091 goto fail_sb_buffer;
30923092 }
30933093
3094− sb->s_bdi->capabilities |= BDI_CAP_CGROUP_WRITEBACK;
3095− sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
30963094 sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
30973095 sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
30983096
fs/buffer.c+0−16
@@ -2771,16 +2771,6 @@ int nobh_writepage(struct page *page, get_block_t *get_block, · 2770 unmodified lines
27712771 /* Is the page fully outside i_size? (truncate in progress) */
27722772 offset = i_size & (PAGE_SIZE-1);
27732773 if (page->index >= end_index+1 || !offset) {
2774− /*
2775− * The page may have dirty, unmapped buffers. For example,
2776− * they may have been added in ext3_writepage(). Make them
2777− * freeable here, so the page does not leak.
2778− */
2779−#if 0
2780− /* Not really sure about this - do we need this ? */
2781− if (page->mapping->a_ops->invalidatepage)
2782− page->mapping->a_ops->invalidatepage(page, offset);
2783−#endif
27842774 unlock_page(page);
27852775 return 0; /* don't care */
27862776 }
@@ -2975,12 +2965,6 @@ int block_write_full_page(struct page *page, get_block_t *get_block, · 188 unmodified lines
29752965 /* Is the page fully outside i_size? (truncate in progress) */
29762966 offset = i_size & (PAGE_SIZE-1);
29772967 if (page->index >= end_index+1 || !offset) {
2978− /*
2979− * The page may have dirty, unmapped buffers. For example,
2980− * they may have been added in ext3_writepage(). Make them
2981− * freeable here, so the page does not leak.
2982− */
2983− do_invalidatepage(page, 0, PAGE_SIZE);
29842968 unlock_page(page);
29852969 return 0; /* don't care */
29862970 }
fs/fs-writeback.c+4−3
@@ -2321,7 +2321,7 @@ void __mark_inode_dirty(struct inode *inode, int flags) · 2320 unmodified lines
23212321
23222322 wb = locked_inode_to_wb_and_lock_list(inode);
23232323
2324− WARN(bdi_cap_writeback_dirty(wb->bdi) &&
2324+ WARN((wb->bdi->capabilities & BDI_CAP_WRITEBACK) &&
23252325 !test_bit(WB_registered, &wb->state),
23262326 "bdi-%s not registered\n", bdi_dev_name(wb->bdi));
23272327
@@ -2346,7 +2346,8 @@ void __mark_inode_dirty(struct inode *inode, int flags) · 18 unmodified lines
23462346 * to make sure background write-back happens
23472347 * later.
23482348 */
2349− if (bdi_cap_writeback_dirty(wb->bdi) && wakeup_bdi)
2349+ if (wakeup_bdi &&
2350+ (wb->bdi->capabilities & BDI_CAP_WRITEBACK))
23502351 wb_wakeup_delayed(wb);
23512352 return;
23522353 }
@@ -2581,7 +2582,7 @@ int write_inode_now(struct inode *inode, int sync) · 228 unmodified lines
25812582 .range_end = LLONG_MAX,
25822583 };
25832584
2584− if (!mapping_cap_writeback_dirty(inode->i_mapping))
2585+ if (!mapping_can_writeback(inode->i_mapping))
25852586 wbc.nr_to_write = 0;
25862587
25872588 might_sleep();
fs/fuse/inode.c+2−2
@@ -1049,9 +1049,9 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) · 1048 unmodified lines
10491049 if (err)
10501050 return err;
10511051
1052− sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
10531052 /* fuse does it's own writeback accounting */
1054− sb->s_bdi->capabilities = BDI_CAP_NO_ACCT_WB | BDI_CAP_STRICTLIMIT;
1053+ sb->s_bdi->capabilities &= ~BDI_CAP_WRITEBACK_ACCT;
1054+ sb->s_bdi->capabilities |= BDI_CAP_STRICTLIMIT;
10551055
10561056 /*
10571057 * For a single fuse filesystem use max 1% of dirty +
fs/namei.c+2−2
@@ -568,8 +568,8 @@ static bool path_connected(struct vfsmount *mnt, struct dentry *dentry) · 567 unmodified lines
568568 {
569569 struct super_block *sb = mnt->mnt_sb;
570570
571− /* Bind mounts and multi-root filesystems can have disconnected paths */
572− if (!(sb->s_iflags & SB_I_MULTIROOT) && (mnt->mnt_root == sb->s_root))
571+ /* Bind mounts can have disconnected paths */
572+ if (mnt->mnt_root == sb->s_root)
573573 return true;
574574
575575 return is_subdir(dentry, mnt->mnt_root);
fs/nfs/super.c+1−8
@@ -1200,13 +1200,6 @@ static void nfs_get_cache_cookie(struct super_block *sb, · 1199 unmodified lines
12001200 }
12011201 #endif
12021202
1203−static void nfs_set_readahead(struct backing_dev_info *bdi,
1204− unsigned long iomax_pages)
1205−{
1206− bdi->ra_pages = VM_READAHEAD_PAGES;
1207− bdi->io_pages = iomax_pages;
1208−}
1209−
12101203 int nfs_get_tree_common(struct fs_context *fc)
12111204 {
12121205 struct nfs_fs_context *ctx = nfs_fc2context(fc);
@@ -1251,7 +1244,7 @@ int nfs_get_tree_common(struct fs_context *fc) · 38 unmodified lines
12511244 MINOR(server->s_dev));
12521245 if (error)
12531246 goto error_splat_super;
1254− nfs_set_readahead(s->s_bdi, server->rpages);
1247+ s->s_bdi->io_pages = server->rpages;
12551248 server->super = s;
12561249 }
12571250
fs/ocfs2/cluster/heartbeat.c+10−18
@@ -1766,7 +1766,6 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, · 1765 unmodified lines
17661766 int sectsize;
17671767 char *p = (char *)page;
17681768 struct fd f;
1769− struct inode *inode;
17701769 ssize_t ret = -EINVAL;
17711770 int live_threshold;
17721771
@@ -1793,20 +1792,16 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, · 20 unmodified lines
17931792 reg->hr_block_bytes == 0)
17941793 goto out2;
17951794
1796− inode = igrab(f.file->f_mapping->host);
1797− if (inode == NULL)
1795+ if (!S_ISBLK(f.file->f_mapping->host->i_mode))
17981796 goto out2;
17991797
1800− if (!S_ISBLK(inode->i_mode))
1801− goto out3;
1802−
1803− reg->hr_bdev = I_BDEV(f.file->f_mapping->host);
1804− ret = blkdev_get(reg->hr_bdev, FMODE_WRITE | FMODE_READ, NULL);
1805− if (ret) {
1798+ reg->hr_bdev = blkdev_get_by_dev(f.file->f_mapping->host->i_rdev,
1799+ FMODE_WRITE | FMODE_READ, NULL);
1800+ if (IS_ERR(reg->hr_bdev)) {
1801+ ret = PTR_ERR(reg->hr_bdev);
18061802 reg->hr_bdev = NULL;
1807− goto out3;
1803+ goto out2;
18081804 }
1809− inode = NULL;
18101805
18111806 bdevname(reg->hr_bdev, reg->hr_dev_name);
18121807
@@ -1909,16 +1904,13 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, · 96 unmodified lines
19091904 config_item_name(&reg->hr_item), reg->hr_dev_name);
19101905
19111906 out3:
1912− iput(inode);
1907+ if (ret < 0) {
1908+ blkdev_put(reg->hr_bdev, FMODE_READ | FMODE_WRITE);
1909+ reg->hr_bdev = NULL;
1910+ }
19131911 out2:
19141912 fdput(f);
19151913 out:
1916− if (ret < 0) {
1917− if (reg->hr_bdev) {
1918− blkdev_put(reg->hr_bdev, FMODE_READ|FMODE_WRITE);
1919− reg->hr_bdev = NULL;
1920− }
1921− }
19221914 return ret;
19231915 }
19241916
fs/super.c+2−0
@@ -1256,6 +1256,8 @@ static int set_bdev_super(struct super_block *s, void *data) · 1255 unmodified lines
12561256 s->s_dev = s->s_bdev->bd_dev;
12571257 s->s_bdi = bdi_get(s->s_bdev->bd_bdi);
12581258
1259+ if (blk_queue_stable_writes(s->s_bdev->bd_disk->queue))
1260+ s->s_iflags |= SB_I_STABLE_WRITES;
12591261 return 0;
12601262 }
12611263
fs/ubifs/super.c+2−0
@@ -2177,6 +2177,8 @@ static int ubifs_fill_super(struct super_block *sb, void *data, int silent) · 2176 unmodified lines
21772177 c->vi.vol_id);
21782178 if (err)
21792179 goto out_close;
2180+ sb->s_bdi->ra_pages = 0;
2181+ sb->s_bdi->io_pages = 0;
21802182
21812183 sb->s_fs_info = c;
21822184 sb->s_magic = UBIFS_SUPER_MAGIC;
fs/vboxsf/super.c+2−0
@@ -167,6 +167,8 @@ static int vboxsf_fill_super(struct super_block *sb, struct fs_context *fc) · 166 unmodified lines
167167 err = super_setup_bdi_name(sb, "vboxsf-%d", sbi->bdi_id);
168168 if (err)
169169 goto fail_free;
170+ sb->s_bdi->ra_pages = 0;
171+ sb->s_bdi->io_pages = 0;
170172
171173 /* Turn source into a shfl_string and map the folder */
172174 size = strlen(fc->source) + 1;
include/linux/backing-dev.h+13−65
@@ -110,33 +110,14 @@ int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio); · 109 unmodified lines
110110 /*
111111 * Flags in backing_dev_info::capability
112112 *
113− * The first three flags control whether dirty pages will contribute to the
114− * VM's accounting and whether writepages() should be called for dirty pages
115− * (something that would not, for example, be appropriate for ramfs)
116− *
117− * WARNING: these flags are closely related and should not normally be
118− * used separately. The BDI_CAP_NO_ACCT_AND_WRITEBACK combines these
119− * three flags into a single convenience macro.
120− *
121− * BDI_CAP_NO_ACCT_DIRTY: Dirty pages shouldn't contribute to accounting
122− * BDI_CAP_NO_WRITEBACK: Don't write pages back
123− * BDI_CAP_NO_ACCT_WB: Don't automatically account writeback pages
124− * BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold.
125− *
126− * BDI_CAP_CGROUP_WRITEBACK: Supports cgroup-aware writeback.
127− * BDI_CAP_SYNCHRONOUS_IO: Device is so fast that asynchronous IO would be
128− * inefficient.
113+ * BDI_CAP_WRITEBACK: Supports dirty page writeback, and dirty pages
114+ * should contribute to accounting
115+ * BDI_CAP_WRITEBACK_ACCT: Automatically account writeback pages
116+ * BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold
129117 */
130−#define BDI_CAP_NO_ACCT_DIRTY 0x00000001
131−#define BDI_CAP_NO_WRITEBACK 0x00000002
132−#define BDI_CAP_NO_ACCT_WB 0x00000004
133−#define BDI_CAP_STABLE_WRITES 0x00000008
134−#define BDI_CAP_STRICTLIMIT 0x00000010
135−#define BDI_CAP_CGROUP_WRITEBACK 0x00000020
136−#define BDI_CAP_SYNCHRONOUS_IO 0x00000040
137−
138−#define BDI_CAP_NO_ACCT_AND_WRITEBACK \
139− (BDI_CAP_NO_WRITEBACK | BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_ACCT_WB)
118+#define BDI_CAP_WRITEBACK (1 << 0)
119+#define BDI_CAP_WRITEBACK_ACCT (1 << 1)
120+#define BDI_CAP_STRICTLIMIT (1 << 2)
140121
141122 extern struct backing_dev_info noop_backing_dev_info;
142123
@@ -175,41 +156,9 @@ static inline int wb_congested(struct bdi_writeback *wb, int cong_bits) · 32 unmodified lines
175156 long congestion_wait(int sync, long timeout);
176157 long wait_iff_congested(int sync, long timeout);
177158
178−static inline bool bdi_cap_synchronous_io(struct backing_dev_info *bdi)
179−{
180− return bdi->capabilities & BDI_CAP_SYNCHRONOUS_IO;
181−}
182−
183−static inline bool bdi_cap_stable_pages_required(struct backing_dev_info *bdi)
184−{
185− return bdi->capabilities & BDI_CAP_STABLE_WRITES;
186−}
187−
188−static inline bool bdi_cap_writeback_dirty(struct backing_dev_info *bdi)
189−{
190− return !(bdi->capabilities & BDI_CAP_NO_WRITEBACK);
191−}
192−
193−static inline bool bdi_cap_account_dirty(struct backing_dev_info *bdi)
194−{
195− return !(bdi->capabilities & BDI_CAP_NO_ACCT_DIRTY);
196−}
197−
198−static inline bool bdi_cap_account_writeback(struct backing_dev_info *bdi)
199−{
200− /* Paranoia: BDI_CAP_NO_WRITEBACK implies BDI_CAP_NO_ACCT_WB */
201− return !(bdi->capabilities & (BDI_CAP_NO_ACCT_WB |
202− BDI_CAP_NO_WRITEBACK));
203−}
204−
205−static inline bool mapping_cap_writeback_dirty(struct address_space *mapping)
206−{
207− return bdi_cap_writeback_dirty(inode_to_bdi(mapping->host));
208−}
209−
210−static inline bool mapping_cap_account_dirty(struct address_space *mapping)
159+static inline bool mapping_can_writeback(struct address_space *mapping)
211160 {
212− return bdi_cap_account_dirty(inode_to_bdi(mapping->host));
161+ return inode_to_bdi(mapping->host)->capabilities & BDI_CAP_WRITEBACK;
213162 }
214163
215164 static inline int bdi_sched_wait(void *word)
@@ -233,9 +182,9 @@ int inode_congested(struct inode *inode, int cong_bits); · 17 unmodified lines
233182 * inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode
234183 * @inode: inode of interest
235184 *
236− * cgroup writeback requires support from both the bdi and filesystem.
237− * Also, both memcg and iocg have to be on the default hierarchy. Test
238− * whether all conditions are met.
185+ * Cgroup writeback requires support from the filesystem. Also, both memcg and
186+ * iocg have to be on the default hierarchy. Test whether all conditions are
187+ * met.
239188 *
240189 * Note that the test result may change dynamically on the same inode
241190 * depending on how memcg and iocg are configured.
@@ -246,8 +195,7 @@ static inline bool inode_cgwb_enabled(struct inode *inode) · 4 unmodified lines
246195
247196 return cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
248197 cgroup_subsys_on_dfl(io_cgrp_subsys) &&
249− bdi_cap_account_dirty(bdi) &&
250− (bdi->capabilities & BDI_CAP_CGROUP_WRITEBACK) &&
198+ (bdi->capabilities & BDI_CAP_WRITEBACK) &&
251199 (inode->i_sb->s_iflags & SB_I_CGROUPWB);
252200 }
253201
include/linux/blk-mq.h+12−3
@@ -139,6 +139,10 @@ struct blk_mq_hw_ctx { · 138 unmodified lines
139139 * shared across request queues.
140140 */
141141 atomic_t nr_active;
142+ /**
143+ * @elevator_queued: Number of queued requests on hctx.
144+ */
145+ atomic_t elevator_queued;
142146
143147 /** @cpuhp_online: List to store request if CPU is going to die */
144148 struct hlist_node cpuhp_online;
@@ -231,6 +235,9 @@ enum hctx_type { · 86 unmodified lines
231235 * @flags: Zero or more BLK_MQ_F_* flags.
232236 * @driver_data: Pointer to data owned by the block driver that created this
233237 * tag set.
238+ * @__bitmap_tags: A shared tags sbitmap, used over all hctx's
239+ * @__breserved_tags:
240+ * A shared reserved tags sbitmap, used over all hctx's
234241 * @tags: Tag sets. One tag set per hardware queue. Has @nr_hw_queues
235242 * elements.
236243 * @tag_list_lock: Serializes tag_list accesses.
@@ -249,7 +256,10 @@ struct blk_mq_tag_set { · 12 unmodified lines
249256 unsigned int timeout;
250257 unsigned int flags;
251258 void *driver_data;
259+ atomic_t active_queues_shared_sbitmap;
252260
261+ struct sbitmap_queue __bitmap_tags;
262+ struct sbitmap_queue __breserved_tags;
253263 struct blk_mq_tags **tags;
254264
255265 struct mutex tag_list_lock;
@@ -378,12 +388,13 @@ struct blk_mq_ops { · 122 unmodified lines
378388
379389 enum {
380390 BLK_MQ_F_SHOULD_MERGE = 1 << 0,
381− BLK_MQ_F_TAG_SHARED = 1 << 1,
391+ BLK_MQ_F_TAG_QUEUE_SHARED = 1 << 1,
382392 /*
383393 * Set when this device requires underlying blk-mq device for
384394 * completing IO:
385395 */
386396 BLK_MQ_F_STACKING = 1 << 2,
397+ BLK_MQ_F_TAG_HCTX_SHARED = 1 << 3,
387398 BLK_MQ_F_BLOCKING = 1 << 5,
388399 BLK_MQ_F_NO_SCHED = 1 << 6,
389400 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
@@ -489,8 +500,6 @@ void blk_mq_kick_requeue_list(struct request_queue *q); · 99 unmodified lines
489500 void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs);
490501 void blk_mq_complete_request(struct request *rq);
491502 bool blk_mq_complete_request_remote(struct request *rq);
492−bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
493− struct bio *bio, unsigned int nr_segs);
494503 bool blk_mq_queue_stopped(struct request_queue *q);
495504 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
496505 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
include/linux/blk_types.h+3−4
@@ -20,7 +20,7 @@ typedef void (bio_end_io_t) (struct bio *); · 19 unmodified lines
2020 struct bio_crypt_ctx;
2121
2222 struct block_device {
23− dev_t bd_dev; /* not a kdev_t - it's a search key */
23+ dev_t bd_dev;
2424 int bd_openers;
2525 struct inode * bd_inode; /* will die */
2626 struct super_block * bd_super;
@@ -37,7 +37,8 @@ struct block_device { · 10 unmodified lines
3737 struct hd_struct * bd_part;
3838 /* number of times partitions within this device have been opened. */
3939 unsigned bd_part_count;
40− int bd_invalidated;
40+
41+ spinlock_t bd_size_lock; /* for bd_inode->i_size updates */
4142 struct gendisk * bd_disk;
4243 struct backing_dev_info *bd_bdi;
4344
@@ -255,8 +256,6 @@ enum { · 211 unmodified lines
255256 BIO_NO_PAGE_REF, /* don't put release vec pages */
256257 BIO_CLONED, /* doesn't own data */
257258 BIO_BOUNCED, /* bio is a bounce bio */
258− BIO_USER_MAPPED, /* contains user pages */
259− BIO_NULL_MAPPED, /* contains invalid user pages */
260259 BIO_WORKINGSET, /* contains userspace workingset pages */
261260 BIO_QUIET, /* Make BIO Quiet */
262261 BIO_CHAIN, /* chained bio, ->bi_remaining in effect */
include/linux/blkdev.h+34−10
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424 #include <linux/percpu-refcount.h>
2525 #include <linux/scatterlist.h>
2626 #include <linux/blkzoned.h>
27+#include <linux/pm.h>
2728
2829 struct module;
2930 struct scsi_ioctl_command;
@@ -458,7 +459,7 @@ struct request_queue { · 428 unmodified lines
458459
459460 #ifdef CONFIG_PM
460461 struct device *dev;
461− int rpm_status;
462+ enum rpm_status rpm_status;
462463 unsigned int nr_pending;
463464 #endif
464465
@@ -484,6 +485,8 @@ struct request_queue { · 19 unmodified lines
484485 struct timer_list timeout;
485486 struct work_struct timeout_work;
486487
488+ atomic_t nr_active_requests_shared_sbitmap;
489+
487490 struct list_head icq_list;
488491 #ifdef CONFIG_BLK_CGROUP
489492 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
@@ -603,6 +606,7 @@ struct request_queue { · 113 unmodified lines
603606 #define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */
604607 #define QUEUE_FLAG_DEAD 13 /* queue tear-down finished */
605608 #define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */
609+#define QUEUE_FLAG_STABLE_WRITES 15 /* don't modify blks until WB is done */
606610 #define QUEUE_FLAG_POLL 16 /* IO polling enabled if set */
607611 #define QUEUE_FLAG_WC 17 /* Write back caching */
608612 #define QUEUE_FLAG_FUA 18 /* device supports FUA writes */
@@ -615,6 +619,7 @@ struct request_queue { · 6 unmodified lines
615619 #define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */
616620 #define QUEUE_FLAG_ZONE_RESETALL 26 /* supports Zone Reset All */
617621 #define QUEUE_FLAG_RQ_ALLOC_TIME 27 /* record rq->alloc_time_ns */
622+#define QUEUE_FLAG_HCTX_ACTIVE 28 /* at least one blk-mq hctx is active */
618623
619624 #define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
620625 (1 << QUEUE_FLAG_SAME_COMP))
@@ -631,6 +636,8 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q); · 10 unmodified lines
631636 #define blk_queue_noxmerges(q) \
632637 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
633638 #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
639+#define blk_queue_stable_writes(q) \
640+ test_bit(QUEUE_FLAG_STABLE_WRITES, &(q)->queue_flags)
634641 #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
635642 #define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
636643 #define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
@@ -1059,11 +1066,17 @@ static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q, · 422 unmodified lines
10591066 static inline unsigned int blk_max_size_offset(struct request_queue *q,
10601067 sector_t offset)
10611068 {
1062− if (!q->limits.chunk_sectors)
1069+ unsigned int chunk_sectors = q->limits.chunk_sectors;
1070+
1071+ if (!chunk_sectors)
10631072 return q->limits.max_sectors;
10641073
1065− return min(q->limits.max_sectors, (unsigned int)(q->limits.chunk_sectors -
1066− (offset & (q->limits.chunk_sectors - 1))));
1074+ if (likely(is_power_of_2(chunk_sectors)))
1075+ chunk_sectors -= offset & (chunk_sectors - 1);
1076+ else
1077+ chunk_sectors -= sector_div(offset, chunk_sectors);
1078+
1079+ return min(q->limits.max_sectors, chunk_sectors);
10671080 }
10681081
10691082 static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
@@ -1130,6 +1143,7 @@ extern void blk_queue_max_zone_append_sectors(struct request_queue *q, · 60 unmodified lines
11301143 extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
11311144 extern void blk_queue_alignment_offset(struct request_queue *q,
11321145 unsigned int alignment);
1146+void blk_queue_update_readahead(struct request_queue *q);
11331147 extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
11341148 extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
11351149 extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
@@ -1455,10 +1469,9 @@ static inline int bdev_alignment_offset(struct block_device *bdev) · 319 unmodified lines
14551469
14561470 if (q->limits.misaligned)
14571471 return -1;
1458−
14591472 if (bdev != bdev->bd_contains)
1460− return bdev->bd_part->alignment_offset;
1461−
1473+ return queue_limit_alignment_offset(&q->limits,
1474+ bdev->bd_part->start_sect);
14621475 return q->limits.alignment_offset;
14631476 }
14641477
@@ -1498,8 +1511,8 @@ static inline int bdev_discard_alignment(struct block_device *bdev) · 33 unmodified lines
14981511 struct request_queue *q = bdev_get_queue(bdev);
14991512
15001513 if (bdev != bdev->bd_contains)
1501− return bdev->bd_part->discard_alignment;
1502−
1514+ return queue_limit_discard_alignment(&q->limits,
1515+ bdev->bd_part->start_sect);
15031516 return q->limits.discard_alignment;
15041517 }
15051518
@@ -1930,6 +1943,11 @@ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, · 424 unmodified lines
19301943 void disk_end_io_acct(struct gendisk *disk, unsigned int op,
19311944 unsigned long start_time);
19321945
1946+unsigned long part_start_io_acct(struct gendisk *disk, struct hd_struct **part,
1947+ struct bio *bio);
1948+void part_end_io_acct(struct hd_struct *part, struct bio *bio,
1949+ unsigned long start_time);
1950+
19331951 /**
19341952 * bio_start_io_acct - start I/O accounting for bio based drivers
19351953 * @bio: bio to start account for
@@ -1967,7 +1985,6 @@ void blkdev_show(struct seq_file *seqf, off_t offset); · 31 unmodified lines
19671985 #define BLKDEV_MAJOR_MAX 0
19681986 #endif
19691987
1970−int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder);
19711988 struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
19721989 void *holder);
19731990 struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder);
@@ -1984,11 +2001,18 @@ void bdput(struct block_device *); · 10 unmodified lines
19842001
19852002 #ifdef CONFIG_BLOCK
19862003 void invalidate_bdev(struct block_device *bdev);
2004+int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
2005+ loff_t lend);
19872006 int sync_blockdev(struct block_device *bdev);
19882007 #else
19892008 static inline void invalidate_bdev(struct block_device *bdev)
19902009 {
19912010 }
2011+static inline int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
2012+ loff_t lstart, loff_t lend)
2013+{
2014+ return 0;
2015+}
19922016 static inline int sync_blockdev(struct block_device *bdev)
19932017 {
19942018 return 0;
include/linux/fs.h+1−1
@@ -1385,7 +1385,7 @@ extern int send_sigurg(struct fown_struct *fown); · 1384 unmodified lines
13851385 #define SB_I_CGROUPWB 0x00000001 /* cgroup-aware writeback enabled */
13861386 #define SB_I_NOEXEC 0x00000002 /* Ignore executables on this fs */
13871387 #define SB_I_NODEV 0x00000004 /* Ignore devices on this fs */
1388−#define SB_I_MULTIROOT 0x00000008 /* Multiple roots to the dentry tree */
1388+#define SB_I_STABLE_WRITES 0x00000008 /* don't modify blks until WB is done */
13891389
13901390 /* sb->s_iflags to limit user namespace mounts */
13911391 #define SB_I_USERNS_VISIBLE 0x00000010 /* fstype already mounted */
include/linux/genhd.h+7−8
@@ -65,8 +65,6 @@ struct hd_struct { · 64 unmodified lines
6565 struct disk_stats __percpu *dkstats;
6666 struct percpu_ref ref;
6767
68− sector_t alignment_offset;
69− unsigned int discard_alignment;
7068 struct device __dev;
7169 struct kobject *holder_dir;
7270 int policy, partno;
@@ -193,6 +191,8 @@ struct gendisk { · 120 unmodified lines
193191 void *private_data;
194192
195193 int flags;
194+ unsigned long state;
195+#define GD_NEED_PART_SCAN 0
196196 struct rw_semaphore lookup_sem;
197197 struct kobject *slave_dir;
198198
@@ -315,9 +315,8 @@ static inline int get_disk_ro(struct gendisk *disk) · 116 unmodified lines
315315 extern void disk_block_events(struct gendisk *disk);
316316 extern void disk_unblock_events(struct gendisk *disk);
317317 extern void disk_flush_events(struct gendisk *disk, unsigned int mask);
318−extern void set_capacity_revalidate_and_notify(struct gendisk *disk,
319− sector_t size, bool revalidate);
320−extern unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask);
318+void set_capacity_revalidate_and_notify(struct gendisk *disk, sector_t size,
319+ bool update_bdev);
321320
322321 /* drivers/char/random.c */
323322 extern void add_disk_randomness(struct gendisk *disk) __latent_entropy;
@@ -372,10 +371,10 @@ extern void blk_unregister_region(dev_t devt, unsigned long range); · 48 unmodified lines
372371 int register_blkdev(unsigned int major, const char *name);
373372 void unregister_blkdev(unsigned int major, const char *name);
374373
375−int revalidate_disk(struct gendisk *disk);
376−int check_disk_change(struct block_device *bdev);
374+void revalidate_disk_size(struct gendisk *disk, bool verbose);
375+bool bdev_check_media_change(struct block_device *bdev);
377376 int __invalidate_device(struct block_device *bdev, bool kill_dirty);
378−void bd_set_size(struct block_device *bdev, loff_t size);
377+void bd_set_nr_sectors(struct block_device *bdev, sector_t sectors);
379378
380379 /* for drivers/char/raw.c: */
381380 int blkdev_ioctl(struct block_device *, fmode_t, unsigned, unsigned long);
include/linux/ide.h+0−2
@@ -490,8 +490,6 @@ enum { · 489 unmodified lines
490490 IDE_DFLAG_NOPROBE = BIT(9),
491491 /* need to do check_media_change() */
492492 IDE_DFLAG_REMOVABLE = BIT(10),
493− /* needed for removable devices */
494− IDE_DFLAG_ATTACH = BIT(11),
495493 IDE_DFLAG_FORCED_GEOM = BIT(12),
496494 /* disallow setting unmask bit */
497495 IDE_DFLAG_NO_UNMASK = BIT(13),
include/linux/suspend.h+2−2
@@ -473,9 +473,9 @@ static inline int hibernate_quiet_exec(int (*func)(void *data), void *data) { · 472 unmodified lines
473473 #endif /* CONFIG_HIBERNATION */
474474
475475 #ifdef CONFIG_HIBERNATION_SNAPSHOT_DEV
476−int is_hibernate_resume_dev(const struct inode *);
476+int is_hibernate_resume_dev(dev_t dev);
477477 #else
478−static inline int is_hibernate_resume_dev(const struct inode *i) { return 0; }
478+static inline int is_hibernate_resume_dev(dev_t dev) { return 0; }
479479 #endif
480480
481481 /* Hibernation and suspend events */
include/linux/swap.h+2−1
@@ -467,7 +467,8 @@ extern int swapcache_prepare(swp_entry_t); · 466 unmodified lines
467467 extern void swap_free(swp_entry_t);
468468 extern void swapcache_free_entries(swp_entry_t *entries, int n);
469469 extern int free_swap_and_cache(swp_entry_t);
470−extern int swap_type_of(dev_t, sector_t, struct block_device **);
470+int swap_type_of(dev_t device, sector_t offset);
471+int find_first_swap(dev_t *device);
471472 extern unsigned int count_swap_pages(int, int);
472473 extern sector_t map_swap_page(struct page *, struct block_device **);
473474 extern sector_t swapdev_block(int, pgoff_t);
include/trace/events/iocost.h+9−17
@@ -26,7 +26,6 @@ TRACE_EVENT(iocost_iocg_activate, · 25 unmodified lines
2626 __field(u64, vrate)
2727 __field(u64, last_period)
2828 __field(u64, cur_period)
29− __field(u64, last_vtime)
3029 __field(u64, vtime)
3130 __field(u32, weight)
3231 __field(u32, inuse)
@@ -42,7 +41,6 @@ TRACE_EVENT(iocost_iocg_activate, · 9 unmodified lines
4241 __entry->vrate = now->vrate;
4342 __entry->last_period = last_period;
4443 __entry->cur_period = cur_period;
45− __entry->last_vtime = iocg->last_vtime;
4644 __entry->vtime = vtime;
4745 __entry->weight = iocg->weight;
4846 __entry->inuse = iocg->inuse;
@@ -51,13 +49,12 @@ TRACE_EVENT(iocost_iocg_activate, · 2 unmodified lines
5149 ),
5250
5351 TP_printk("[%s:%s] now=%llu:%llu vrate=%llu "
54− "period=%llu->%llu vtime=%llu->%llu "
52+ "period=%llu->%llu vtime=%llu "
5553 "weight=%u/%u hweight=%llu/%llu",
5654 __get_str(devname), __get_str(cgroup),
5755 __entry->now, __entry->vnow, __entry->vrate,
5856 __entry->last_period, __entry->cur_period,
59− __entry->last_vtime, __entry->vtime,
60− __entry->inuse, __entry->weight,
57+ __entry->vtime, __entry->inuse, __entry->weight,
6158 __entry->hweight_inuse, __entry->hweight_active
6259 )
6360 );
@@ -98,7 +95,7 @@ DECLARE_EVENT_CLASS(iocg_inuse_update, · 34 unmodified lines
9895 )
9996 );
10097
101−DEFINE_EVENT(iocg_inuse_update, iocost_inuse_takeback,
98+DEFINE_EVENT(iocg_inuse_update, iocost_inuse_shortage,
10299
103100 TP_PROTO(struct ioc_gq *iocg, const char *path, struct ioc_now *now,
104101 u32 old_inuse, u32 new_inuse,
@@ -108,7 +105,7 @@ DEFINE_EVENT(iocg_inuse_update, iocost_inuse_takeback, · 3 unmodified lines
108105 old_hw_inuse, new_hw_inuse)
109106 );
110107
111−DEFINE_EVENT(iocg_inuse_update, iocost_inuse_giveaway,
108+DEFINE_EVENT(iocg_inuse_update, iocost_inuse_transfer,
112109
113110 TP_PROTO(struct ioc_gq *iocg, const char *path, struct ioc_now *now,
114111 u32 old_inuse, u32 new_inuse,
@@ -118,7 +115,7 @@ DEFINE_EVENT(iocg_inuse_update, iocost_inuse_giveaway, · 3 unmodified lines
118115 old_hw_inuse, new_hw_inuse)
119116 );
120117
121−DEFINE_EVENT(iocg_inuse_update, iocost_inuse_reset,
118+DEFINE_EVENT(iocg_inuse_update, iocost_inuse_adjust,
122119
123120 TP_PROTO(struct ioc_gq *iocg, const char *path, struct ioc_now *now,
124121 u32 old_inuse, u32 new_inuse,
@@ -131,11 +128,9 @@ DEFINE_EVENT(iocg_inuse_update, iocost_inuse_reset, · 6 unmodified lines
131128 TRACE_EVENT(iocost_ioc_vrate_adj,
132129
133130 TP_PROTO(struct ioc *ioc, u64 new_vrate, u32 *missed_ppm,
134− u32 rq_wait_pct, int nr_lagging, int nr_shortages,
135− int nr_surpluses),
131+ u32 rq_wait_pct, int nr_lagging, int nr_shortages),
136132
137− TP_ARGS(ioc, new_vrate, missed_ppm, rq_wait_pct, nr_lagging, nr_shortages,
138− nr_surpluses),
133+ TP_ARGS(ioc, new_vrate, missed_ppm, rq_wait_pct, nr_lagging, nr_shortages),
139134
140135 TP_STRUCT__entry (
141136 __string(devname, ioc_name(ioc))
@@ -147,7 +142,6 @@ TRACE_EVENT(iocost_ioc_vrate_adj, · 5 unmodified lines
147142 __field(u32, rq_wait_pct)
148143 __field(int, nr_lagging)
149144 __field(int, nr_shortages)
150− __field(int, nr_surpluses)
151145 ),
152146
153147 TP_fast_assign(
@@ -160,15 +154,13 @@ TRACE_EVENT(iocost_ioc_vrate_adj, · 6 unmodified lines
160154 __entry->rq_wait_pct = rq_wait_pct;
161155 __entry->nr_lagging = nr_lagging;
162156 __entry->nr_shortages = nr_shortages;
163− __entry->nr_surpluses = nr_surpluses;
164157 ),
165158
166− TP_printk("[%s] vrate=%llu->%llu busy=%d missed_ppm=%u:%u rq_wait_pct=%u lagging=%d shortages=%d surpluses=%d",
159+ TP_printk("[%s] vrate=%llu->%llu busy=%d missed_ppm=%u:%u rq_wait_pct=%u lagging=%d shortages=%d",
167160 __get_str(devname), __entry->old_vrate, __entry->new_vrate,
168161 __entry->busy_level,
169162 __entry->read_missed_ppm, __entry->write_missed_ppm,
170− __entry->rq_wait_pct, __entry->nr_lagging, __entry->nr_shortages,
171− __entry->nr_surpluses
163+ __entry->rq_wait_pct, __entry->nr_lagging, __entry->nr_shortages
172164 )
173165 );
174166
include/uapi/linux/capability.h+2−0
@@ -288,6 +288,8 @@ struct vfs_ns_cap_data { · 287 unmodified lines
288288 processes and setting the scheduling algorithm used by another
289289 process. */
290290 /* Allow setting cpu affinity on other processes */
291+/* Allow setting realtime ioprio class */
292+/* Allow setting ioprio class on other processes */
291293
292294 #define CAP_SYS_NICE 23
293295
kernel/power/swap.c+9−12
@@ -335,26 +335,23 @@ static int swsusp_swap_check(void) · 334 unmodified lines
335335 {
336336 int res;
337337
338− res = swap_type_of(swsusp_resume_device, swsusp_resume_block,
339− &hib_resume_bdev);
338+ if (swsusp_resume_device)
339+ res = swap_type_of(swsusp_resume_device, swsusp_resume_block);
340+ else
341+ res = find_first_swap(&swsusp_resume_device);
340342 if (res < 0)
341343 return res;
342−
343344 root_swap = res;
344− res = blkdev_get(hib_resume_bdev, FMODE_WRITE, NULL);
345− if (res)
346− return res;
345+
346+ hib_resume_bdev = blkdev_get_by_dev(swsusp_resume_device, FMODE_WRITE,
347+ NULL);
348+ if (IS_ERR(hib_resume_bdev))
349+ return PTR_ERR(hib_resume_bdev);
347350
348351 res = set_blocksize(hib_resume_bdev, PAGE_SIZE);
349352 if (res < 0)
350353 blkdev_put(hib_resume_bdev, FMODE_WRITE);
351354
352− /*
353− * Update the resume device to the one actually used,
354− * so the test_resume mode can use it in case it is
355− * invoked from hibernate() to test the snapshot.
356− */
357− swsusp_resume_device = hib_resume_bdev->bd_dev;
358355 return res;
359356 }
360357
kernel/power/user.c+9−17
@@ -35,12 +35,12 @@ static struct snapshot_data { · 34 unmodified lines
3535 bool ready;
3636 bool platform_support;
3737 bool free_bitmaps;
38− struct inode *bd_inode;
38+ dev_t dev;
3939 } snapshot_state;
4040
41−int is_hibernate_resume_dev(const struct inode *bd_inode)
41+int is_hibernate_resume_dev(dev_t dev)
4242 {
43− return hibernation_available() && snapshot_state.bd_inode == bd_inode;
43+ return hibernation_available() && snapshot_state.dev == dev;
4444 }
4545
4646 static int snapshot_open(struct inode *inode, struct file *filp)
@@ -69,8 +69,7 @@ static int snapshot_open(struct inode *inode, struct file *filp) · 22 unmodified lines
6969 memset(&data->handle, 0, sizeof(struct snapshot_handle));
7070 if ((filp->f_flags & O_ACCMODE) == O_RDONLY) {
7171 /* Hibernating. The image device should be accessible. */
72− data->swap = swsusp_resume_device ?
73− swap_type_of(swsusp_resume_device, 0, NULL) : -1;
72+ data->swap = swap_type_of(swsusp_resume_device, 0);
7473 data->mode = O_RDONLY;
7574 data->free_bitmaps = false;
7675 error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls);
@@ -101,7 +100,7 @@ static int snapshot_open(struct inode *inode, struct file *filp) · 24 unmodified lines
101100 data->frozen = false;
102101 data->ready = false;
103102 data->platform_support = false;
104− data->bd_inode = NULL;
103+ data->dev = 0;
105104
106105 Unlock:
107106 unlock_system_sleep();
@@ -117,7 +116,7 @@ static int snapshot_release(struct inode *inode, struct file *filp) · 9 unmodified lines
117116
118117 swsusp_free();
119118 data = filp->private_data;
120− data->bd_inode = NULL;
119+ data->dev = 0;
121120 free_all_swap_pages(data->swap);
122121 if (data->frozen) {
123122 pm_restore_gfp_mask();
@@ -210,7 +209,6 @@ struct compat_resume_swap_area { · 86 unmodified lines
210209 static int snapshot_set_swap_area(struct snapshot_data *data,
211210 void __user *argp)
212211 {
213− struct block_device *bdev;
214212 sector_t offset;
215213 dev_t swdev;
216214
@@ -237,16 +235,10 @@ static int snapshot_set_swap_area(struct snapshot_data *data, · 20 unmodified lines
237235 * User space encodes device types as two-byte values,
238236 * so we need to recode them
239237 */
240− if (!swdev) {
241− data->swap = -1;
242− return -EINVAL;
243− }
244− data->swap = swap_type_of(swdev, offset, &bdev);
238+ data->swap = swap_type_of(swdev, offset);
245239 if (data->swap < 0)
246− return -ENODEV;
247−
248− data->bd_inode = bdev->bd_inode;
249− bdput(bdev);
240+ return swdev ? -ENODEV : -EINVAL;
241+ data->dev = swdev;
250242 return 0;
251243 }
252244
kernel/trace/blktrace.c+1−1
@@ -793,7 +793,7 @@ static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) · 792 unmodified lines
793793 return cgroup_id(bio_blkcg(bio)->css.cgroup);
794794 }
795795 #else
796−u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)
796+static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)
797797 {
798798 return 0;
799799 }
mm/backing-dev.c+7−7
@@ -14,9 +14,7 @@ · 13 unmodified lines
1414 #include <linux/device.h>
1515 #include <trace/events/writeback.h>
1616
17−struct backing_dev_info noop_backing_dev_info = {
18− .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK,
19−};
17+struct backing_dev_info noop_backing_dev_info;
2018 EXPORT_SYMBOL_GPL(noop_backing_dev_info);
2119
2220 static struct class *bdi_class;
@@ -204,10 +202,9 @@ static ssize_t stable_pages_required_show(struct device *dev, · 181 unmodified lines
204202 struct device_attribute *attr,
205203 char *page)
206204 {
207− struct backing_dev_info *bdi = dev_get_drvdata(dev);
208−
209− return snprintf(page, PAGE_SIZE-1, "%d\n",
210− bdi_cap_stable_pages_required(bdi) ? 1 : 0);
205+ dev_warn_once(dev,
206+ "the stable_pages_required attribute has been removed. Use the stable_writes queue attribute instead.\n");
207+ return snprintf(page, PAGE_SIZE-1, "%d\n", 0);
211208 }
212209 static DEVICE_ATTR_RO(stable_pages_required);
213210
@@ -746,6 +743,9 @@ struct backing_dev_info *bdi_alloc(int node_id) · 532 unmodified lines
746743 kfree(bdi);
747744 return NULL;
748745 }
746+ bdi->capabilities = BDI_CAP_WRITEBACK | BDI_CAP_WRITEBACK_ACCT;
747+ bdi->ra_pages = VM_READAHEAD_PAGES;
748+ bdi->io_pages = VM_READAHEAD_PAGES;
749749 return bdi;
750750 }
751751 EXPORT_SYMBOL(bdi_alloc);
mm/filemap.c+2−2
@@ -414,7 +414,7 @@ int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, · 413 unmodified lines
414414 .range_end = end,
415415 };
416416
417− if (!mapping_cap_writeback_dirty(mapping) ||
417+ if (!mapping_can_writeback(mapping) ||
418418 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
419419 return 0;
420420
@@ -1800,7 +1800,7 @@ repeat: · 1379 unmodified lines
18001800 no_page:
18011801 if (!page && (fgp_flags & FGP_CREAT)) {
18021802 int err;
1803− if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
1803+ if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
18041804 gfp_mask |= __GFP_WRITE;
18051805 if (fgp_flags & FGP_NOFS)
18061806 gfp_mask &= ~__GFP_FS;
mm/memcontrol.c+1−1
@@ -5643,7 +5643,7 @@ static int mem_cgroup_move_account(struct page *page, · 5642 unmodified lines
56435643 if (PageDirty(page)) {
56445644 struct address_space *mapping = page_mapping(page);
56455645
5646− if (mapping_cap_account_dirty(mapping)) {
5646+ if (mapping_can_writeback(mapping)) {
56475647 __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
56485648 -nr_pages);
56495649 __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
mm/memory-failure.c+1−1
@@ -1006,7 +1006,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, · 1005 unmodified lines
10061006 */
10071007 mapping = page_mapping(hpage);
10081008 if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
1009− mapping_cap_writeback_dirty(mapping)) {
1009+ mapping_can_writeback(mapping)) {
10101010 if (page_mkclean(hpage)) {
10111011 SetPageDirty(hpage);
10121012 } else {
mm/migrate.c+1−1
@@ -503,7 +503,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 502 unmodified lines
503503 __dec_lruvec_state(old_lruvec, NR_SHMEM);
504504 __inc_lruvec_state(new_lruvec, NR_SHMEM);
505505 }
506− if (dirty && mapping_cap_account_dirty(mapping)) {
506+ if (dirty && mapping_can_writeback(mapping)) {
507507 __dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);
508508 __dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING);
509509 __inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY);
mm/mmap.c+1−1
@@ -1666,7 +1666,7 @@ int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) · 1665 unmodified lines
16661666
16671667 /* Can the mapping track the dirty pages? */
16681668 return vma->vm_file && vma->vm_file->f_mapping &&
1669− mapping_cap_account_dirty(vma->vm_file->f_mapping);
1669+ mapping_can_writeback(vma->vm_file->f_mapping);
16701670 }
16711671
16721672 /*
mm/page-writeback.c+9−9
@@ -1882,7 +1882,7 @@ void balance_dirty_pages_ratelimited(struct address_space *mapping) · 1881 unmodified lines
18821882 int ratelimit;
18831883 int *p;
18841884
1885− if (!bdi_cap_account_dirty(bdi))
1885+ if (!(bdi->capabilities & BDI_CAP_WRITEBACK))
18861886 return;
18871887
18881888 if (inode_cgwb_enabled(inode))
@@ -2423,7 +2423,7 @@ void account_page_dirtied(struct page *page, struct address_space *mapping) · 534 unmodified lines
24232423
24242424 trace_writeback_dirty_page(page, mapping);
24252425
2426− if (mapping_cap_account_dirty(mapping)) {
2426+ if (mapping_can_writeback(mapping)) {
24272427 struct bdi_writeback *wb;
24282428
24292429 inode_attach_wb(inode, page);
@@ -2450,7 +2450,7 @@ void account_page_dirtied(struct page *page, struct address_space *mapping) · 20 unmodified lines
24502450 void account_page_cleaned(struct page *page, struct address_space *mapping,
24512451 struct bdi_writeback *wb)
24522452 {
2453− if (mapping_cap_account_dirty(mapping)) {
2453+ if (mapping_can_writeback(mapping)) {
24542454 dec_lruvec_page_state(page, NR_FILE_DIRTY);
24552455 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
24562456 dec_wb_stat(wb, WB_RECLAIMABLE);
@@ -2513,7 +2513,7 @@ void account_page_redirty(struct page *page) · 56 unmodified lines
25132513 {
25142514 struct address_space *mapping = page->mapping;
25152515
2516− if (mapping && mapping_cap_account_dirty(mapping)) {
2516+ if (mapping && mapping_can_writeback(mapping)) {
25172517 struct inode *inode = mapping->host;
25182518 struct bdi_writeback *wb;
25192519 struct wb_lock_cookie cookie = {};
@@ -2625,7 +2625,7 @@ void __cancel_dirty_page(struct page *page) · 105 unmodified lines
26252625 {
26262626 struct address_space *mapping = page_mapping(page);
26272627
2628− if (mapping_cap_account_dirty(mapping)) {
2628+ if (mapping_can_writeback(mapping)) {
26292629 struct inode *inode = mapping->host;
26302630 struct bdi_writeback *wb;
26312631 struct wb_lock_cookie cookie = {};
@@ -2665,7 +2665,7 @@ int clear_page_dirty_for_io(struct page *page) · 33 unmodified lines
26652665
26662666 VM_BUG_ON_PAGE(!PageLocked(page), page);
26672667
2668− if (mapping && mapping_cap_account_dirty(mapping)) {
2668+ if (mapping && mapping_can_writeback(mapping)) {
26692669 struct inode *inode = mapping->host;
26702670 struct bdi_writeback *wb;
26712671 struct wb_lock_cookie cookie = {};
@@ -2738,7 +2738,7 @@ int test_clear_page_writeback(struct page *page) · 66 unmodified lines
27382738 if (ret) {
27392739 __xa_clear_mark(&mapping->i_pages, page_index(page),
27402740 PAGECACHE_TAG_WRITEBACK);
2741− if (bdi_cap_account_writeback(bdi)) {
2741+ if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
27422742 struct bdi_writeback *wb = inode_to_wb(inode);
27432743
27442744 dec_wb_stat(wb, WB_WRITEBACK);
@@ -2791,7 +2791,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write) · 46 unmodified lines
27912791 PAGECACHE_TAG_WRITEBACK);
27922792
27932793 xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
2794− if (bdi_cap_account_writeback(bdi))
2794+ if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT)
27952795 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK);
27962796
27972797 /*
@@ -2849,7 +2849,7 @@ EXPORT_SYMBOL_GPL(wait_on_page_writeback); · 51 unmodified lines
28492849 */
28502850 void wait_for_stable_page(struct page *page)
28512851 {
2852− if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host)))
2852+ if (page->mapping->host->i_sb->s_iflags & SB_I_STABLE_WRITES)
28532853 wait_on_page_writeback(page);
28542854 }
28552855 EXPORT_SYMBOL_GPL(wait_for_stable_page);
mm/page_io.c+10−8
@@ -403,15 +403,17 @@ int swap_readpage(struct page *page, bool synchronous) · 402 unmodified lines
403403 goto out;
404404 }
405405
406− ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
407− if (!ret) {
408− if (trylock_page(page)) {
409− swap_slot_free_notify(page);
410− unlock_page(page);
411− }
406+ if (sis->flags & SWP_SYNCHRONOUS_IO) {
407+ ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
408+ if (!ret) {
409+ if (trylock_page(page)) {
410+ swap_slot_free_notify(page);
411+ unlock_page(page);
412+ }
412413
413− count_vm_event(PSWPIN);
414− goto out;
414+ count_vm_event(PSWPIN);
415+ goto out;
416+ }
415417 }
416418
417419 ret = 0;
mm/swapfile.c+26−23
@@ -1801,13 +1801,12 @@ int free_swap_and_cache(swp_entry_t entry) · 1800 unmodified lines
18011801 *
18021802 * This is needed for the suspend to disk (aka swsusp).
18031803 */
1804−int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
1804+int swap_type_of(dev_t device, sector_t offset)
18051805 {
1806− struct block_device *bdev = NULL;
18071806 int type;
18081807
1809− if (device)
1810− bdev = bdget(device);
1808+ if (!device)
1809+ return -1;
18111810
18121811 spin_lock(&swap_lock);
18131812 for (type = 0; type < nr_swapfiles; type++) {
@@ -1816,30 +1815,34 @@ int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p) · 2 unmodified lines
18161815 if (!(sis->flags & SWP_WRITEOK))
18171816 continue;
18181817
1819− if (!bdev) {
1820− if (bdev_p)
1821− *bdev_p = bdgrab(sis->bdev);
1822−
1823− spin_unlock(&swap_lock);
1824− return type;
1825− }
1826− if (bdev == sis->bdev) {
1818+ if (device == sis->bdev->bd_dev) {
18271819 struct swap_extent *se = first_se(sis);
18281820
18291821 if (se->start_block == offset) {
1830− if (bdev_p)
1831− *bdev_p = bdgrab(sis->bdev);
1832−
18331822 spin_unlock(&swap_lock);
1834− bdput(bdev);
18351823 return type;
18361824 }
18371825 }
18381826 }
18391827 spin_unlock(&swap_lock);
1840− if (bdev)
1841− bdput(bdev);
1828+ return -ENODEV;
1829+}
18421830
1831+int find_first_swap(dev_t *device)
1832+{
1833+ int type;
1834+
1835+ spin_lock(&swap_lock);
1836+ for (type = 0; type < nr_swapfiles; type++) {
1837+ struct swap_info_struct *sis = swap_info[type];
1838+
1839+ if (!(sis->flags & SWP_WRITEOK))
1840+ continue;
1841+ *device = sis->bdev->bd_dev;
1842+ spin_unlock(&swap_lock);
1843+ return type;
1844+ }
1845+ spin_unlock(&swap_lock);
18431846 return -ENODEV;
18441847 }
18451848
@@ -2920,10 +2923,10 @@ static int claim_swapfile(struct swap_info_struct *p, struct inode *inode) · 1074 unmodified lines
29202923 int error;
29212924
29222925 if (S_ISBLK(inode->i_mode)) {
2923− p->bdev = bdgrab(I_BDEV(inode));
2924− error = blkdev_get(p->bdev,
2926+ p->bdev = blkdev_get_by_dev(inode->i_rdev,
29252927 FMODE_READ | FMODE_WRITE | FMODE_EXCL, p);
2926− if (error < 0) {
2928+ if (IS_ERR(p->bdev)) {
2929+ error = PTR_ERR(p->bdev);
29272930 p->bdev = NULL;
29282931 return error;
29292932 }
@@ -3234,10 +3237,10 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) · 304 unmodified lines
32343237 goto bad_swap_unlock_inode;
32353238 }
32363239
3237− if (bdi_cap_stable_pages_required(inode_to_bdi(inode)))
3240+ if (p->bdev && blk_queue_stable_writes(p->bdev->bd_disk->queue))
32383241 p->flags |= SWP_STABLE_WRITES;
32393242
3240− if (bdi_cap_synchronous_io(inode_to_bdi(inode)))
3243+ if (p->bdev && p->bdev->bd_disk->fops->rw_page)
32413244 p->flags |= SWP_SYNCHRONOUS_IO;
32423245
32433246 if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
tools/cgroup/iocost_monitor.py+19−35
@@ -45,8 +45,7 @@ except: · 44 unmodified lines
4545 err('The kernel does not have iocost enabled')
4646
4747 IOC_RUNNING = prog['IOC_RUNNING'].value_()
48−NR_USAGE_SLOTS = prog['NR_USAGE_SLOTS'].value_()
49−HWEIGHT_WHOLE = prog['HWEIGHT_WHOLE'].value_()
48+WEIGHT_ONE = prog['WEIGHT_ONE'].value_()
5049 VTIME_PER_SEC = prog['VTIME_PER_SEC'].value_()
5150 VTIME_PER_USEC = prog['VTIME_PER_USEC'].value_()
5251 AUTOP_SSD_FAST = prog['AUTOP_SSD_FAST'].value_()
@@ -100,7 +99,7 @@ class IocStat: · 47 unmodified lines
10099 self.period_ms = ioc.period_us.value_() / 1_000
101100 self.period_at = ioc.period_at.value_() / 1_000_000
102101 self.vperiod_at = ioc.period_at_vtime.value_() / VTIME_PER_SEC
103− self.vrate_pct = ioc.vtime_rate.counter.value_() * 100 / VTIME_PER_USEC
102+ self.vrate_pct = ioc.vtime_base_rate.value_() * 100 / VTIME_PER_USEC
104103 self.busy_level = ioc.busy_level.value_()
105104 self.autop_idx = ioc.autop_idx.value_()
106105 self.user_cost_model = ioc.user_cost_model.value_()
@@ -136,7 +135,7 @@ class IocStat: · 29 unmodified lines
136135
137136 def table_header_str(self):
138137 return f'{"":25} active {"weight":>9} {"hweight%":>13} {"inflt%":>6} ' \
139− f'{"dbt":>3} {"delay":>6} {"usages%"}'
138+ f'{"debt":>7} {"delay":>7} {"usage%"}'
140139
141140 class IocgStat:
142141 def __init__(self, iocg):
@@ -144,11 +143,11 @@ class IocgStat: · 1 unmodified line
144143 blkg = iocg.pd.blkg
145144
146145 self.is_active = not list_empty(iocg.active_list.address_of_())
147− self.weight = iocg.weight.value_()
148− self.active = iocg.active.value_()
149− self.inuse = iocg.inuse.value_()
150− self.hwa_pct = iocg.hweight_active.value_() * 100 / HWEIGHT_WHOLE
151− self.hwi_pct = iocg.hweight_inuse.value_() * 100 / HWEIGHT_WHOLE
146+ self.weight = iocg.weight.value_() / WEIGHT_ONE
147+ self.active = iocg.active.value_() / WEIGHT_ONE
148+ self.inuse = iocg.inuse.value_() / WEIGHT_ONE
149+ self.hwa_pct = iocg.hweight_active.value_() * 100 / WEIGHT_ONE
150+ self.hwi_pct = iocg.hweight_inuse.value_() * 100 / WEIGHT_ONE
152151 self.address = iocg.value_()
153152
154153 vdone = iocg.done_vtime.counter.value_()
@@ -160,23 +159,13 @@ class IocgStat: · 5 unmodified lines
160159 else:
161160 self.inflight_pct = 0
162161
163− # vdebt used to be an atomic64_t and is now u64, support both
164− try:
165− self.debt_ms = iocg.abs_vdebt.counter.value_() / VTIME_PER_USEC / 1000
166− except:
167− self.debt_ms = iocg.abs_vdebt.value_() / VTIME_PER_USEC / 1000
168−
169− self.use_delay = blkg.use_delay.counter.value_()
170− self.delay_ms = blkg.delay_nsec.counter.value_() / 1_000_000
171−
172− usage_idx = iocg.usage_idx.value_()
173− self.usages = []
174− self.usage = 0
175− for i in range(NR_USAGE_SLOTS):
176− usage = iocg.usages[(usage_idx + 1 + i) % NR_USAGE_SLOTS].value_()
177− upct = usage * 100 / HWEIGHT_WHOLE
178− self.usages.append(upct)
179− self.usage = max(self.usage, upct)
162+ self.usage = (100 * iocg.usage_delta_us.value_() /
163+ ioc.period_us.value_()) if self.active else 0
164+ self.debt_ms = iocg.abs_vdebt.value_() / VTIME_PER_USEC / 1000
165+ if blkg.use_delay.counter.value_() != 0:
166+ self.delay_ms = blkg.delay_nsec.counter.value_() / 1_000_000
167+ else:
168+ self.delay_ms = 0
180169
181170 def dict(self, now, path):
182171 out = { 'cgroup' : path,
@@ -189,25 +178,20 @@ class IocgStat: · 6 unmodified lines
189178 'hweight_inuse_pct' : self.hwi_pct,
190179 'inflight_pct' : self.inflight_pct,
191180 'debt_ms' : self.debt_ms,
192− 'use_delay' : self.use_delay,
193181 'delay_ms' : self.delay_ms,
194182 'usage_pct' : self.usage,
195183 'address' : self.address }
196− for i in range(len(self.usages)):
197− out[f'usage_pct_{i}'] = str(self.usages[i])
198184 return out
199185
200186 def table_row_str(self, path):
201187 out = f'{path[-28:]:28} ' \
202188 f'{"*" if self.is_active else " "} ' \
203− f'{self.inuse:5}/{self.active:5} ' \
189+ f'{round(self.inuse):5}/{round(self.active):5} ' \
204190 f'{self.hwi_pct:6.2f}/{self.hwa_pct:6.2f} ' \
205191 f'{self.inflight_pct:6.2f} ' \
206− f'{min(math.ceil(self.debt_ms), 999):3} ' \
207− f'{min(self.use_delay, 99):2}*'\
208− f'{min(math.ceil(self.delay_ms), 999):03} '
209− for u in self.usages:
210− out += f'{min(round(u), 999):03d}:'
192+ f'{self.debt_ms:7.2f} ' \
193+ f'{self.delay_ms:7.2f} '\
194+ f'{min(self.usage, 999):6.2f}'
211195 out = out.rstrip(':')
212196 return out
213197