lael-labs/linux

Documentation/filesystems/locking.rst+0−3

@@ -488,9 +488,6 @@ getgeo: no · 487 unmodified lines
488	swap_slot_free_notify: no (see below)	488	swap_slot_free_notify: no (see below)
489	======================= ===================	489	======================= ===================
490		490
491	unlock_native_capacity and revalidate_disk are called only from
492	check_disk_change().
493
494	swap_slot_free_notify is called with swap_lock and sometimes the page lock	491	swap_slot_free_notify is called with swap_lock and sometimes the page lock
495	held.	492	held.
496		493

block/Kconfig+0−2

@@ -161,8 +161,6 @@ config BLK_WBT_MQ · 160 unmodified lines
161	depends on BLK_WBT	161	depends on BLK_WBT
162	help	162	help
163	Enable writeback throttling by default on multiqueue devices.	163	Enable writeback throttling by default on multiqueue devices.
164	Multiqueue currently doesn't have support for IO scheduling,
165	enabling this option is recommended.
166		164
167	config BLK_DEBUG_FS	165	config BLK_DEBUG_FS
168	bool "Block layer debugging information in debugfs"	166	bool "Block layer debugging information in debugfs"

block/bfq-iosched.c+7−2

 {
 	struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
+	if (!atomic_read(&hctx->elevator_queued))
+		return false;
 	/*
 	 * Avoiding lock: a race on bfqd->busy_queues should cause at
 	 * most a call to dispatch for nothing
 		rq = list_first_entry(list, struct request, queuelist);
 		list_del_init(&rq->queuelist);
 		bfq_insert_request(hctx, rq, at_head);
+		atomic_inc(&hctx->elevator_queued);
 	}
 }
 		bfq_completed_request(bfqq, bfqd);
 		bfq_finish_requeue_request_body(bfqq);
+		atomic_dec(&rq->mq_hctx->elevator_queued);
 		spin_unlock_irqrestore(&bfqd->lock, flags);
 	} else {
 	struct blk_mq_tags *tags = hctx->sched_tags;
 	unsigned int min_shallow;
-	min_shallow = bfq_update_depths(bfqd, &tags->bitmap_tags);
+	min_shallow = bfq_update_depths(bfqd, tags->bitmap_tags);
-	sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, min_shallow);
+	sbitmap_queue_min_shallow_depth(tags->bitmap_tags, min_shallow);
 }
 static int bfq_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int index)

block/blk-cgroup.c+26−6

 					     async_bio_work);
 	struct bio_list bios = BIO_EMPTY_LIST;
 	struct bio *bio;
+	struct blk_plug plug;
+	bool need_plug = false;
 	/* as long as there are pending bios, @blkg can't go away */
 	spin_lock_bh(&blkg->async_bio_lock);
 	bio_list_init(&blkg->async_bios);
 	spin_unlock_bh(&blkg->async_bio_lock);
+	/* start plug only when bio_list contains at least 2 bios */
+	if (bios.head && bios.head->bi_next) {
+		need_plug = true;
+		blk_start_plug(&plug);
+	}
 	while ((bio = bio_list_pop(&bios)))
 		submit_bio(bio);
+	if (need_plug)
+		blk_finish_plug(&plug);
 }
 /**
 static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
 {
 	unsigned long pflags;
+	bool clamp;
 	u64 now = ktime_to_ns(ktime_get());
 	u64 exp;
 	u64 delay_nsec = 0;
 	int tok;
 	while (blkg->parent) {
-		if (atomic_read(&blkg->use_delay)) {
+		int use_delay = atomic_read(&blkg->use_delay);
+		if (use_delay) {
+			u64 this_delay;
 			blkcg_scale_delay(blkg, now);
-			delay_nsec = max_t(u64, delay_nsec,
+			this_delay = atomic64_read(&blkg->delay_nsec);
-					   atomic64_read(&blkg->delay_nsec));
+			if (this_delay > delay_nsec) {
+				delay_nsec = this_delay;
+				clamp = use_delay > 0;
+			}
 		}
 		blkg = blkg->parent;
 	}
 	 * Let's not sleep for all eternity if we've amassed a huge delay.
 	 * Swapping or metadata IO can accumulate 10's of seconds worth of
 	 * delay, and we want userspace to be able to do _something_ so cap the
-	 * delays at 1 second.  If there's 10's of seconds worth of delay then
+	 * delays at 0.25s. If there's 10's of seconds worth of delay then the
-	 * the tasks will be delayed for 1 second for every syscall.
+	 * tasks will be delayed for 0.25 second for every syscall. If
+	 * blkcg_set_delay() was used as indicated by negative use_delay, the
+	 * caller is responsible for regulating the range.
 	 */
-	delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
+	if (clamp)
+		delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
 	if (use_memdelay)
 		psi_memstall_enter(&pflags);

block/blk-core.c+63−176

@@ -116,8 +116,8 @@ void blk_rq_init(struct request_queue q, struct request rq) · 115 unmodified lines
116	rq->__sector = (sector_t) -1;	116	rq->__sector = (sector_t) -1;
117	INIT_HLIST_NODE(&rq->hash);	117	INIT_HLIST_NODE(&rq->hash);
118	RB_CLEAR_NODE(&rq->rb_node);	118	RB_CLEAR_NODE(&rq->rb_node);
119	rq->tag = -1;	119	rq->tag = BLK_MQ_NO_TAG;
120	rq->internal_tag = -1;	120	rq->internal_tag = BLK_MQ_NO_TAG;
121	rq->start_time_ns = ktime_get_ns();	121	rq->start_time_ns = ktime_get_ns();
122	rq->part = NULL;	122	rq->part = NULL;
123	refcount_set(&rq->ref, 1);	123	refcount_set(&rq->ref, 1);
@@ -538,11 +538,10 @@ struct request_queue *blk_alloc_queue(int node_id) · 414 unmodified lines
538	if (!q->stats)	538	if (!q->stats)
539	goto fail_stats;	539	goto fail_stats;
540		540
541	q->backing_dev_info->ra_pages = VM_READAHEAD_PAGES;
542	q->backing_dev_info->io_pages = VM_READAHEAD_PAGES;
543	q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
544	q->node = node_id;	541	q->node = node_id;
545		542
		543	atomic_set(&q->nr_active_requests_shared_sbitmap, 0);
		544
546	timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,	545	timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
547	laptop_mode_timer_fn, 0);	546	laptop_mode_timer_fn, 0);
548	timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);	547	timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
@@ -643,162 +642,6 @@ void blk_put_request(struct request *req) · 94 unmodified lines
643	}	642	}
644	EXPORT_SYMBOL(blk_put_request);	643	EXPORT_SYMBOL(blk_put_request);
645		644
646	static void blk_account_io_merge_bio(struct request *req)
647	{
648	if (!blk_do_io_stat(req))
649	return;
650
651	part_stat_lock();
652	part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
653	part_stat_unlock();
654	}
655
656	bool bio_attempt_back_merge(struct request req, struct bio bio,
657	unsigned int nr_segs)
658	{
659	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
660
661	if (!ll_back_merge_fn(req, bio, nr_segs))
662	return false;
663
664	trace_block_bio_backmerge(req->q, req, bio);
665	rq_qos_merge(req->q, req, bio);
666
667	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
668	blk_rq_set_mixed_merge(req);
669
670	req->biotail->bi_next = bio;
671	req->biotail = bio;
672	req->__data_len += bio->bi_iter.bi_size;
673
674	bio_crypt_free_ctx(bio);
675
676	blk_account_io_merge_bio(req);
677	return true;
678	}
679
680	bool bio_attempt_front_merge(struct request req, struct bio bio,
681	unsigned int nr_segs)
682	{
683	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
684
685	if (!ll_front_merge_fn(req, bio, nr_segs))
686	return false;
687
688	trace_block_bio_frontmerge(req->q, req, bio);
689	rq_qos_merge(req->q, req, bio);
690
691	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
692	blk_rq_set_mixed_merge(req);
693
694	bio->bi_next = req->bio;
695	req->bio = bio;
696
697	req->__sector = bio->bi_iter.bi_sector;
698	req->__data_len += bio->bi_iter.bi_size;
699
700	bio_crypt_do_front_merge(req, bio);
701
702	blk_account_io_merge_bio(req);
703	return true;
704	}
705
706	bool bio_attempt_discard_merge(struct request_queue q, struct request req,
707	struct bio *bio)
708	{
709	unsigned short segments = blk_rq_nr_discard_segments(req);
710
711	if (segments >= queue_max_discard_segments(q))
712	goto no_merge;
713	if (blk_rq_sectors(req) + bio_sectors(bio) >
714	blk_rq_get_max_sectors(req, blk_rq_pos(req)))
715	goto no_merge;
716
717	rq_qos_merge(q, req, bio);
718
719	req->biotail->bi_next = bio;
720	req->biotail = bio;
721	req->__data_len += bio->bi_iter.bi_size;
722	req->nr_phys_segments = segments + 1;
723
724	blk_account_io_merge_bio(req);
725	return true;
726	no_merge:
727	req_set_nomerge(q, req);
728	return false;
729	}
730
731	/**
732	* blk_attempt_plug_merge - try to merge with %current's plugged list
733	* @q: request_queue new bio is being queued at
734	* @bio: new bio being queued
735	* @nr_segs: number of segments in @bio
736	* @same_queue_rq: pointer to &struct request that gets filled in when
737	* another request associated with @q is found on the plug list
738	* (optional, may be %NULL)
739	*
740	* Determine whether @bio being queued on @q can be merged with a request
741	* on %current's plugged list. Returns %true if merge was successful,
742	* otherwise %false.
743	*
744	* Plugging coalesces IOs from the same issuer for the same purpose without
745	* going through @q->queue_lock. As such it's more of an issuing mechanism
746	* than scheduling, and the request, while may have elvpriv data, is not
747	* added on the elevator at this point. In addition, we don't have
748	* reliable access to the elevator outside queue lock. Only check basic
749	* merging parameters without querying the elevator.
750	*
751	* Caller must ensure !blk_queue_nomerges(q) beforehand.
752	*/
753	bool blk_attempt_plug_merge(struct request_queue q, struct bio bio,
754	unsigned int nr_segs, struct request **same_queue_rq)
755	{
756	struct blk_plug *plug;
757	struct request *rq;
758	struct list_head *plug_list;
759
760	plug = blk_mq_plug(q, bio);
761	if (!plug)
762	return false;
763
764	plug_list = &plug->mq_list;
765
766	list_for_each_entry_reverse(rq, plug_list, queuelist) {
767	bool merged = false;
768
769	if (rq->q == q && same_queue_rq) {
770	/*
771	* Only blk-mq multiple hardware queues case checks the
772	* rq in the same queue, there should be only one such
773	* rq in a queue
774	**/
775	*same_queue_rq = rq;
776	}
777
778	if (rq->q != q \|\| !blk_rq_merge_ok(rq, bio))
779	continue;
780
781	switch (blk_try_merge(rq, bio)) {
782	case ELEVATOR_BACK_MERGE:
783	merged = bio_attempt_back_merge(rq, bio, nr_segs);
784	break;
785	case ELEVATOR_FRONT_MERGE:
786	merged = bio_attempt_front_merge(rq, bio, nr_segs);
787	break;
788	case ELEVATOR_DISCARD_MERGE:
789	merged = bio_attempt_discard_merge(q, rq, bio);
790	break;
791	default:
792	break;
793	}
794
795	if (merged)
796	return true;
797	}
798
799	return false;
800	}
801
802	static void handle_bad_sector(struct bio *bio, sector_t maxsector)	645	static void handle_bad_sector(struct bio *bio, sector_t maxsector)
803	{	646	{
804	char b[BDEVNAME_SIZE];	647	char b[BDEVNAME_SIZE];
@@ -1301,14 +1144,28 @@ EXPORT_SYMBOL(submit_bio); · 496 unmodified lines
1301	* limits when retrying requests on other queues. Those requests need	1144	* limits when retrying requests on other queues. Those requests need
1302	* to be checked against the new queue limits again during dispatch.	1145	* to be checked against the new queue limits again during dispatch.
1303	*/	1146	*/
1304	static int blk_cloned_rq_check_limits(struct request_queue *q,	1147	static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q,
1305	struct request *rq)	1148	struct request *rq)
1306	{	1149	{
1307	if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {	1150	unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
		1151
		1152	if (blk_rq_sectors(rq) > max_sectors) {
		1153	/*
		1154	* SCSI device does not have a good way to return if
		1155	* Write Same/Zero is actually supported. If a device rejects
		1156	* a non-read/write command (discard, write same,etc.) the
		1157	* low-level device driver will set the relevant queue limit to
		1158	* 0 to prevent blk-lib from issuing more of the offending
		1159	* operations. Commands queued prior to the queue limit being
		1160	* reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
		1161	* errors being propagated to upper layers.
		1162	*/
		1163	if (max_sectors == 0)
		1164	return BLK_STS_NOTSUPP;
		1165
1308	printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",	1166	printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
1309	__func__, blk_rq_sectors(rq),	1167	__func__, blk_rq_sectors(rq), max_sectors);
1310	blk_queue_get_max_sectors(q, req_op(rq)));	1168	return BLK_STS_IOERR;
1311	return -EIO;
1312	}	1169	}
1313		1170
1314	/*	1171	/*
@@ -1321,10 +1178,10 @@ static int blk_cloned_rq_check_limits(struct request_queue *q, · 6 unmodified lines
1321	if (rq->nr_phys_segments > queue_max_segments(q)) {	1178	if (rq->nr_phys_segments > queue_max_segments(q)) {
1322	printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",	1179	printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",
1323	__func__, rq->nr_phys_segments, queue_max_segments(q));	1180	__func__, rq->nr_phys_segments, queue_max_segments(q));
1324	return -EIO;	1181	return BLK_STS_IOERR;
1325	}	1182	}
1326		1183
1327	return 0;	1184	return BLK_STS_OK;
1328	}	1185	}
1329		1186
1330	/**	1187	/**
@@ -1334,8 +1191,11 @@ static int blk_cloned_rq_check_limits(struct request_queue *q, · 3 unmodified lines
1334	*/	1191	*/
1335	blk_status_t blk_insert_cloned_request(struct request_queue q, struct request rq)	1192	blk_status_t blk_insert_cloned_request(struct request_queue q, struct request rq)
1336	{	1193	{
1337	if (blk_cloned_rq_check_limits(q, rq))	1194	blk_status_t ret;
1338	return BLK_STS_IOERR;	1195
		1196	ret = blk_cloned_rq_check_limits(q, rq);
		1197	if (ret != BLK_STS_OK)
		1198	return ret;
1339		1199
1340	if (rq->rq_disk &&	1200	if (rq->rq_disk &&
1341	should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))	1201	should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
@@ -1461,10 +1321,9 @@ void blk_account_io_start(struct request *rq) · 119 unmodified lines
1461	part_stat_unlock();	1321	part_stat_unlock();
1462	}	1322	}
1463		1323
1464	unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,	1324	static unsigned long __part_start_io_acct(struct hd_struct *part,
1465	unsigned int op)	1325	unsigned int sectors, unsigned int op)
1466	{	1326	{
1467	struct hd_struct *part = &disk->part0;
1468	const int sgrp = op_stat_group(op);	1327	const int sgrp = op_stat_group(op);
1469	unsigned long now = READ_ONCE(jiffies);	1328	unsigned long now = READ_ONCE(jiffies);
1470		1329
@@ -1477,12 +1336,26 @@ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, · 6 unmodified lines
1477		1336
1478	return now;	1337	return now;
1479	}	1338	}
		1339
		1340	unsigned long part_start_io_acct(struct gendisk disk, struct hd_struct *part,
		1341	struct bio *bio)
		1342	{
		1343	*part = disk_map_sector_rcu(disk, bio->bi_iter.bi_sector);
		1344
		1345	return __part_start_io_acct(*part, bio_sectors(bio), bio_op(bio));
		1346	}
		1347	EXPORT_SYMBOL_GPL(part_start_io_acct);
		1348
		1349	unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
		1350	unsigned int op)
		1351	{
		1352	return __part_start_io_acct(&disk->part0, sectors, op);
		1353	}
1480	EXPORT_SYMBOL(disk_start_io_acct);	1354	EXPORT_SYMBOL(disk_start_io_acct);
1481		1355
1482	void disk_end_io_acct(struct gendisk *disk, unsigned int op,	1356	static void __part_end_io_acct(struct hd_struct *part, unsigned int op,
1483	unsigned long start_time)	1357	unsigned long start_time)
1484	{	1358	{
1485	struct hd_struct *part = &disk->part0;
1486	const int sgrp = op_stat_group(op);	1359	const int sgrp = op_stat_group(op);
1487	unsigned long now = READ_ONCE(jiffies);	1360	unsigned long now = READ_ONCE(jiffies);
1488	unsigned long duration = now - start_time;	1361	unsigned long duration = now - start_time;
@@ -1493,6 +1366,20 @@ void disk_end_io_acct(struct gendisk *disk, unsigned int op, · 4 unmodified lines
1493	part_stat_local_dec(part, in_flight[op_is_write(op)]);	1366	part_stat_local_dec(part, in_flight[op_is_write(op)]);
1494	part_stat_unlock();	1367	part_stat_unlock();
1495	}	1368	}
		1369
		1370	void part_end_io_acct(struct hd_struct part, struct bio bio,
		1371	unsigned long start_time)
		1372	{
		1373	__part_end_io_acct(part, bio_op(bio), start_time);
		1374	hd_struct_put(part);
		1375	}
		1376	EXPORT_SYMBOL_GPL(part_end_io_acct);
		1377
		1378	void disk_end_io_acct(struct gendisk *disk, unsigned int op,
		1379	unsigned long start_time)
		1380	{
		1381	__part_end_io_acct(&disk->part0, op, start_time);
		1382	}
1496	EXPORT_SYMBOL(disk_end_io_acct);	1383	EXPORT_SYMBOL(disk_end_io_acct);
1497		1384
1498	/*	1385	/*

block/blk-integrity.c+2−2

@@ -408,7 +408,7 @@ void blk_integrity_register(struct gendisk disk, struct blk_integrity template · 407 unmodified lines
408	bi->tuple_size = template->tuple_size;	408	bi->tuple_size = template->tuple_size;
409	bi->tag_size = template->tag_size;	409	bi->tag_size = template->tag_size;
410		410
411	disk->queue->backing_dev_info->capabilities \|= BDI_CAP_STABLE_WRITES;	411	blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, disk->queue);
412		412
413	#ifdef CONFIG_BLK_INLINE_ENCRYPTION	413	#ifdef CONFIG_BLK_INLINE_ENCRYPTION
414	if (disk->queue->ksm) {	414	if (disk->queue->ksm) {
@@ -428,7 +428,7 @@ EXPORT_SYMBOL(blk_integrity_register); · 13 unmodified lines
428	*/	428	*/
429	void blk_integrity_unregister(struct gendisk *disk)	429	void blk_integrity_unregister(struct gendisk *disk)
430	{	430	{
431	disk->queue->backing_dev_info->capabilities &= ~BDI_CAP_STABLE_WRITES;	431	blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, disk->queue);
432	memset(&disk->queue->integrity, 0, sizeof(struct blk_integrity));	432	memset(&disk->queue->integrity, 0, sizeof(struct blk_integrity));
433	}	433	}
434	EXPORT_SYMBOL(blk_integrity_unregister);	434	EXPORT_SYMBOL(blk_integrity_unregister);

block/blk-iocost.c+1180−367

@@ -68,7 +68,7 @@ · 67 unmodified lines
68	* gets 300/(100+300) or 75% share, and A0 and A1 equally splits the rest,	68	* gets 300/(100+300) or 75% share, and A0 and A1 equally splits the rest,
69	* 12.5% each. The distribution mechanism only cares about these flattened	69	* 12.5% each. The distribution mechanism only cares about these flattened
70	* shares. They're called hweights (hierarchical weights) and always add	70	* shares. They're called hweights (hierarchical weights) and always add
71	* upto 1 (HWEIGHT_WHOLE).	71	* upto 1 (WEIGHT_ONE).
72	*	72	*
73	* A given cgroup's vtime runs slower in inverse proportion to its hweight.	73	* A given cgroup's vtime runs slower in inverse proportion to its hweight.
74	* For example, with 12.5% weight, A0's time runs 8 times slower (100/12.5)	74	* For example, with 12.5% weight, A0's time runs 8 times slower (100/12.5)
@@ -179,6 +179,8 @@ · 104 unmodified lines
179	#include <linux/parser.h>	179	#include <linux/parser.h>
180	#include <linux/sched/signal.h>	180	#include <linux/sched/signal.h>
181	#include <linux/blk-cgroup.h>	181	#include <linux/blk-cgroup.h>
		182	#include <asm/local.h>
		183	#include <asm/local64.h>
182	#include "blk-rq-qos.h"	184	#include "blk-rq-qos.h"
183	#include "blk-stat.h"	185	#include "blk-stat.h"
184	#include "blk-wbt.h"	186	#include "blk-wbt.h"
@@ -215,36 +217,21 @@ enum { · 30 unmodified lines
215	MAX_PERIOD = USEC_PER_SEC,	217	MAX_PERIOD = USEC_PER_SEC,
216		218
217	/*	219	/*
218	* A cgroup's vtime can run 50% behind the device vtime, which	220	* iocg->vtime is targeted at 50% behind the device vtime, which
219	* serves as its IO credit buffer. Surplus weight adjustment is	221	* serves as its IO credit buffer. Surplus weight adjustment is
220	* immediately canceled if the vtime margin runs below 10%.	222	* immediately canceled if the vtime margin runs below 10%.
221	*/	223	*/
222	MARGIN_PCT = 50,	224	MARGIN_MIN_PCT = 10,
223	INUSE_MARGIN_PCT = 10,	225	MARGIN_LOW_PCT = 20,
		226	MARGIN_TARGET_PCT = 50,
224		227
225	/* Have some play in waitq timer operations */	228	INUSE_ADJ_STEP_PCT = 25,
226	WAITQ_TIMER_MARGIN_PCT = 5,
227		229
228	/*	230	/* Have some play in timer operations */
229	* vtime can wrap well within a reasonable uptime when vrate is	231	TIMER_SLACK_PCT = 1,
230	* consistently raised. Don't trust recorded cgroup vtime if the
231	* period counter indicates that it's older than 5mins.
232	*/
233	VTIME_VALID_DUR = 300 * USEC_PER_SEC,
234
235	/*
236	* Remember the past three non-zero usages and use the max for
237	* surplus calculation. Three slots guarantee that we remember one
238	* full period usage from the last active stretch even after
239	* partial deactivation and re-activation periods. Don't start
240	* giving away weight before collecting two data points to prevent
241	* hweight adjustments based on one partial activation period.
242	*/
243	NR_USAGE_SLOTS = 3,
244	MIN_VALID_USAGES = 2,
245		232
246	/* 1/64k is granular enough and can easily be handled w/ u32 */	233	/* 1/64k is granular enough and can easily be handled w/ u32 */
247	HWEIGHT_WHOLE = 1 << 16,	234	WEIGHT_ONE = 1 << 16,
248		235
249	/*	236	/*
250	* As vtime is used to calculate the cost of each IO, it needs to	237	* As vtime is used to calculate the cost of each IO, it needs to
@@ -275,16 +262,40 @@ enum { · 24 unmodified lines
275	/* unbusy hysterisis */	262	/* unbusy hysterisis */
276	UNBUSY_THR_PCT = 75,	263	UNBUSY_THR_PCT = 75,
277		264
278	/* don't let cmds which take a very long time pin lagging for too long */	265	/*
279	MAX_LAGGING_PERIODS = 10,	266	* The effect of delay is indirect and non-linear and a huge amount of
		267	* future debt can accumulate abruptly while unthrottled. Linearly scale
		268	* up delay as debt is going up and then let it decay exponentially.
		269	* This gives us quick ramp ups while delay is accumulating and long
		270	* tails which can help reducing the frequency of debt explosions on
		271	* unthrottle. The parameters are experimentally determined.
		272	*
		273	* The delay mechanism provides adequate protection and behavior in many
		274	* cases. However, this is far from ideal and falls shorts on both
		275	* fronts. The debtors are often throttled too harshly costing a
		276	* significant level of fairness and possibly total work while the
		277	* protection against their impacts on the system can be choppy and
		278	* unreliable.
		279	*
		280	* The shortcoming primarily stems from the fact that, unlike for page
		281	* cache, the kernel doesn't have well-defined back-pressure propagation
		282	* mechanism and policies for anonymous memory. Fully addressing this
		283	* issue will likely require substantial improvements in the area.
		284	*/
		285	MIN_DELAY_THR_PCT = 500,
		286	MAX_DELAY_THR_PCT = 25000,
		287	MIN_DELAY = 250,
		288	MAX_DELAY = 250 * USEC_PER_MSEC,
280		289
281	/*	290	/*
282	* If usage% * 1.25 + 2% is lower than hweight% by more than 3%,	291	* Halve debts if total usage keeps staying under 25% w/o any shortages
283	* donate the surplus.	292	* for over 100ms.
284	*/	293	*/
285	SURPLUS_SCALE_PCT = 125, /* * 125% */	294	DEBT_BUSY_USAGE_PCT = 25,
286	SURPLUS_SCALE_ABS = HWEIGHT_WHOLE / 50, /* + 2% */	295	DEBT_REDUCTION_IDLE_DUR = 100 * USEC_PER_MSEC,
287	SURPLUS_MIN_ADJ_DELTA = HWEIGHT_WHOLE / 33, /* 3% */	296
		297	/* don't let cmds which take a very long time pin lagging for too long */
		298	MAX_LAGGING_PERIODS = 10,
288		299
289	/* switch iff the conditions are met for longer than this */	300	/* switch iff the conditions are met for longer than this */
290	AUTOP_CYCLE_NSEC = 10LLU * NSEC_PER_SEC,	301	AUTOP_CYCLE_NSEC = 10LLU * NSEC_PER_SEC,
@@ -372,9 +383,15 @@ struct ioc_params { · 81 unmodified lines
372	u32 too_slow_vrate_pct;	383	u32 too_slow_vrate_pct;
373	};	384	};
374		385
		386	struct ioc_margins {
		387	s64 min;
		388	s64 low;
		389	s64 target;
		390	};
		391
375	struct ioc_missed {	392	struct ioc_missed {
376	u32 nr_met;	393	local_t nr_met;
377	u32 nr_missed;	394	local_t nr_missed;
378	u32 last_met;	395	u32 last_met;
379	u32 last_missed;	396	u32 last_missed;
380	};	397	};
@@ -382,7 +399,7 @@ struct ioc_missed { · 1 unmodified line
382	struct ioc_pcpu_stat {	399	struct ioc_pcpu_stat {
383	struct ioc_missed missed[2];	400	struct ioc_missed missed[2];
384		401
385	u64 rq_wait_ns;	402	local64_t rq_wait_ns;
386	u64 last_rq_wait_ns;	403	u64 last_rq_wait_ns;
387	};	404	};
388		405
@@ -393,8 +410,9 @@ struct ioc { · 4 unmodified lines
393	bool enabled;	410	bool enabled;
394		411
395	struct ioc_params params;	412	struct ioc_params params;
		413	struct ioc_margins margins;
396	u32 period_us;	414	u32 period_us;
397	u32 margin_us;	415	u32 timer_slack_ns;
398	u64 vrate_min;	416	u64 vrate_min;
399	u64 vrate_max;	417	u64 vrate_max;
400		418
@@ -405,18 +423,22 @@ struct ioc { · 4 unmodified lines
405		423
406	enum ioc_running running;	424	enum ioc_running running;
407	atomic64_t vtime_rate;	425	atomic64_t vtime_rate;
		426	u64 vtime_base_rate;
		427	s64 vtime_err;
408		428
409	seqcount_spinlock_t period_seqcount;	429	seqcount_spinlock_t period_seqcount;
410	u32 period_at; /* wallclock starttime */	430	u64 period_at; /* wallclock starttime */
411	u64 period_at_vtime; /* vtime starttime */	431	u64 period_at_vtime; /* vtime starttime */
412		432
413	atomic64_t cur_period; /* inc'd each period */	433	atomic64_t cur_period; /* inc'd each period */
414	int busy_level; /* saturation history */	434	int busy_level; /* saturation history */
415		435
416	u64 inuse_margin_vtime;
417	bool weights_updated;	436	bool weights_updated;
418	atomic_t hweight_gen; /* for lazy hweights */	437	atomic_t hweight_gen; /* for lazy hweights */
419		438
		439	/* the last time debt cancel condition wasn't met */
		440	u64 debt_busy_at;
		441
420	u64 autop_too_fast_at;	442	u64 autop_too_fast_at;
421	u64 autop_too_slow_at;	443	u64 autop_too_slow_at;
422	int autop_idx;	444	int autop_idx;
@@ -424,6 +446,17 @@ struct ioc { · 1 unmodified line
424	bool user_cost_model:1;	446	bool user_cost_model:1;
425	};	447	};
426		448
		449	struct iocg_pcpu_stat {
		450	local64_t abs_vusage;
		451	};
		452
		453	struct iocg_stat {
		454	u64 usage_us;
		455	u64 wait_us;
		456	u64 indebt_us;
		457	u64 indelay_us;
		458	};
		459
427	/* per device-cgroup pair */	460	/* per device-cgroup pair */
428	struct ioc_gq {	461	struct ioc_gq {
429	struct blkg_policy_data pd;	462	struct blkg_policy_data pd;
@@ -443,12 +476,17 @@ struct ioc_gq { · 13 unmodified lines
443	*	476	*
444	* `last_inuse` remembers `inuse` while an iocg is idle to persist	477	* `last_inuse` remembers `inuse` while an iocg is idle to persist
445	* surplus adjustments.	478	* surplus adjustments.
		479	*
		480	* `inuse` may be adjusted dynamically during period. `saved_*` are used
		481	* to determine and track adjustments.
446	*/	482	*/
447	u32 cfg_weight;	483	u32 cfg_weight;
448	u32 weight;	484	u32 weight;
449	u32 active;	485	u32 active;
450	u32 inuse;	486	u32 inuse;
		487
451	u32 last_inuse;	488	u32 last_inuse;
		489	s64 saved_margin;
452		490
453	sector_t cursor; /* to detect randio */	491	sector_t cursor; /* to detect randio */
454		492
@@ -461,14 +499,14 @@ struct ioc_gq { · 6 unmodified lines
461	* `vtime_done` is the same but progressed on completion rather	499	* `vtime_done` is the same but progressed on completion rather
462	* than issue. The delta behind `vtime` represents the cost of	500	* than issue. The delta behind `vtime` represents the cost of
463	* currently in-flight IOs.	501	* currently in-flight IOs.
464	*
465	* `last_vtime` is used to remember `vtime` at the end of the last
466	* period to calculate utilization.
467	*/	502	*/
468	atomic64_t vtime;	503	atomic64_t vtime;
469	atomic64_t done_vtime;	504	atomic64_t done_vtime;
470	u64 abs_vdebt;	505	u64 abs_vdebt;
471	u64 last_vtime;	506
		507	/* current delay in effect and when it started */
		508	u64 delay;
		509	u64 delay_at;
472		510
473	/*	511	/*
474	* The period this iocg was last active in. Used for deactivation	512	* The period this iocg was last active in. Used for deactivation
@@ -477,21 +515,35 @@ struct ioc_gq { · 2 unmodified lines
477	atomic64_t active_period;	515	atomic64_t active_period;
478	struct list_head active_list;	516	struct list_head active_list;
479		517
480	/* see __propagate_active_weight() and current_hweight() for details */	518	/* see __propagate_weights() and current_hweight() for details */
481	u64 child_active_sum;	519	u64 child_active_sum;
482	u64 child_inuse_sum;	520	u64 child_inuse_sum;
		521	u64 child_adjusted_sum;
483	int hweight_gen;	522	int hweight_gen;
484	u32 hweight_active;	523	u32 hweight_active;
485	u32 hweight_inuse;	524	u32 hweight_inuse;
486	bool has_surplus;	525	u32 hweight_donating;
		526	u32 hweight_after_donation;
		527
		528	struct list_head walk_list;
		529	struct list_head surplus_list;
487		530
488	struct wait_queue_head waitq;	531	struct wait_queue_head waitq;
489	struct hrtimer waitq_timer;	532	struct hrtimer waitq_timer;
490	struct hrtimer delay_timer;
491		533
492	/* usage is recorded as fractions of HWEIGHT_WHOLE */	534	/* timestamp at the latest activation */
493	int usage_idx;	535	u64 activated_at;
494	u32 usages[NR_USAGE_SLOTS];	536
		537	/* statistics */
		538	struct iocg_pcpu_stat __percpu *pcpu_stat;
		539	struct iocg_stat local_stat;
		540	struct iocg_stat desc_stat;
		541	struct iocg_stat last_stat;
		542	u64 last_stat_abs_vusage;
		543	u64 usage_delta_us;
		544	u64 wait_since;
		545	u64 indebt_since;
		546	u64 indelay_since;
495		547
496	/* this iocg's depth in the hierarchy and ancestors including self */	548	/* this iocg's depth in the hierarchy and ancestors including self */
497	int level;	549	int level;
@@ -506,7 +558,7 @@ struct ioc_cgrp { · 8 unmodified lines
506		558
507	struct ioc_now {	559	struct ioc_now {
508	u64 now_ns;	560	u64 now_ns;
509	u32 now;	561	u64 now;
510	u64 vnow;	562	u64 vnow;
511	u64 vrate;	563	u64 vrate;
512	};	564	};
@@ -656,7 +708,7 @@ static struct ioc_cgrp blkcg_to_iocc(struct blkcg blkcg) · 143 unmodified lines
656	*/	708	*/
657	static u64 abs_cost_to_cost(u64 abs_cost, u32 hw_inuse)	709	static u64 abs_cost_to_cost(u64 abs_cost, u32 hw_inuse)
658	{	710	{
659	return DIV64_U64_ROUND_UP(abs_cost * HWEIGHT_WHOLE, hw_inuse);	711	return DIV64_U64_ROUND_UP(abs_cost * WEIGHT_ONE, hw_inuse);
660	}	712	}
661		713
662	/*	714	/*
@@ -664,18 +716,56 @@ static u64 abs_cost_to_cost(u64 abs_cost, u32 hw_inuse) · 1 unmodified line
664	*/	716	*/
665	static u64 cost_to_abs_cost(u64 cost, u32 hw_inuse)	717	static u64 cost_to_abs_cost(u64 cost, u32 hw_inuse)
666	{	718	{
667	return DIV64_U64_ROUND_UP(cost * hw_inuse, HWEIGHT_WHOLE);	719	return DIV64_U64_ROUND_UP(cost * hw_inuse, WEIGHT_ONE);
668	}	720	}
669		721
670	static void iocg_commit_bio(struct ioc_gq iocg, struct bio bio, u64 cost)	722	static void iocg_commit_bio(struct ioc_gq iocg, struct bio bio,
		723	u64 abs_cost, u64 cost)
671	{	724	{
		725	struct iocg_pcpu_stat *gcs;
		726
672	bio->bi_iocost_cost = cost;	727	bio->bi_iocost_cost = cost;
673	atomic64_add(cost, &iocg->vtime);	728	atomic64_add(cost, &iocg->vtime);
		729
		730	gcs = get_cpu_ptr(iocg->pcpu_stat);
		731	local64_add(abs_cost, &gcs->abs_vusage);
		732	put_cpu_ptr(gcs);
		733	}
		734
		735	static void iocg_lock(struct ioc_gq iocg, bool lock_ioc, unsigned long flags)
		736	{
		737	if (lock_ioc) {
		738	spin_lock_irqsave(&iocg->ioc->lock, *flags);
		739	spin_lock(&iocg->waitq.lock);
		740	} else {
		741	spin_lock_irqsave(&iocg->waitq.lock, *flags);
		742	}
		743	}
		744
		745	static void iocg_unlock(struct ioc_gq iocg, bool unlock_ioc, unsigned long flags)
		746	{
		747	if (unlock_ioc) {
		748	spin_unlock(&iocg->waitq.lock);
		749	spin_unlock_irqrestore(&iocg->ioc->lock, *flags);
		750	} else {
		751	spin_unlock_irqrestore(&iocg->waitq.lock, *flags);
		752	}
674	}	753	}
675		754
676	#define CREATE_TRACE_POINTS	755	#define CREATE_TRACE_POINTS
677	#include <trace/events/iocost.h>	756	#include <trace/events/iocost.h>
678		757
		758	static void ioc_refresh_margins(struct ioc *ioc)
		759	{
		760	struct ioc_margins *margins = &ioc->margins;
		761	u32 period_us = ioc->period_us;
		762	u64 vrate = ioc->vtime_base_rate;
		763
		764	margins->min = (period_us * MARGIN_MIN_PCT / 100) * vrate;
		765	margins->low = (period_us * MARGIN_LOW_PCT / 100) * vrate;
		766	margins->target = (period_us * MARGIN_TARGET_PCT / 100) * vrate;
		767	}
		768
679	/* latency Qos params changed, update period_us and all the dependent params */	769	/* latency Qos params changed, update period_us and all the dependent params */
680	static void ioc_refresh_period_us(struct ioc *ioc)	770	static void ioc_refresh_period_us(struct ioc *ioc)
681	{	771	{
@@ -709,9 +799,10 @@ static void ioc_refresh_period_us(struct ioc *ioc) · 27 unmodified lines
709		799
710	/* calculate dependent params */	800	/* calculate dependent params */
711	ioc->period_us = period_us;	801	ioc->period_us = period_us;
712	ioc->margin_us = period_us * MARGIN_PCT / 100;	802	ioc->timer_slack_ns = div64_u64(
713	ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(	803	(u64)period_us * NSEC_PER_USEC * TIMER_SLACK_PCT,
714	period_us * VTIME_PER_USEC * INUSE_MARGIN_PCT, 100);	804	100);
		805	ioc_refresh_margins(ioc);
715	}	806	}
716		807
717	static int ioc_autop_idx(struct ioc *ioc)	808	static int ioc_autop_idx(struct ioc *ioc)
@@ -738,8 +829,7 @@ static int ioc_autop_idx(struct ioc *ioc) · 20 unmodified lines
738	return idx;	829	return idx;
739		830
740	/* step up/down based on the vrate */	831	/* step up/down based on the vrate */
741	vrate_pct = div64_u64(atomic64_read(&ioc->vtime_rate) * 100,	832	vrate_pct = div64_u64(ioc->vtime_base_rate * 100, VTIME_PER_USEC);
742	VTIME_PER_USEC);
743	now_ns = ktime_get_ns();	833	now_ns = ktime_get_ns();
744		834
745	if (p->too_fast_vrate_pct && p->too_fast_vrate_pct <= vrate_pct) {	835	if (p->too_fast_vrate_pct && p->too_fast_vrate_pct <= vrate_pct) {
@@ -847,6 +937,43 @@ static bool ioc_refresh_params(struct ioc *ioc, bool force) · 101 unmodified lines
847	return true;	937	return true;
848	}	938	}
849		939
		940	/*
		941	* When an iocg accumulates too much vtime or gets deactivated, we throw away
		942	* some vtime, which lowers the overall device utilization. As the exact amount
		943	* which is being thrown away is known, we can compensate by accelerating the
		944	* vrate accordingly so that the extra vtime generated in the current period
		945	* matches what got lost.
		946	*/
		947	static void ioc_refresh_vrate(struct ioc ioc, struct ioc_now now)
		948	{
		949	s64 pleft = ioc->period_at + ioc->period_us - now->now;
		950	s64 vperiod = ioc->period_us * ioc->vtime_base_rate;
		951	s64 vcomp, vcomp_min, vcomp_max;
		952
		953	lockdep_assert_held(&ioc->lock);
		954
		955	/* we need some time left in this period */
		956	if (pleft <= 0)
		957	goto done;
		958
		959	/*
		960	* Calculate how much vrate should be adjusted to offset the error.
		961	* Limit the amount of adjustment and deduct the adjusted amount from
		962	* the error.
		963	*/
		964	vcomp = -div64_s64(ioc->vtime_err, pleft);
		965	vcomp_min = -(ioc->vtime_base_rate >> 1);
		966	vcomp_max = ioc->vtime_base_rate;
		967	vcomp = clamp(vcomp, vcomp_min, vcomp_max);
		968
		969	ioc->vtime_err += vcomp * pleft;
		970
		971	atomic64_set(&ioc->vtime_rate, ioc->vtime_base_rate + vcomp);
		972	done:
		973	/* bound how much error can accumulate */
		974	ioc->vtime_err = clamp(ioc->vtime_err, -vperiod, vperiod);
		975	}
		976
850	/* take a snapshot of the current [v]time and vrate */	977	/* take a snapshot of the current [v]time and vrate */
851	static void ioc_now(struct ioc ioc, struct ioc_now now)	978	static void ioc_now(struct ioc ioc, struct ioc_now now)
852	{	979	{
@@ -886,16 +1013,25 @@ static void ioc_start_period(struct ioc ioc, struct ioc_now now) · 33 unmodified lines
886		1013
887	/*	1014	/*
888	* Update @iocg's `active` and `inuse` to @active and @inuse, update level	1015	* Update @iocg's `active` and `inuse` to @active and @inuse, update level
889	* weight sums and propagate upwards accordingly.	1016	* weight sums and propagate upwards accordingly. If @save, the current margin
		1017	* is saved to be used as reference for later inuse in-period adjustments.
890	*/	1018	*/
891	static void __propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse)	1019	static void __propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
		1020	bool save, struct ioc_now *now)
892	{	1021	{
893	struct ioc *ioc = iocg->ioc;	1022	struct ioc *ioc = iocg->ioc;
894	int lvl;	1023	int lvl;
895		1024
896	lockdep_assert_held(&ioc->lock);	1025	lockdep_assert_held(&ioc->lock);
897		1026
898	inuse = min(active, inuse);	1027	inuse = clamp_t(u32, inuse, 1, active);
		1028
		1029	iocg->last_inuse = iocg->inuse;
		1030	if (save)
		1031	iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);
		1032
		1033	if (active == iocg->active && inuse == iocg->inuse)
		1034	return;
899		1035
900	for (lvl = iocg->level - 1; lvl >= 0; lvl--) {	1036	for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
901	struct ioc_gq *parent = iocg->ancestors[lvl];	1037	struct ioc_gq *parent = iocg->ancestors[lvl];
@@ -933,7 +1069,7 @@ static void __propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse · 31 unmodified lines
933	ioc->weights_updated = true;	1069	ioc->weights_updated = true;
934	}	1070	}
935		1071
936	static void commit_active_weights(struct ioc *ioc)	1072	static void commit_weights(struct ioc *ioc)
937	{	1073	{
938	lockdep_assert_held(&ioc->lock);	1074	lockdep_assert_held(&ioc->lock);
939		1075
@@ -945,10 +1081,11 @@ static void commit_active_weights(struct ioc *ioc) · 5 unmodified lines
945	}	1081	}
946	}	1082	}
947		1083
948	static void propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse)	1084	static void propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
		1085	bool save, struct ioc_now *now)
949	{	1086	{
950	__propagate_active_weight(iocg, active, inuse);	1087	__propagate_weights(iocg, active, inuse, save, now);
951	commit_active_weights(iocg->ioc);	1088	commit_weights(iocg->ioc);
952	}	1089	}
953		1090
954	static void current_hweight(struct ioc_gq iocg, u32 hw_activep, u32 *hw_inusep)	1091	static void current_hweight(struct ioc_gq iocg, u32 hw_activep, u32 *hw_inusep)
@@ -964,9 +1101,9 @@ static void current_hweight(struct ioc_gq iocg, u32 hw_activep, u32 *hw_inusep · 9 unmodified lines
964	goto out;	1101	goto out;
965		1102
966	/*	1103	/*
967	* Paired with wmb in commit_active_weights(). If we saw the	1104	* Paired with wmb in commit_weights(). If we saw the updated
968	* updated hweight_gen, all the weight updates from	1105	* hweight_gen, all the weight updates from __propagate_weights() are
969	* __propagate_active_weight() are visible too.	1106	* visible too.
970	*	1107	*
971	* We can race with weight updates during calculation and get it	1108	* We can race with weight updates during calculation and get it
972	* wrong. However, hweight_gen would have changed and a future	1109	* wrong. However, hweight_gen would have changed and a future
@@ -975,12 +1112,12 @@ static void current_hweight(struct ioc_gq iocg, u32 hw_activep, u32 *hw_inusep · 2 unmodified lines
975	*/	1112	*/
976	smp_rmb();	1113	smp_rmb();
977		1114
978	hwa = hwi = HWEIGHT_WHOLE;	1115	hwa = hwi = WEIGHT_ONE;
979	for (lvl = 0; lvl <= iocg->level - 1; lvl++) {	1116	for (lvl = 0; lvl <= iocg->level - 1; lvl++) {
980	struct ioc_gq *parent = iocg->ancestors[lvl];	1117	struct ioc_gq *parent = iocg->ancestors[lvl];
981	struct ioc_gq *child = iocg->ancestors[lvl + 1];	1118	struct ioc_gq *child = iocg->ancestors[lvl + 1];
982	u32 active_sum = READ_ONCE(parent->child_active_sum);	1119	u64 active_sum = READ_ONCE(parent->child_active_sum);
983	u32 inuse_sum = READ_ONCE(parent->child_inuse_sum);	1120	u64 inuse_sum = READ_ONCE(parent->child_inuse_sum);
984	u32 active = READ_ONCE(child->active);	1121	u32 active = READ_ONCE(child->active);
985	u32 inuse = READ_ONCE(child->inuse);	1122	u32 inuse = READ_ONCE(child->inuse);
986		1123
@@ -988,11 +1125,11 @@ static void current_hweight(struct ioc_gq iocg, u32 hw_activep, u32 *hw_inusep · 1 unmodified line
988	if (!active_sum \|\| !inuse_sum)	1125	if (!active_sum \|\| !inuse_sum)
989	continue;	1126	continue;
990		1127
991	active_sum = max(active, active_sum);	1128	active_sum = max_t(u64, active, active_sum);
992	hwa = hwa * active / active_sum; /* max 16bits * 10000 */	1129	hwa = div64_u64((u64)hwa * active, active_sum);
993		1130
994	inuse_sum = max(inuse, inuse_sum);	1131	inuse_sum = max_t(u64, inuse, inuse_sum);
995	hwi = hwi * inuse / inuse_sum; /* max 16bits * 10000 */	1132	hwi = div64_u64((u64)hwi * inuse, inuse_sum);
996	}	1133	}
997		1134
998	iocg->hweight_active = max_t(u32, hwa, 1);	1135	iocg->hweight_active = max_t(u32, hwa, 1);
@@ -1005,7 +1142,33 @@ out: · 6 unmodified lines
1005	*hw_inusep = iocg->hweight_inuse;	1142	*hw_inusep = iocg->hweight_inuse;
1006	}	1143	}
1007		1144
1008	static void weight_updated(struct ioc_gq *iocg)	1145	/*
		1146	* Calculate the hweight_inuse @iocg would get with max @inuse assuming all the
		1147	* other weights stay unchanged.
		1148	*/
		1149	static u32 current_hweight_max(struct ioc_gq *iocg)
		1150	{
		1151	u32 hwm = WEIGHT_ONE;
		1152	u32 inuse = iocg->active;
		1153	u64 child_inuse_sum;
		1154	int lvl;
		1155
		1156	lockdep_assert_held(&iocg->ioc->lock);
		1157
		1158	for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
		1159	struct ioc_gq *parent = iocg->ancestors[lvl];
		1160	struct ioc_gq *child = iocg->ancestors[lvl + 1];
		1161
		1162	child_inuse_sum = parent->child_inuse_sum + inuse - child->inuse;
		1163	hwm = div64_u64((u64)hwm * inuse, child_inuse_sum);
		1164	inuse = DIV64_U64_ROUND_UP(parent->active * child_inuse_sum,
		1165	parent->child_active_sum);
		1166	}
		1167
		1168	return max_t(u32, hwm, 1);
		1169	}
		1170
		1171	static void weight_updated(struct ioc_gq iocg, struct ioc_now now)
1009	{	1172	{
1010	struct ioc *ioc = iocg->ioc;	1173	struct ioc *ioc = iocg->ioc;
1011	struct blkcg_gq *blkg = iocg_to_blkg(iocg);	1174	struct blkcg_gq *blkg = iocg_to_blkg(iocg);
@@ -1016,16 +1179,15 @@ static void weight_updated(struct ioc_gq *iocg) · 4 unmodified lines
1016		1179
1017	weight = iocg->cfg_weight ?: iocc->dfl_weight;	1180	weight = iocg->cfg_weight ?: iocc->dfl_weight;
1018	if (weight != iocg->weight && iocg->active)	1181	if (weight != iocg->weight && iocg->active)
1019	propagate_active_weight(iocg, weight,	1182	propagate_weights(iocg, weight, iocg->inuse, true, now);
1020	DIV64_U64_ROUND_UP(iocg->inuse * weight, iocg->weight));
1021	iocg->weight = weight;	1183	iocg->weight = weight;
1022	}	1184	}
1023		1185
1024	static bool iocg_activate(struct ioc_gq iocg, struct ioc_now now)	1186	static bool iocg_activate(struct ioc_gq iocg, struct ioc_now now)
1025	{	1187	{
1026	struct ioc *ioc = iocg->ioc;	1188	struct ioc *ioc = iocg->ioc;
1027	u64 last_period, cur_period, max_period_delta;	1189	u64 last_period, cur_period;
1028	u64 vtime, vmargin, vmin;	1190	u64 vtime, vtarget;
1029	int i;	1191	int i;
1030		1192
1031	/*	1193	/*
@@ -1064,22 +1226,15 @@ static bool iocg_activate(struct ioc_gq iocg, struct ioc_now now) · 32 unmodified lines
1064	goto fail_unlock;	1226	goto fail_unlock;
1065		1227
1066	/*	1228	/*
1067	* vtime may wrap when vrate is raised substantially due to	1229	* Always start with the target budget. On deactivation, we throw away
1068	* underestimated IO costs. Look at the period and ignore its	1230	* anything above it.
1069	* vtime if the iocg has been idle for too long. Also, cap the
1070	* budget it can start with to the margin.
1071	*/	1231	*/
1072	max_period_delta = DIV64_U64_ROUND_UP(VTIME_VALID_DUR, ioc->period_us);	1232	vtarget = now->vnow - ioc->margins.target;
1073	vtime = atomic64_read(&iocg->vtime);	1233	vtime = atomic64_read(&iocg->vtime);
1074	vmargin = ioc->margin_us * now->vrate;
1075	vmin = now->vnow - vmargin;
1076		1234
1077	if (last_period + max_period_delta < cur_period \|\|	1235	atomic64_add(vtarget - vtime, &iocg->vtime);
1078	time_before64(vtime, vmin)) {	1236	atomic64_add(vtarget - vtime, &iocg->done_vtime);
1079	atomic64_add(vmin - vtime, &iocg->vtime);	1237	vtime = vtarget;
1080	atomic64_add(vmin - vtime, &iocg->done_vtime);
1081	vtime = vmin;
1082	}
1083		1238
1084	/*	1239	/*
1085	* Activate, propagate weight and start period timer if not	1240	* Activate, propagate weight and start period timer if not
@@ -1088,16 +1243,18 @@ static bool iocg_activate(struct ioc_gq iocg, struct ioc_now now) · 2 unmodified lines
1088	*/	1243	*/
1089	iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;	1244	iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;
1090	list_add(&iocg->active_list, &ioc->active_iocgs);	1245	list_add(&iocg->active_list, &ioc->active_iocgs);
1091	propagate_active_weight(iocg, iocg->weight,	1246
1092	iocg->last_inuse ?: iocg->weight);	1247	propagate_weights(iocg, iocg->weight,
		1248	iocg->last_inuse ?: iocg->weight, true, now);
1093		1249
1094	TRACE_IOCG_PATH(iocg_activate, iocg, now,	1250	TRACE_IOCG_PATH(iocg_activate, iocg, now,
1095	last_period, cur_period, vtime);	1251	last_period, cur_period, vtime);
1096		1252
1097	iocg->last_vtime = vtime;	1253	iocg->activated_at = now->now;
1098		1254
1099	if (ioc->running == IOC_IDLE) {	1255	if (ioc->running == IOC_IDLE) {
1100	ioc->running = IOC_RUNNING;	1256	ioc->running = IOC_RUNNING;
		1257	ioc->debt_busy_at = now->now;
1101	ioc_start_period(ioc, now);	1258	ioc_start_period(ioc, now);
1102	}	1259	}
1103		1260
@@ -1110,6 +1267,110 @@ fail_unlock: · 6 unmodified lines
1110	return false;	1267	return false;
1111	}	1268	}
1112		1269
		1270	static bool iocg_kick_delay(struct ioc_gq iocg, struct ioc_now now)
		1271	{
		1272	struct ioc *ioc = iocg->ioc;
		1273	struct blkcg_gq *blkg = iocg_to_blkg(iocg);
		1274	u64 tdelta, delay, new_delay;
		1275	s64 vover, vover_pct;
		1276	u32 hwa;
		1277
		1278	lockdep_assert_held(&iocg->waitq.lock);
		1279
		1280	/* calculate the current delay in effect - 1/2 every second */
		1281	tdelta = now->now - iocg->delay_at;
		1282	if (iocg->delay)
		1283	delay = iocg->delay >> div64_u64(tdelta, USEC_PER_SEC);
		1284	else
		1285	delay = 0;
		1286
		1287	/* calculate the new delay from the debt amount */
		1288	current_hweight(iocg, &hwa, NULL);
		1289	vover = atomic64_read(&iocg->vtime) +
		1290	abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;
		1291	vover_pct = div64_s64(100 * vover,
		1292	ioc->period_us * ioc->vtime_base_rate);
		1293
		1294	if (vover_pct <= MIN_DELAY_THR_PCT)
		1295	new_delay = 0;
		1296	else if (vover_pct >= MAX_DELAY_THR_PCT)
		1297	new_delay = MAX_DELAY;
		1298	else
		1299	new_delay = MIN_DELAY +
		1300	div_u64((MAX_DELAY - MIN_DELAY) *
		1301	(vover_pct - MIN_DELAY_THR_PCT),
		1302	MAX_DELAY_THR_PCT - MIN_DELAY_THR_PCT);
		1303
		1304	/* pick the higher one and apply */
		1305	if (new_delay > delay) {
		1306	iocg->delay = new_delay;
		1307	iocg->delay_at = now->now;
		1308	delay = new_delay;
		1309	}
		1310
		1311	if (delay >= MIN_DELAY) {
		1312	if (!iocg->indelay_since)
		1313	iocg->indelay_since = now->now;
		1314	blkcg_set_delay(blkg, delay * NSEC_PER_USEC);
		1315	return true;
		1316	} else {
		1317	if (iocg->indelay_since) {
		1318	iocg->local_stat.indelay_us += now->now - iocg->indelay_since;
		1319	iocg->indelay_since = 0;
		1320	}
		1321	iocg->delay = 0;
		1322	blkcg_clear_delay(blkg);
		1323	return false;
		1324	}
		1325	}
		1326
		1327	static void iocg_incur_debt(struct ioc_gq *iocg, u64 abs_cost,
		1328	struct ioc_now *now)
		1329	{
		1330	struct iocg_pcpu_stat *gcs;
		1331
		1332	lockdep_assert_held(&iocg->ioc->lock);
		1333	lockdep_assert_held(&iocg->waitq.lock);
		1334	WARN_ON_ONCE(list_empty(&iocg->active_list));
		1335
		1336	/*
		1337	* Once in debt, debt handling owns inuse. @iocg stays at the minimum
		1338	* inuse donating all of it share to others until its debt is paid off.
		1339	*/
		1340	if (!iocg->abs_vdebt && abs_cost) {
		1341	iocg->indebt_since = now->now;
		1342	propagate_weights(iocg, iocg->active, 0, false, now);
		1343	}
		1344
		1345	iocg->abs_vdebt += abs_cost;
		1346
		1347	gcs = get_cpu_ptr(iocg->pcpu_stat);
		1348	local64_add(abs_cost, &gcs->abs_vusage);
		1349	put_cpu_ptr(gcs);
		1350	}
		1351
		1352	static void iocg_pay_debt(struct ioc_gq *iocg, u64 abs_vpay,
		1353	struct ioc_now *now)
		1354	{
		1355	lockdep_assert_held(&iocg->ioc->lock);
		1356	lockdep_assert_held(&iocg->waitq.lock);
		1357
		1358	/* make sure that nobody messed with @iocg */
		1359	WARN_ON_ONCE(list_empty(&iocg->active_list));
		1360	WARN_ON_ONCE(iocg->inuse > 1);
		1361
		1362	iocg->abs_vdebt -= min(abs_vpay, iocg->abs_vdebt);
		1363
		1364	/* if debt is paid in full, restore inuse */
		1365	if (!iocg->abs_vdebt) {
		1366	iocg->local_stat.indebt_us += now->now - iocg->indebt_since;
		1367	iocg->indebt_since = 0;
		1368
		1369	propagate_weights(iocg, iocg->active, iocg->last_inuse,
		1370	false, now);
		1371	}
		1372	}
		1373
1113	static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode,	1374	static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode,
1114	int flags, void *key)	1375	int flags, void *key)
1115	{	1376	{
@@ -1122,7 +1383,7 @@ static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode, · 6 unmodified lines
1122	if (ctx->vbudget < 0)	1383	if (ctx->vbudget < 0)
1123	return -1;	1384	return -1;
1124		1385
1125	iocg_commit_bio(ctx->iocg, wait->bio, cost);	1386	iocg_commit_bio(ctx->iocg, wait->bio, wait->abs_cost, cost);
1126		1387
1127	/*	1388	/*
1128	* autoremove_wake_function() removes the wait entry only when it	1389	* autoremove_wake_function() removes the wait entry only when it
@@ -1136,132 +1397,106 @@ static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode, · 7 unmodified lines
1136	return 0;	1397	return 0;
1137	}	1398	}
1138		1399
1139	static void iocg_kick_waitq(struct ioc_gq iocg, struct ioc_now now)	1400	/*
		1401	* Calculate the accumulated budget, pay debt if @pay_debt and wake up waiters
		1402	* accordingly. When @pay_debt is %true, the caller must be holding ioc->lock in
		1403	* addition to iocg->waitq.lock.
		1404	*/
		1405	static void iocg_kick_waitq(struct ioc_gq *iocg, bool pay_debt,
		1406	struct ioc_now *now)
1140	{	1407	{
1141	struct ioc *ioc = iocg->ioc;	1408	struct ioc *ioc = iocg->ioc;
1142	struct iocg_wake_ctx ctx = { .iocg = iocg };	1409	struct iocg_wake_ctx ctx = { .iocg = iocg };
1143	u64 margin_ns = (u64)(ioc->period_us *	1410	u64 vshortage, expires, oexpires;
1144	WAITQ_TIMER_MARGIN_PCT / 100) * NSEC_PER_USEC;
1145	u64 vdebt, vshortage, expires, oexpires;
1146	s64 vbudget;	1411	s64 vbudget;
1147	u32 hw_inuse;	1412	u32 hwa;
1148		1413
1149	lockdep_assert_held(&iocg->waitq.lock);	1414	lockdep_assert_held(&iocg->waitq.lock);
1150		1415
1151	current_hweight(iocg, NULL, &hw_inuse);	1416	current_hweight(iocg, &hwa, NULL);
1152	vbudget = now->vnow - atomic64_read(&iocg->vtime);	1417	vbudget = now->vnow - atomic64_read(&iocg->vtime);
1153		1418
1154	/* pay off debt */	1419	/* pay off debt */
1155	vdebt = abs_cost_to_cost(iocg->abs_vdebt, hw_inuse);	1420	if (pay_debt && iocg->abs_vdebt && vbudget > 0) {
1156	if (vdebt && vbudget > 0) {	1421	u64 abs_vbudget = cost_to_abs_cost(vbudget, hwa);
1157	u64 delta = min_t(u64, vbudget, vdebt);	1422	u64 abs_vpay = min_t(u64, abs_vbudget, iocg->abs_vdebt);
1158	u64 abs_delta = min(cost_to_abs_cost(delta, hw_inuse),	1423	u64 vpay = abs_cost_to_cost(abs_vpay, hwa);
1159	iocg->abs_vdebt);	1424
		1425	lockdep_assert_held(&ioc->lock);
		1426
		1427	atomic64_add(vpay, &iocg->vtime);
		1428	atomic64_add(vpay, &iocg->done_vtime);
		1429	iocg_pay_debt(iocg, abs_vpay, now);
		1430	vbudget -= vpay;
		1431	}
		1432
		1433	if (iocg->abs_vdebt \|\| iocg->delay)
		1434	iocg_kick_delay(iocg, now);
1160		1435
1161	atomic64_add(delta, &iocg->vtime);	1436	/*
1162	atomic64_add(delta, &iocg->done_vtime);	1437	* Debt can still be outstanding if we haven't paid all yet or the
1163	iocg->abs_vdebt -= abs_delta;	1438	* caller raced and called without @pay_debt. Shouldn't wake up waiters
		1439	* under debt. Make sure @vbudget reflects the outstanding amount and is
		1440	* not positive.
		1441	*/
		1442	if (iocg->abs_vdebt) {
		1443	s64 vdebt = abs_cost_to_cost(iocg->abs_vdebt, hwa);
		1444	vbudget = min_t(s64, 0, vbudget - vdebt);
1164	}	1445	}
1165		1446
1166	/*	1447	/*
1167	* Wake up the ones which are due and see how much vtime we'll need	1448	* Wake up the ones which are due and see how much vtime we'll need for
1168	* for the next one.	1449	* the next one. As paying off debt restores hw_inuse, it must be read
		1450	* after the above debt payment.
1169	*/	1451	*/
1170	ctx.hw_inuse = hw_inuse;	1452	ctx.vbudget = vbudget;
1171	ctx.vbudget = vbudget - vdebt;	1453	current_hweight(iocg, NULL, &ctx.hw_inuse);
		1454
1172	__wake_up_locked_key(&iocg->waitq, TASK_NORMAL, &ctx);	1455	__wake_up_locked_key(&iocg->waitq, TASK_NORMAL, &ctx);
1173	if (!waitqueue_active(&iocg->waitq))	1456
		1457	if (!waitqueue_active(&iocg->waitq)) {
		1458	if (iocg->wait_since) {
		1459	iocg->local_stat.wait_us += now->now - iocg->wait_since;
		1460	iocg->wait_since = 0;
		1461	}
1174	return;	1462	return;
		1463	}
		1464
		1465	if (!iocg->wait_since)
		1466	iocg->wait_since = now->now;
		1467
1175	if (WARN_ON_ONCE(ctx.vbudget >= 0))	1468	if (WARN_ON_ONCE(ctx.vbudget >= 0))
1176	return;	1469	return;
1177		1470
1178	/* determine next wakeup, add a quarter margin to guarantee chunking */	1471	/* determine next wakeup, add a timer margin to guarantee chunking */
1179	vshortage = -ctx.vbudget;	1472	vshortage = -ctx.vbudget;
1180	expires = now->now_ns +	1473	expires = now->now_ns +
1181	DIV64_U64_ROUND_UP(vshortage, now->vrate) * NSEC_PER_USEC;	1474	DIV64_U64_ROUND_UP(vshortage, ioc->vtime_base_rate) *
1182	expires += margin_ns / 4;	1475	NSEC_PER_USEC;
		1476	expires += ioc->timer_slack_ns;
1183		1477
1184	/* if already active and close enough, don't bother */	1478	/* if already active and close enough, don't bother */
1185	oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->waitq_timer));	1479	oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->waitq_timer));
1186	if (hrtimer_is_queued(&iocg->waitq_timer) &&	1480	if (hrtimer_is_queued(&iocg->waitq_timer) &&
1187	abs(oexpires - expires) <= margin_ns / 4)	1481	abs(oexpires - expires) <= ioc->timer_slack_ns)
1188	return;	1482	return;
1189		1483
1190	hrtimer_start_range_ns(&iocg->waitq_timer, ns_to_ktime(expires),	1484	hrtimer_start_range_ns(&iocg->waitq_timer, ns_to_ktime(expires),
1191	margin_ns / 4, HRTIMER_MODE_ABS);	1485	ioc->timer_slack_ns, HRTIMER_MODE_ABS);
1192	}	1486	}
1193		1487
1194	static enum hrtimer_restart iocg_waitq_timer_fn(struct hrtimer *timer)	1488	static enum hrtimer_restart iocg_waitq_timer_fn(struct hrtimer *timer)
1195	{	1489	{
1196	struct ioc_gq *iocg = container_of(timer, struct ioc_gq, waitq_timer);	1490	struct ioc_gq *iocg = container_of(timer, struct ioc_gq, waitq_timer);
		1491	bool pay_debt = READ_ONCE(iocg->abs_vdebt);
1197	struct ioc_now now;	1492	struct ioc_now now;
1198	unsigned long flags;	1493	unsigned long flags;
1199		1494
1200	ioc_now(iocg->ioc, &now);	1495	ioc_now(iocg->ioc, &now);
1201		1496
1202	spin_lock_irqsave(&iocg->waitq.lock, flags);	1497	iocg_lock(iocg, pay_debt, &flags);
1203	iocg_kick_waitq(iocg, &now);	1498	iocg_kick_waitq(iocg, pay_debt, &now);
1204	spin_unlock_irqrestore(&iocg->waitq.lock, flags);	1499	iocg_unlock(iocg, pay_debt, &flags);
1205
1206	return HRTIMER_NORESTART;
1207	}
1208
1209	static bool iocg_kick_delay(struct ioc_gq iocg, struct ioc_now now)
1210	{
1211	struct ioc *ioc = iocg->ioc;
1212	struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1213	u64 vtime = atomic64_read(&iocg->vtime);
1214	u64 vmargin = ioc->margin_us * now->vrate;
1215	u64 margin_ns = ioc->margin_us * NSEC_PER_USEC;
1216	u64 delta_ns, expires, oexpires;
1217	u32 hw_inuse;
1218
1219	lockdep_assert_held(&iocg->waitq.lock);
1220
1221	/* debt-adjust vtime */
1222	current_hweight(iocg, NULL, &hw_inuse);
1223	vtime += abs_cost_to_cost(iocg->abs_vdebt, hw_inuse);
1224
1225	/*
1226	* Clear or maintain depending on the overage. Non-zero vdebt is what
1227	* guarantees that @iocg is online and future iocg_kick_delay() will
1228	* clear use_delay. Don't leave it on when there's no vdebt.
1229	*/
1230	if (!iocg->abs_vdebt \|\| time_before_eq64(vtime, now->vnow)) {
1231	blkcg_clear_delay(blkg);
1232	return false;
1233	}
1234	if (!atomic_read(&blkg->use_delay) &&
1235	time_before_eq64(vtime, now->vnow + vmargin))
1236	return false;
1237
1238	/* use delay */
1239	delta_ns = DIV64_U64_ROUND_UP(vtime - now->vnow,
1240	now->vrate) * NSEC_PER_USEC;
1241	blkcg_set_delay(blkg, delta_ns);
1242	expires = now->now_ns + delta_ns;
1243
1244	/* if already active and close enough, don't bother */
1245	oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->delay_timer));
1246	if (hrtimer_is_queued(&iocg->delay_timer) &&
1247	abs(oexpires - expires) <= margin_ns / 4)
1248	return true;
1249
1250	hrtimer_start_range_ns(&iocg->delay_timer, ns_to_ktime(expires),
1251	margin_ns / 4, HRTIMER_MODE_ABS);
1252	return true;
1253	}
1254
1255	static enum hrtimer_restart iocg_delay_timer_fn(struct hrtimer *timer)
1256	{
1257	struct ioc_gq *iocg = container_of(timer, struct ioc_gq, delay_timer);
1258	struct ioc_now now;
1259	unsigned long flags;
1260
1261	spin_lock_irqsave(&iocg->waitq.lock, flags);
1262	ioc_now(iocg->ioc, &now);
1263	iocg_kick_delay(iocg, &now);
1264	spin_unlock_irqrestore(&iocg->waitq.lock, flags);
1265		1500
1266	return HRTIMER_NORESTART;	1501	return HRTIMER_NORESTART;
1267	}	1502	}
@@ -1278,8 +1513,8 @@ static void ioc_lat_stat(struct ioc ioc, u32 missed_ppm_ar, u32 *rq_wait_pct_p · 10 unmodified lines
1278	u64 this_rq_wait_ns;	1513	u64 this_rq_wait_ns;
1279		1514
1280	for (rw = READ; rw <= WRITE; rw++) {	1515	for (rw = READ; rw <= WRITE; rw++) {
1281	u32 this_met = READ_ONCE(stat->missed[rw].nr_met);	1516	u32 this_met = local_read(&stat->missed[rw].nr_met);
1282	u32 this_missed = READ_ONCE(stat->missed[rw].nr_missed);	1517	u32 this_missed = local_read(&stat->missed[rw].nr_missed);
1283		1518
1284	nr_met[rw] += this_met - stat->missed[rw].last_met;	1519	nr_met[rw] += this_met - stat->missed[rw].last_met;
1285	nr_missed[rw] += this_missed - stat->missed[rw].last_missed;	1520	nr_missed[rw] += this_missed - stat->missed[rw].last_missed;
@@ -1287,7 +1522,7 @@ static void ioc_lat_stat(struct ioc ioc, u32 missed_ppm_ar, u32 *rq_wait_pct_p · 1 unmodified line
1287	stat->missed[rw].last_missed = this_missed;	1522	stat->missed[rw].last_missed = this_missed;
1288	}	1523	}
1289		1524
1290	this_rq_wait_ns = READ_ONCE(stat->rq_wait_ns);	1525	this_rq_wait_ns = local64_read(&stat->rq_wait_ns);
1291	rq_wait_ns += this_rq_wait_ns - stat->last_rq_wait_ns;	1526	rq_wait_ns += this_rq_wait_ns - stat->last_rq_wait_ns;
1292	stat->last_rq_wait_ns = this_rq_wait_ns;	1527	stat->last_rq_wait_ns = this_rq_wait_ns;
1293	}	1528	}
@@ -1322,18 +1557,426 @@ static bool iocg_is_idle(struct ioc_gq *iocg) · 28 unmodified lines
1322	return true;	1557	return true;
1323	}	1558	}
1324		1559
1325	/* returns usage with margin added if surplus is large enough */	1560	/*
1326	static u32 surplus_adjusted_hweight_inuse(u32 usage, u32 hw_inuse)	1561	* Call this function on the target leaf @iocg's to build pre-order traversal
		1562	* list of all the ancestors in @inner_walk. The inner nodes are linked through
		1563	* ->walk_list and the caller is responsible for dissolving the list after use.
		1564	*/
		1565	static void iocg_build_inner_walk(struct ioc_gq *iocg,
		1566	struct list_head *inner_walk)
1327	{	1567	{
1328	/* add margin */	1568	int lvl;
1329	usage = DIV_ROUND_UP(usage * SURPLUS_SCALE_PCT, 100);
1330	usage += SURPLUS_SCALE_ABS;
1331		1569
1332	/* don't bother if the surplus is too small */	1570	WARN_ON_ONCE(!list_empty(&iocg->walk_list));
1333	if (usage + SURPLUS_MIN_ADJ_DELTA > hw_inuse)	1571
1334	return 0;	1572	/* find the first ancestor which hasn't been visited yet */
		1573	for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
		1574	if (!list_empty(&iocg->ancestors[lvl]->walk_list))
		1575	break;
		1576	}
		1577
		1578	/* walk down and visit the inner nodes to get pre-order traversal */
		1579	while (++lvl <= iocg->level - 1) {
		1580	struct ioc_gq *inner = iocg->ancestors[lvl];
		1581
		1582	/* record traversal order */
		1583	list_add_tail(&inner->walk_list, inner_walk);
		1584	}
		1585	}
		1586
		1587	/* collect per-cpu counters and propagate the deltas to the parent */
		1588	static void iocg_flush_stat_one(struct ioc_gq iocg, struct ioc_now now)
		1589	{
		1590	struct ioc *ioc = iocg->ioc;
		1591	struct iocg_stat new_stat;
		1592	u64 abs_vusage = 0;
		1593	u64 vusage_delta;
		1594	int cpu;
		1595
		1596	lockdep_assert_held(&iocg->ioc->lock);
		1597
		1598	/* collect per-cpu counters */
		1599	for_each_possible_cpu(cpu) {
		1600	abs_vusage += local64_read(
		1601	per_cpu_ptr(&iocg->pcpu_stat->abs_vusage, cpu));
		1602	}
		1603	vusage_delta = abs_vusage - iocg->last_stat_abs_vusage;
		1604	iocg->last_stat_abs_vusage = abs_vusage;
		1605
		1606	iocg->usage_delta_us = div64_u64(vusage_delta, ioc->vtime_base_rate);
		1607	iocg->local_stat.usage_us += iocg->usage_delta_us;
		1608
		1609	/* propagate upwards */
		1610	new_stat.usage_us =
		1611	iocg->local_stat.usage_us + iocg->desc_stat.usage_us;
		1612	new_stat.wait_us =
		1613	iocg->local_stat.wait_us + iocg->desc_stat.wait_us;
		1614	new_stat.indebt_us =
		1615	iocg->local_stat.indebt_us + iocg->desc_stat.indebt_us;
		1616	new_stat.indelay_us =
		1617	iocg->local_stat.indelay_us + iocg->desc_stat.indelay_us;
		1618
		1619	/* propagate the deltas to the parent */
		1620	if (iocg->level > 0) {
		1621	struct iocg_stat *parent_stat =
		1622	&iocg->ancestors[iocg->level - 1]->desc_stat;
		1623
		1624	parent_stat->usage_us +=
		1625	new_stat.usage_us - iocg->last_stat.usage_us;
		1626	parent_stat->wait_us +=
		1627	new_stat.wait_us - iocg->last_stat.wait_us;
		1628	parent_stat->indebt_us +=
		1629	new_stat.indebt_us - iocg->last_stat.indebt_us;
		1630	parent_stat->indelay_us +=
		1631	new_stat.indelay_us - iocg->last_stat.indelay_us;
		1632	}
		1633
		1634	iocg->last_stat = new_stat;
		1635	}
1335		1636
1336	return usage;	1637	/* get stat counters ready for reading on all active iocgs */
		1638	static void iocg_flush_stat(struct list_head target_iocgs, struct ioc_now now)
		1639	{
		1640	LIST_HEAD(inner_walk);
		1641	struct ioc_gq iocg, tiocg;
		1642
		1643	/* flush leaves and build inner node walk list */
		1644	list_for_each_entry(iocg, target_iocgs, active_list) {
		1645	iocg_flush_stat_one(iocg, now);
		1646	iocg_build_inner_walk(iocg, &inner_walk);
		1647	}
		1648
		1649	/* keep flushing upwards by walking the inner list backwards */
		1650	list_for_each_entry_safe_reverse(iocg, tiocg, &inner_walk, walk_list) {
		1651	iocg_flush_stat_one(iocg, now);
		1652	list_del_init(&iocg->walk_list);
		1653	}
		1654	}
		1655
		1656	/*
		1657	* Determine what @iocg's hweight_inuse should be after donating unused
		1658	* capacity. @hwm is the upper bound and used to signal no donation. This
		1659	* function also throws away @iocg's excess budget.
		1660	*/
		1661	static u32 hweight_after_donation(struct ioc_gq *iocg, u32 old_hwi, u32 hwm,
		1662	u32 usage, struct ioc_now *now)
		1663	{
		1664	struct ioc *ioc = iocg->ioc;
		1665	u64 vtime = atomic64_read(&iocg->vtime);
		1666	s64 excess, delta, target, new_hwi;
		1667
		1668	/* debt handling owns inuse for debtors */
		1669	if (iocg->abs_vdebt)
		1670	return 1;
		1671
		1672	/* see whether minimum margin requirement is met */
		1673	if (waitqueue_active(&iocg->waitq) \|\|
		1674	time_after64(vtime, now->vnow - ioc->margins.min))
		1675	return hwm;
		1676
		1677	/* throw away excess above target */
		1678	excess = now->vnow - vtime - ioc->margins.target;
		1679	if (excess > 0) {
		1680	atomic64_add(excess, &iocg->vtime);
		1681	atomic64_add(excess, &iocg->done_vtime);
		1682	vtime += excess;
		1683	ioc->vtime_err -= div64_u64(excess * old_hwi, WEIGHT_ONE);
		1684	}
		1685
		1686	/*
		1687	* Let's say the distance between iocg's and device's vtimes as a
		1688	* fraction of period duration is delta. Assuming that the iocg will
		1689	* consume the usage determined above, we want to determine new_hwi so
		1690	* that delta equals MARGIN_TARGET at the end of the next period.
		1691	*
		1692	* We need to execute usage worth of IOs while spending the sum of the
		1693	* new budget (1 - MARGIN_TARGET) and the leftover from the last period
		1694	* (delta):
		1695	*
		1696	* usage = (1 - MARGIN_TARGET + delta) * new_hwi
		1697	*
		1698	* Therefore, the new_hwi is:
		1699	*
		1700	* new_hwi = usage / (1 - MARGIN_TARGET + delta)
		1701	*/
		1702	delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),
		1703	now->vnow - ioc->period_at_vtime);
		1704	target = WEIGHT_ONE * MARGIN_TARGET_PCT / 100;
		1705	new_hwi = div64_s64(WEIGHT_ONE * usage, WEIGHT_ONE - target + delta);
		1706
		1707	return clamp_t(s64, new_hwi, 1, hwm);
		1708	}
		1709
		1710	/*
		1711	* For work-conservation, an iocg which isn't using all of its share should
		1712	* donate the leftover to other iocgs. There are two ways to achieve this - 1.
		1713	* bumping up vrate accordingly 2. lowering the donating iocg's inuse weight.
		1714	*
		1715	* #1 is mathematically simpler but has the drawback of requiring synchronous
		1716	* global hweight_inuse updates when idle iocg's get activated or inuse weights
		1717	* change due to donation snapbacks as it has the possibility of grossly
		1718	* overshooting what's allowed by the model and vrate.
		1719	*
		1720	* #2 is inherently safe with local operations. The donating iocg can easily
		1721	* snap back to higher weights when needed without worrying about impacts on
		1722	* other nodes as the impacts will be inherently correct. This also makes idle
		1723	* iocg activations safe. The only effect activations have is decreasing
		1724	* hweight_inuse of others, the right solution to which is for those iocgs to
		1725	* snap back to higher weights.
		1726	*
		1727	* So, we go with #2. The challenge is calculating how each donating iocg's
		1728	* inuse should be adjusted to achieve the target donation amounts. This is done
		1729	* using Andy's method described in the following pdf.
		1730	*
		1731	* https://drive.google.com/file/d/1PsJwxPFtjUnwOY1QJ5AeICCcsL7BM3bo
		1732	*
		1733	* Given the weights and target after-donation hweight_inuse values, Andy's
		1734	* method determines how the proportional distribution should look like at each
		1735	* sibling level to maintain the relative relationship between all non-donating
		1736	* pairs. To roughly summarize, it divides the tree into donating and
		1737	* non-donating parts, calculates global donation rate which is used to
		1738	* determine the target hweight_inuse for each node, and then derives per-level
		1739	* proportions.
		1740	*
		1741	* The following pdf shows that global distribution calculated this way can be
		1742	* achieved by scaling inuse weights of donating leaves and propagating the
		1743	* adjustments upwards proportionally.
		1744	*
		1745	* https://drive.google.com/file/d/1vONz1-fzVO7oY5DXXsLjSxEtYYQbOvsE
		1746	*
		1747	* Combining the above two, we can determine how each leaf iocg's inuse should
		1748	* be adjusted to achieve the target donation.
		1749	*
		1750	* https://drive.google.com/file/d/1WcrltBOSPN0qXVdBgnKm4mdp9FhuEFQN
		1751	*
		1752	* The inline comments use symbols from the last pdf.
		1753	*
		1754	* b is the sum of the absolute budgets in the subtree. 1 for the root node.
		1755	* f is the sum of the absolute budgets of non-donating nodes in the subtree.
		1756	* t is the sum of the absolute budgets of donating nodes in the subtree.
		1757	* w is the weight of the node. w = w_f + w_t
		1758	* w_f is the non-donating portion of w. w_f = w * f / b
		1759	* w_b is the donating portion of w. w_t = w * t / b
		1760	* s is the sum of all sibling weights. s = Sum(w) for siblings
		1761	* s_f and s_t are the non-donating and donating portions of s.
		1762	*
		1763	* Subscript p denotes the parent's counterpart and ' the adjusted value - e.g.
		1764	* w_pt is the donating portion of the parent's weight and w'_pt the same value
		1765	* after adjustments. Subscript r denotes the root node's values.
		1766	*/
		1767	static void transfer_surpluses(struct list_head surpluses, struct ioc_now now)
		1768	{
		1769	LIST_HEAD(over_hwa);
		1770	LIST_HEAD(inner_walk);
		1771	struct ioc_gq iocg, tiocg, *root_iocg;
		1772	u32 after_sum, over_sum, over_target, gamma;
		1773
		1774	/*
		1775	* It's pretty unlikely but possible for the total sum of
		1776	* hweight_after_donation's to be higher than WEIGHT_ONE, which will
		1777	* confuse the following calculations. If such condition is detected,
		1778	* scale down everyone over its full share equally to keep the sum below
		1779	* WEIGHT_ONE.
		1780	*/
		1781	after_sum = 0;
		1782	over_sum = 0;
		1783	list_for_each_entry(iocg, surpluses, surplus_list) {
		1784	u32 hwa;
		1785
		1786	current_hweight(iocg, &hwa, NULL);
		1787	after_sum += iocg->hweight_after_donation;
		1788
		1789	if (iocg->hweight_after_donation > hwa) {
		1790	over_sum += iocg->hweight_after_donation;
		1791	list_add(&iocg->walk_list, &over_hwa);
		1792	}
		1793	}
		1794
		1795	if (after_sum >= WEIGHT_ONE) {
		1796	/*
		1797	* The delta should be deducted from the over_sum, calculate
		1798	* target over_sum value.
		1799	*/
		1800	u32 over_delta = after_sum - (WEIGHT_ONE - 1);
		1801	WARN_ON_ONCE(over_sum <= over_delta);
		1802	over_target = over_sum - over_delta;
		1803	} else {
		1804	over_target = 0;
		1805	}
		1806
		1807	list_for_each_entry_safe(iocg, tiocg, &over_hwa, walk_list) {
		1808	if (over_target)
		1809	iocg->hweight_after_donation =
		1810	div_u64((u64)iocg->hweight_after_donation *
		1811	over_target, over_sum);
		1812	list_del_init(&iocg->walk_list);
		1813	}
		1814
		1815	/*
		1816	* Build pre-order inner node walk list and prepare for donation
		1817	* adjustment calculations.
		1818	*/
		1819	list_for_each_entry(iocg, surpluses, surplus_list) {
		1820	iocg_build_inner_walk(iocg, &inner_walk);
		1821	}
		1822
		1823	root_iocg = list_first_entry(&inner_walk, struct ioc_gq, walk_list);
		1824	WARN_ON_ONCE(root_iocg->level > 0);
		1825
		1826	list_for_each_entry(iocg, &inner_walk, walk_list) {
		1827	iocg->child_adjusted_sum = 0;
		1828	iocg->hweight_donating = 0;
		1829	iocg->hweight_after_donation = 0;
		1830	}
		1831
		1832	/*
		1833	* Propagate the donating budget (b_t) and after donation budget (b'_t)
		1834	* up the hierarchy.
		1835	*/
		1836	list_for_each_entry(iocg, surpluses, surplus_list) {
		1837	struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
		1838
		1839	parent->hweight_donating += iocg->hweight_donating;
		1840	parent->hweight_after_donation += iocg->hweight_after_donation;
		1841	}
		1842
		1843	list_for_each_entry_reverse(iocg, &inner_walk, walk_list) {
		1844	if (iocg->level > 0) {
		1845	struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
		1846
		1847	parent->hweight_donating += iocg->hweight_donating;
		1848	parent->hweight_after_donation += iocg->hweight_after_donation;
		1849	}
		1850	}
		1851
		1852	/*
		1853	* Calculate inner hwa's (b) and make sure the donation values are
		1854	* within the accepted ranges as we're doing low res calculations with
		1855	* roundups.
		1856	*/
		1857	list_for_each_entry(iocg, &inner_walk, walk_list) {
		1858	if (iocg->level) {
		1859	struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
		1860
		1861	iocg->hweight_active = DIV64_U64_ROUND_UP(
		1862	(u64)parent->hweight_active * iocg->active,
		1863	parent->child_active_sum);
		1864
		1865	}
		1866
		1867	iocg->hweight_donating = min(iocg->hweight_donating,
		1868	iocg->hweight_active);
		1869	iocg->hweight_after_donation = min(iocg->hweight_after_donation,
		1870	iocg->hweight_donating - 1);
		1871	if (WARN_ON_ONCE(iocg->hweight_active <= 1 \|\|
		1872	iocg->hweight_donating <= 1 \|\|
		1873	iocg->hweight_after_donation == 0)) {
		1874	pr_warn("iocg: invalid donation weights in ");
		1875	pr_cont_cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup);
		1876	pr_cont(": active=%u donating=%u after=%u\n",
		1877	iocg->hweight_active, iocg->hweight_donating,
		1878	iocg->hweight_after_donation);
		1879	}
		1880	}
		1881
		1882	/*
		1883	* Calculate the global donation rate (gamma) - the rate to adjust
		1884	* non-donating budgets by.
		1885	*
		1886	* No need to use 64bit multiplication here as the first operand is
		1887	* guaranteed to be smaller than WEIGHT_ONE (1<<16).
		1888	*
		1889	* We know that there are beneficiary nodes and the sum of the donating
		1890	* hweights can't be whole; however, due to the round-ups during hweight
		1891	* calculations, root_iocg->hweight_donating might still end up equal to
		1892	* or greater than whole. Limit the range when calculating the divider.
		1893	*
		1894	* gamma = (1 - t_r') / (1 - t_r)
		1895	*/
		1896	gamma = DIV_ROUND_UP(
		1897	(WEIGHT_ONE - root_iocg->hweight_after_donation) * WEIGHT_ONE,
		1898	WEIGHT_ONE - min_t(u32, root_iocg->hweight_donating, WEIGHT_ONE - 1));
		1899
		1900	/*
		1901	* Calculate adjusted hwi, child_adjusted_sum and inuse for the inner
		1902	* nodes.
		1903	*/
		1904	list_for_each_entry(iocg, &inner_walk, walk_list) {
		1905	struct ioc_gq *parent;
		1906	u32 inuse, wpt, wptp;
		1907	u64 st, sf;
		1908
		1909	if (iocg->level == 0) {
		1910	/* adjusted weight sum for 1st level: s' = s * b_pf / b'_pf */
		1911	iocg->child_adjusted_sum = DIV64_U64_ROUND_UP(
		1912	iocg->child_active_sum * (WEIGHT_ONE - iocg->hweight_donating),
		1913	WEIGHT_ONE - iocg->hweight_after_donation);
		1914	continue;
		1915	}
		1916
		1917	parent = iocg->ancestors[iocg->level - 1];
		1918
		1919	/* b' = gamma * b_f + b_t' */
		1920	iocg->hweight_inuse = DIV64_U64_ROUND_UP(
		1921	(u64)gamma * (iocg->hweight_active - iocg->hweight_donating),
		1922	WEIGHT_ONE) + iocg->hweight_after_donation;
		1923
		1924	/* w' = s' * b' / b'_p */
		1925	inuse = DIV64_U64_ROUND_UP(
		1926	(u64)parent->child_adjusted_sum * iocg->hweight_inuse,
		1927	parent->hweight_inuse);
		1928
		1929	/* adjusted weight sum for children: s' = s_f + s_t * w'_pt / w_pt */
		1930	st = DIV64_U64_ROUND_UP(
		1931	iocg->child_active_sum * iocg->hweight_donating,
		1932	iocg->hweight_active);
		1933	sf = iocg->child_active_sum - st;
		1934	wpt = DIV64_U64_ROUND_UP(
		1935	(u64)iocg->active * iocg->hweight_donating,
		1936	iocg->hweight_active);
		1937	wptp = DIV64_U64_ROUND_UP(
		1938	(u64)inuse * iocg->hweight_after_donation,
		1939	iocg->hweight_inuse);
		1940
		1941	iocg->child_adjusted_sum = sf + DIV64_U64_ROUND_UP(st * wptp, wpt);
		1942	}
		1943
		1944	/*
		1945	* All inner nodes now have ->hweight_inuse and ->child_adjusted_sum and
		1946	* we can finally determine leaf adjustments.
		1947	*/
		1948	list_for_each_entry(iocg, surpluses, surplus_list) {
		1949	struct ioc_gq *parent = iocg->ancestors[iocg->level - 1];
		1950	u32 inuse;
		1951
		1952	/*
		1953	* In-debt iocgs participated in the donation calculation with
		1954	* the minimum target hweight_inuse. Configuring inuse
		1955	* accordingly would work fine but debt handling expects
		1956	* @iocg->inuse stay at the minimum and we don't wanna
		1957	* interfere.
		1958	*/
		1959	if (iocg->abs_vdebt) {
		1960	WARN_ON_ONCE(iocg->inuse > 1);
		1961	continue;
		1962	}
		1963
		1964	/* w' = s' * b' / b'_p, note that b' == b'_t for donating leaves */
		1965	inuse = DIV64_U64_ROUND_UP(
		1966	parent->child_adjusted_sum * iocg->hweight_after_donation,
		1967	parent->hweight_inuse);
		1968
		1969	TRACE_IOCG_PATH(inuse_transfer, iocg, now,
		1970	iocg->inuse, inuse,
		1971	iocg->hweight_inuse,
		1972	iocg->hweight_after_donation);
		1973
		1974	__propagate_weights(iocg, iocg->active, inuse, true, now);
		1975	}
		1976
		1977	/* walk list should be dissolved after use */
		1978	list_for_each_entry_safe(iocg, tiocg, &inner_walk, walk_list)
		1979	list_del_init(&iocg->walk_list);
1337	}	1980	}
1338		1981
1339	static void ioc_timer_fn(struct timer_list *timer)	1982	static void ioc_timer_fn(struct timer_list *timer)
@@ -1341,12 +1984,14 @@ static void ioc_timer_fn(struct timer_list *timer) · 1 unmodified line
1341	struct ioc *ioc = container_of(timer, struct ioc, timer);	1984	struct ioc *ioc = container_of(timer, struct ioc, timer);
1342	struct ioc_gq iocg, tiocg;	1985	struct ioc_gq iocg, tiocg;
1343	struct ioc_now now;	1986	struct ioc_now now;
1344	int nr_surpluses = 0, nr_shortages = 0, nr_lagging = 0;	1987	LIST_HEAD(surpluses);
		1988	int nr_debtors = 0, nr_shortages = 0, nr_lagging = 0;
		1989	u64 usage_us_sum = 0;
1345	u32 ppm_rthr = MILLION - ioc->params.qos[QOS_RPPM];	1990	u32 ppm_rthr = MILLION - ioc->params.qos[QOS_RPPM];
1346	u32 ppm_wthr = MILLION - ioc->params.qos[QOS_WPPM];	1991	u32 ppm_wthr = MILLION - ioc->params.qos[QOS_WPPM];
1347	u32 missed_ppm[2], rq_wait_pct;	1992	u32 missed_ppm[2], rq_wait_pct;
1348	u64 period_vtime;	1993	u64 period_vtime;
1349	int prev_busy_level, i;	1994	int prev_busy_level;
1350		1995
1351	/* how were the latencies during the period? */	1996	/* how were the latencies during the period? */
1352	ioc_lat_stat(ioc, missed_ppm, &rq_wait_pct);	1997	ioc_lat_stat(ioc, missed_ppm, &rq_wait_pct);
@@ -1370,30 +2015,71 @@ static void ioc_timer_fn(struct timer_list *timer) · 17 unmodified lines
1370	*/	2015	*/
1371	list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {	2016	list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {
1372	if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&	2017	if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
1373	!iocg_is_idle(iocg))	2018	!iocg->delay && !iocg_is_idle(iocg))
1374	continue;	2019	continue;
1375		2020
1376	spin_lock(&iocg->waitq.lock);	2021	spin_lock(&iocg->waitq.lock);
1377		2022
1378	if (waitqueue_active(&iocg->waitq) \|\| iocg->abs_vdebt) {	2023	/* flush wait and indebt stat deltas */
		2024	if (iocg->wait_since) {
		2025	iocg->local_stat.wait_us += now.now - iocg->wait_since;
		2026	iocg->wait_since = now.now;
		2027	}
		2028	if (iocg->indebt_since) {
		2029	iocg->local_stat.indebt_us +=
		2030	now.now - iocg->indebt_since;
		2031	iocg->indebt_since = now.now;
		2032	}
		2033	if (iocg->indelay_since) {
		2034	iocg->local_stat.indelay_us +=
		2035	now.now - iocg->indelay_since;
		2036	iocg->indelay_since = now.now;
		2037	}
		2038
		2039	if (waitqueue_active(&iocg->waitq) \|\| iocg->abs_vdebt \|\|
		2040	iocg->delay) {
1379	/* might be oversleeping vtime / hweight changes, kick */	2041	/* might be oversleeping vtime / hweight changes, kick */
1380	iocg_kick_waitq(iocg, &now);	2042	iocg_kick_waitq(iocg, true, &now);
1381	iocg_kick_delay(iocg, &now);	2043	if (iocg->abs_vdebt)
		2044	nr_debtors++;
1382	} else if (iocg_is_idle(iocg)) {	2045	} else if (iocg_is_idle(iocg)) {
1383	/* no waiter and idle, deactivate */	2046	/* no waiter and idle, deactivate */
1384	iocg->last_inuse = iocg->inuse;	2047	u64 vtime = atomic64_read(&iocg->vtime);
1385	__propagate_active_weight(iocg, 0, 0);	2048	s64 excess;
		2049
		2050	/*
		2051	* @iocg has been inactive for a full duration and will
		2052	* have a high budget. Account anything above target as
		2053	* error and throw away. On reactivation, it'll start
		2054	* with the target budget.
		2055	*/
		2056	excess = now.vnow - vtime - ioc->margins.target;
		2057	if (excess > 0) {
		2058	u32 old_hwi;
		2059
		2060	current_hweight(iocg, NULL, &old_hwi);
		2061	ioc->vtime_err -= div64_u64(excess * old_hwi,
		2062	WEIGHT_ONE);
		2063	}
		2064
		2065	__propagate_weights(iocg, 0, 0, false, &now);
1386	list_del_init(&iocg->active_list);	2066	list_del_init(&iocg->active_list);
1387	}	2067	}
1388		2068
1389	spin_unlock(&iocg->waitq.lock);	2069	spin_unlock(&iocg->waitq.lock);
1390	}	2070	}
1391	commit_active_weights(ioc);	2071	commit_weights(ioc);
		2072
		2073	/*
		2074	* Wait and indebt stat are flushed above and the donation calculation
		2075	* below needs updated usage stat. Let's bring stat up-to-date.
		2076	*/
		2077	iocg_flush_stat(&ioc->active_iocgs, &now);
1392		2078
1393	/* calc usages and see whether some weights need to be moved around */	2079	/* calc usage and see whether some weights need to be moved around */
1394	list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {	2080	list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
1395	u64 vdone, vtime, vusage, vmargin, vmin;	2081	u64 vdone, vtime, usage_us, usage_dur;
1396	u32 hw_active, hw_inuse, usage;	2082	u32 usage, hw_active, hw_inuse;
1397		2083
1398	/*	2084	/*
1399	* Collect unused and wind vtime closer to vnow to prevent	2085	* Collect unused and wind vtime closer to vnow to prevent
@@ -1417,116 +2103,105 @@ static void ioc_timer_fn(struct timer_list *timer) · 17 unmodified lines
1417	time_before64(vdone, now.vnow - period_vtime))	2103	time_before64(vdone, now.vnow - period_vtime))
1418	nr_lagging++;	2104	nr_lagging++;
1419		2105
1420	if (waitqueue_active(&iocg->waitq))
1421	vusage = now.vnow - iocg->last_vtime;
1422	else if (time_before64(iocg->last_vtime, vtime))
1423	vusage = vtime - iocg->last_vtime;
1424	else
1425	vusage = 0;
1426
1427	iocg->last_vtime += vusage;
1428	/*	2106	/*
1429	* Factor in in-flight vtime into vusage to avoid	2107	* Determine absolute usage factoring in in-flight IOs to avoid
1430	* high-latency completions appearing as idle. This should	2108	* high-latency completions appearing as idle.
1431	* be done after the above ->last_time adjustment.
1432	*/	2109	*/
1433	vusage = max(vusage, vtime - vdone);	2110	usage_us = iocg->usage_delta_us;
1434		2111	usage_us_sum += usage_us;
1435	/* calculate hweight based usage ratio and record */	2112
1436	if (vusage) {	2113	if (vdone != vtime) {
1437	usage = DIV64_U64_ROUND_UP(vusage * hw_inuse,	2114	u64 inflight_us = DIV64_U64_ROUND_UP(
1438	period_vtime);	2115	cost_to_abs_cost(vtime - vdone, hw_inuse),
1439	iocg->usage_idx = (iocg->usage_idx + 1) % NR_USAGE_SLOTS;	2116	ioc->vtime_base_rate);
1440	iocg->usages[iocg->usage_idx] = usage;	2117	usage_us = max(usage_us, inflight_us);
1441	} else {
1442	usage = 0;
1443	}	2118	}
1444		2119
		2120	/* convert to hweight based usage ratio */
		2121	if (time_after64(iocg->activated_at, ioc->period_at))
		2122	usage_dur = max_t(u64, now.now - iocg->activated_at, 1);
		2123	else
		2124	usage_dur = max_t(u64, now.now - ioc->period_at, 1);
		2125
		2126	usage = clamp_t(u32,
		2127	DIV64_U64_ROUND_UP(usage_us * WEIGHT_ONE,
		2128	usage_dur),
		2129	1, WEIGHT_ONE);
		2130
1445	/* see whether there's surplus vtime */	2131	/* see whether there's surplus vtime */
1446	vmargin = ioc->margin_us * now.vrate;	2132	WARN_ON_ONCE(!list_empty(&iocg->surplus_list));
1447	vmin = now.vnow - vmargin;	2133	if (hw_inuse < hw_active \|\|
1448		2134	(!waitqueue_active(&iocg->waitq) &&
1449	iocg->has_surplus = false;	2135	time_before64(vtime, now.vnow - ioc->margins.low))) {
1450		2136	u32 hwa, old_hwi, hwm, new_hwi;
1451	if (!waitqueue_active(&iocg->waitq) &&
1452	time_before64(vtime, vmin)) {
1453	u64 delta = vmin - vtime;
1454
1455	/* throw away surplus vtime */
1456	atomic64_add(delta, &iocg->vtime);
1457	atomic64_add(delta, &iocg->done_vtime);
1458	iocg->last_vtime += delta;
1459	/* if usage is sufficiently low, maybe it can donate */
1460	if (surplus_adjusted_hweight_inuse(usage, hw_inuse)) {
1461	iocg->has_surplus = true;
1462	nr_surpluses++;
1463	}
1464	} else if (hw_inuse < hw_active) {
1465	u32 new_hwi, new_inuse;
1466		2137
1467	/* was donating but might need to take back some */	2138	/*
1468	if (waitqueue_active(&iocg->waitq)) {	2139	* Already donating or accumulated enough to start.
1469	new_hwi = hw_active;	2140	* Determine the donation amount.
		2141	*/
		2142	current_hweight(iocg, &hwa, &old_hwi);
		2143	hwm = current_hweight_max(iocg);
		2144	new_hwi = hweight_after_donation(iocg, old_hwi, hwm,
		2145	usage, &now);
		2146	if (new_hwi < hwm) {
		2147	iocg->hweight_donating = hwa;
		2148	iocg->hweight_after_donation = new_hwi;
		2149	list_add(&iocg->surplus_list, &surpluses);
1470	} else {	2150	} else {
1471	new_hwi = max(hw_inuse,	2151	TRACE_IOCG_PATH(inuse_shortage, iocg, &now,
1472	usage * SURPLUS_SCALE_PCT / 100 +	2152	iocg->inuse, iocg->active,
1473	SURPLUS_SCALE_ABS);	2153	iocg->hweight_inuse, new_hwi);
1474	}
1475		2154
1476	new_inuse = div64_u64((u64)iocg->inuse * new_hwi,	2155	__propagate_weights(iocg, iocg->active,
1477	hw_inuse);	2156	iocg->active, true, &now);
1478	new_inuse = clamp_t(u32, new_inuse, 1, iocg->active);	2157	nr_shortages++;
1479
1480	if (new_inuse > iocg->inuse) {
1481	TRACE_IOCG_PATH(inuse_takeback, iocg, &now,
1482	iocg->inuse, new_inuse,
1483	hw_inuse, new_hwi);
1484	__propagate_active_weight(iocg, iocg->weight,
1485	new_inuse);
1486	}	2158	}
1487	} else {	2159	} else {
1488	/* genuninely out of vtime */	2160	/* genuinely short on vtime */
1489	nr_shortages++;	2161	nr_shortages++;
1490	}	2162	}
1491	}	2163	}
1492		2164
1493	if (!nr_shortages \|\| !nr_surpluses)	2165	if (!list_empty(&surpluses) && nr_shortages)
1494	goto skip_surplus_transfers;	2166	transfer_surpluses(&surpluses, &now);
1495		2167
1496	/* there are both shortages and surpluses, transfer surpluses */	2168	commit_weights(ioc);
1497	list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
1498	u32 usage, hw_active, hw_inuse, new_hwi, new_inuse;
1499	int nr_valid = 0;
1500		2169
1501	if (!iocg->has_surplus)	2170	/* surplus list should be dissolved after use */
1502	continue;	2171	list_for_each_entry_safe(iocg, tiocg, &surpluses, surplus_list)
		2172	list_del_init(&iocg->surplus_list);
1503		2173
1504	/* base the decision on max historical usage */	2174	/*
1505	for (i = 0, usage = 0; i < NR_USAGE_SLOTS; i++) {	2175	* A low weight iocg can amass a large amount of debt, for example, when
1506	if (iocg->usages[i]) {	2176	* anonymous memory gets reclaimed aggressively. If the system has a lot
1507	usage = max(usage, iocg->usages[i]);	2177	* of memory paired with a slow IO device, the debt can span multiple
1508	nr_valid++;	2178	* seconds or more. If there are no other subsequent IO issuers, the
		2179	* in-debt iocg may end up blocked paying its debt while the IO device
		2180	* is idle.
		2181	*
		2182	* The following protects against such pathological cases. If the device
		2183	* has been sufficiently idle for a substantial amount of time, the
		2184	* debts are halved. The criteria are on the conservative side as we
		2185	* want to resolve the rare extreme cases without impacting regular
		2186	* operation by forgiving debts too readily.
		2187	*/
		2188	if (nr_shortages \|\|
		2189	div64_u64(100 * usage_us_sum, now.now - ioc->period_at) >=
		2190	DEBT_BUSY_USAGE_PCT)
		2191	ioc->debt_busy_at = now.now;
		2192
		2193	if (nr_debtors &&
		2194	now.now - ioc->debt_busy_at >= DEBT_REDUCTION_IDLE_DUR) {
		2195	list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
		2196	if (iocg->abs_vdebt) {
		2197	spin_lock(&iocg->waitq.lock);
		2198	iocg->abs_vdebt /= 2;
		2199	iocg_kick_waitq(iocg, true, &now);
		2200	spin_unlock(&iocg->waitq.lock);
1509	}	2201	}
1510	}	2202	}
1511	if (nr_valid < MIN_VALID_USAGES)	2203	ioc->debt_busy_at = now.now;
1512	continue;
1513
1514	current_hweight(iocg, &hw_active, &hw_inuse);
1515	new_hwi = surplus_adjusted_hweight_inuse(usage, hw_inuse);
1516	if (!new_hwi)
1517	continue;
1518
1519	new_inuse = DIV64_U64_ROUND_UP((u64)iocg->inuse * new_hwi,
1520	hw_inuse);
1521	if (new_inuse < iocg->inuse) {
1522	TRACE_IOCG_PATH(inuse_giveaway, iocg, &now,
1523	iocg->inuse, new_inuse,
1524	hw_inuse, new_hwi);
1525	__propagate_active_weight(iocg, iocg->weight, new_inuse);
1526	}
1527	}	2204	}
1528	skip_surplus_transfers:
1529	commit_active_weights(ioc);
1530		2205
1531	/*	2206	/*
1532	* If q is getting clogged or we're missing too much, we're issuing	2207	* If q is getting clogged or we're missing too much, we're issuing
@@ -1554,11 +2229,9 @@ skip_surplus_transfers: · 21 unmodified lines
1554		2229
1555	/*	2230	/*
1556	* If there are IOs spanning multiple periods, wait	2231	* If there are IOs spanning multiple periods, wait
1557	* them out before pushing the device harder. If	2232	* them out before pushing the device harder.
1558	* there are surpluses, let redistribution work it
1559	* out first.
1560	*/	2233	*/
1561	if (!nr_lagging && !nr_surpluses)	2234	if (!nr_lagging)
1562	ioc->busy_level--;	2235	ioc->busy_level--;
1563	} else {	2236	} else {
1564	/*	2237	/*
@@ -1577,7 +2250,7 @@ skip_surplus_transfers: · 12 unmodified lines
1577	ioc->busy_level = clamp(ioc->busy_level, -1000, 1000);	2250	ioc->busy_level = clamp(ioc->busy_level, -1000, 1000);
1578		2251
1579	if (ioc->busy_level > 0 \|\| (ioc->busy_level < 0 && !nr_lagging)) {	2252	if (ioc->busy_level > 0 \|\| (ioc->busy_level < 0 && !nr_lagging)) {
1580	u64 vrate = atomic64_read(&ioc->vtime_rate);	2253	u64 vrate = ioc->vtime_base_rate;
1581	u64 vrate_min = ioc->vrate_min, vrate_max = ioc->vrate_max;	2254	u64 vrate_min = ioc->vrate_min, vrate_max = ioc->vrate_max;
1582		2255
1583	/* rq_wait signal is always reliable, ignore user vrate_min */	2256	/* rq_wait signal is always reliable, ignore user vrate_min */
@@ -1612,16 +2285,14 @@ skip_surplus_transfers: · 28 unmodified lines
1612	}	2285	}
1613		2286
1614	trace_iocost_ioc_vrate_adj(ioc, vrate, missed_ppm, rq_wait_pct,	2287	trace_iocost_ioc_vrate_adj(ioc, vrate, missed_ppm, rq_wait_pct,
1615	nr_lagging, nr_shortages,	2288	nr_lagging, nr_shortages);
1616	nr_surpluses);
1617		2289
1618	atomic64_set(&ioc->vtime_rate, vrate);	2290	ioc->vtime_base_rate = vrate;
1619	ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(	2291	ioc_refresh_margins(ioc);
1620	ioc->period_us * vrate * INUSE_MARGIN_PCT, 100);
1621	} else if (ioc->busy_level != prev_busy_level \|\| nr_lagging) {	2292	} else if (ioc->busy_level != prev_busy_level \|\| nr_lagging) {
1622	trace_iocost_ioc_vrate_adj(ioc, atomic64_read(&ioc->vtime_rate),	2293	trace_iocost_ioc_vrate_adj(ioc, atomic64_read(&ioc->vtime_rate),
1623	missed_ppm, rq_wait_pct, nr_lagging,	2294	missed_ppm, rq_wait_pct, nr_lagging,
1624	nr_shortages, nr_surpluses);	2295	nr_shortages);
1625	}	2296	}
1626		2297
1627	ioc_refresh_params(ioc, false);	2298	ioc_refresh_params(ioc, false);
@@ -1637,11 +2308,74 @@ skip_surplus_transfers: · 9 unmodified lines
1637	ioc_start_period(ioc, &now);	2308	ioc_start_period(ioc, &now);
1638	} else {	2309	} else {
1639	ioc->busy_level = 0;	2310	ioc->busy_level = 0;
		2311	ioc->vtime_err = 0;
1640	ioc->running = IOC_IDLE;	2312	ioc->running = IOC_IDLE;
1641	}	2313	}
		2314
		2315	ioc_refresh_vrate(ioc, &now);
		2316	}
		2317
		2318	spin_unlock_irq(&ioc->lock);
		2319	}
		2320
		2321	static u64 adjust_inuse_and_calc_cost(struct ioc_gq *iocg, u64 vtime,
		2322	u64 abs_cost, struct ioc_now *now)
		2323	{
		2324	struct ioc *ioc = iocg->ioc;
		2325	struct ioc_margins *margins = &ioc->margins;
		2326	u32 __maybe_unused old_inuse = iocg->inuse, __maybe_unused old_hwi;
		2327	u32 hwi, adj_step;
		2328	s64 margin;
		2329	u64 cost, new_inuse;
		2330
		2331	current_hweight(iocg, NULL, &hwi);
		2332	old_hwi = hwi;
		2333	cost = abs_cost_to_cost(abs_cost, hwi);
		2334	margin = now->vnow - vtime - cost;
		2335
		2336	/* debt handling owns inuse for debtors */
		2337	if (iocg->abs_vdebt)
		2338	return cost;
		2339
		2340	/*
		2341	* We only increase inuse during period and do so iff the margin has
		2342	* deteriorated since the previous adjustment.
		2343	*/
		2344	if (margin >= iocg->saved_margin \|\| margin >= margins->low \|\|
		2345	iocg->inuse == iocg->active)
		2346	return cost;
		2347
		2348	spin_lock_irq(&ioc->lock);
		2349
		2350	/* we own inuse only when @iocg is in the normal active state */
		2351	if (iocg->abs_vdebt \|\| list_empty(&iocg->active_list)) {
		2352	spin_unlock_irq(&ioc->lock);
		2353	return cost;
1642	}	2354	}
1643		2355
		2356	/*
		2357	* Bump up inuse till @abs_cost fits in the existing budget.
		2358	* adj_step must be determined after acquiring ioc->lock - we might
		2359	* have raced and lost to another thread for activation and could
		2360	* be reading 0 iocg->active before ioc->lock which will lead to
		2361	* infinite loop.
		2362	*/
		2363	new_inuse = iocg->inuse;
		2364	adj_step = DIV_ROUND_UP(iocg->active * INUSE_ADJ_STEP_PCT, 100);
		2365	do {
		2366	new_inuse = new_inuse + adj_step;
		2367	propagate_weights(iocg, iocg->active, new_inuse, true, now);
		2368	current_hweight(iocg, NULL, &hwi);
		2369	cost = abs_cost_to_cost(abs_cost, hwi);
		2370	} while (time_after64(vtime + cost, now->vnow) &&
		2371	iocg->inuse != iocg->active);
		2372
1644	spin_unlock_irq(&ioc->lock);	2373	spin_unlock_irq(&ioc->lock);
		2374
		2375	TRACE_IOCG_PATH(inuse_adjust, iocg, now,
		2376	old_inuse, iocg->inuse, old_hwi, hwi);
		2377
		2378	return cost;
1645	}	2379	}
1646		2380
1647	static void calc_vtime_cost_builtin(struct bio bio, struct ioc_gq iocg,	2381	static void calc_vtime_cost_builtin(struct bio bio, struct ioc_gq iocg,
@@ -1725,38 +2459,25 @@ static void ioc_rqos_throttle(struct rq_qos rqos, struct bio bio) · 77 unmodified lines
1725	struct ioc_gq *iocg = blkg_to_iocg(blkg);	2459	struct ioc_gq *iocg = blkg_to_iocg(blkg);
1726	struct ioc_now now;	2460	struct ioc_now now;
1727	struct iocg_wait wait;	2461	struct iocg_wait wait;
1728	u32 hw_active, hw_inuse;
1729	u64 abs_cost, cost, vtime;	2462	u64 abs_cost, cost, vtime;
		2463	bool use_debt, ioc_locked;
		2464	unsigned long flags;
1730		2465
1731	/* bypass IOs if disabled or for root cgroup */	2466	/* bypass IOs if disabled or for root cgroup */
1732	if (!ioc->enabled \|\| !iocg->level)	2467	if (!ioc->enabled \|\| !iocg->level)
1733	return;	2468	return;
1734		2469
1735	/* always activate so that even 0 cost IOs get protected to some level */
1736	if (!iocg_activate(iocg, &now))
1737	return;
1738
1739	/* calculate the absolute vtime cost */	2470	/* calculate the absolute vtime cost */
1740	abs_cost = calc_vtime_cost(bio, iocg, false);	2471	abs_cost = calc_vtime_cost(bio, iocg, false);
1741	if (!abs_cost)	2472	if (!abs_cost)
1742	return;	2473	return;
1743		2474
1744	iocg->cursor = bio_end_sector(bio);	2475	if (!iocg_activate(iocg, &now))
		2476	return;
1745		2477
		2478	iocg->cursor = bio_end_sector(bio);
1746	vtime = atomic64_read(&iocg->vtime);	2479	vtime = atomic64_read(&iocg->vtime);
1747	current_hweight(iocg, &hw_active, &hw_inuse);	2480	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
1748
1749	if (hw_inuse < hw_active &&
1750	time_after_eq64(vtime + ioc->inuse_margin_vtime, now.vnow)) {
1751	TRACE_IOCG_PATH(inuse_reset, iocg, &now,
1752	iocg->inuse, iocg->weight, hw_inuse, hw_active);
1753	spin_lock_irq(&ioc->lock);
1754	propagate_active_weight(iocg, iocg->weight, iocg->weight);
1755	spin_unlock_irq(&ioc->lock);
1756	current_hweight(iocg, &hw_active, &hw_inuse);
1757	}
1758
1759	cost = abs_cost_to_cost(abs_cost, hw_inuse);
1760		2481
1761	/*	2482	/*
1762	* If no one's waiting and within budget, issue right away. The	2483	* If no one's waiting and within budget, issue right away. The
@@ -1765,21 +2486,32 @@ static void ioc_rqos_throttle(struct rq_qos rqos, struct bio bio) · 2 unmodified lines
1765	*/	2486	*/
1766	if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&	2487	if (!waitqueue_active(&iocg->waitq) && !iocg->abs_vdebt &&
1767	time_before_eq64(vtime + cost, now.vnow)) {	2488	time_before_eq64(vtime + cost, now.vnow)) {
1768	iocg_commit_bio(iocg, bio, cost);	2489	iocg_commit_bio(iocg, bio, abs_cost, cost);
1769	return;	2490	return;
1770	}	2491	}
1771		2492
1772	/*	2493	/*
1773	* We activated above but w/o any synchronization. Deactivation is	2494	* We're over budget. This can be handled in two ways. IOs which may
1774	* synchronized with waitq.lock and we won't get deactivated as long	2495	* cause priority inversions are punted to @ioc->aux_iocg and charged as
1775	* as we're waiting or has debt, so we're good if we're activated	2496	* debt. Otherwise, the issuer is blocked on @iocg->waitq. Debt handling
1776	* here. In the unlikely case that we aren't, just issue the IO.	2497	* requires @ioc->lock, waitq handling @iocg->waitq.lock. Determine
		2498	* whether debt handling is needed and acquire locks accordingly.
1777	*/	2499	*/
1778	spin_lock_irq(&iocg->waitq.lock);	2500	use_debt = bio_issue_as_root_blkg(bio) \|\| fatal_signal_pending(current);
		2501	ioc_locked = use_debt \|\| READ_ONCE(iocg->abs_vdebt);
		2502	retry_lock:
		2503	iocg_lock(iocg, ioc_locked, &flags);
1779		2504
		2505	/*
		2506	* @iocg must stay activated for debt and waitq handling. Deactivation
		2507	* is synchronized against both ioc->lock and waitq.lock and we won't
		2508	* get deactivated as long as we're waiting or has debt, so we're good
		2509	* if we're activated here. In the unlikely cases that we aren't, just
		2510	* issue the IO.
		2511	*/
1780	if (unlikely(list_empty(&iocg->active_list))) {	2512	if (unlikely(list_empty(&iocg->active_list))) {
1781	spin_unlock_irq(&iocg->waitq.lock);	2513	iocg_unlock(iocg, ioc_locked, &flags);
1782	iocg_commit_bio(iocg, bio, cost);	2514	iocg_commit_bio(iocg, bio, abs_cost, cost);
1783	return;	2515	return;
1784	}	2516	}
1785		2517
@@ -1800,15 +2532,26 @@ static void ioc_rqos_throttle(struct rq_qos rqos, struct bio bio) · 14 unmodified lines
1800	* clear them and leave @iocg inactive w/ dangling use_delay heavily	2532	* clear them and leave @iocg inactive w/ dangling use_delay heavily
1801	* penalizing the cgroup and its descendants.	2533	* penalizing the cgroup and its descendants.
1802	*/	2534	*/
1803	if (bio_issue_as_root_blkg(bio) \|\| fatal_signal_pending(current)) {	2535	if (use_debt) {
1804	iocg->abs_vdebt += abs_cost;	2536	iocg_incur_debt(iocg, abs_cost, &now);
1805	if (iocg_kick_delay(iocg, &now))	2537	if (iocg_kick_delay(iocg, &now))
1806	blkcg_schedule_throttle(rqos->q,	2538	blkcg_schedule_throttle(rqos->q,
1807	(bio->bi_opf & REQ_SWAP) == REQ_SWAP);	2539	(bio->bi_opf & REQ_SWAP) == REQ_SWAP);
1808	spin_unlock_irq(&iocg->waitq.lock);	2540	iocg_unlock(iocg, ioc_locked, &flags);
1809	return;	2541	return;
1810	}	2542	}
1811		2543
		2544	/* guarantee that iocgs w/ waiters have maximum inuse */
		2545	if (!iocg->abs_vdebt && iocg->inuse != iocg->active) {
		2546	if (!ioc_locked) {
		2547	iocg_unlock(iocg, false, &flags);
		2548	ioc_locked = true;
		2549	goto retry_lock;
		2550	}
		2551	propagate_weights(iocg, iocg->active, iocg->active, true,
		2552	&now);
		2553	}
		2554
1812	/*	2555	/*
1813	* Append self to the waitq and schedule the wakeup timer if we're	2556	* Append self to the waitq and schedule the wakeup timer if we're
1814	* the first waiter. The timer duration is calculated based on the	2557	* the first waiter. The timer duration is calculated based on the
@@ -1829,9 +2572,9 @@ static void ioc_rqos_throttle(struct rq_qos rqos, struct bio bio) · 14 unmodified lines
1829	wait.committed = false; /* will be set true by waker */	2572	wait.committed = false; /* will be set true by waker */
1830		2573
1831	__add_wait_queue_entry_tail(&iocg->waitq, &wait.wait);	2574	__add_wait_queue_entry_tail(&iocg->waitq, &wait.wait);
1832	iocg_kick_waitq(iocg, &now);	2575	iocg_kick_waitq(iocg, ioc_locked, &now);
1833		2576
1834	spin_unlock_irq(&iocg->waitq.lock);	2577	iocg_unlock(iocg, ioc_locked, &flags);
1835		2578
1836	while (true) {	2579	while (true) {
1837	set_current_state(TASK_UNINTERRUPTIBLE);	2580	set_current_state(TASK_UNINTERRUPTIBLE);
@@ -1851,8 +2594,7 @@ static void ioc_rqos_merge(struct rq_qos rqos, struct request rq, · 13 unmodified lines
1851	struct ioc *ioc = iocg->ioc;	2594	struct ioc *ioc = iocg->ioc;
1852	sector_t bio_end = bio_end_sector(bio);	2595	sector_t bio_end = bio_end_sector(bio);
1853	struct ioc_now now;	2596	struct ioc_now now;
1854	u32 hw_inuse;	2597	u64 vtime, abs_cost, cost;
1855	u64 abs_cost, cost;
1856	unsigned long flags;	2598	unsigned long flags;
1857		2599
1858	/* bypass if disabled or for root cgroup */	2600	/* bypass if disabled or for root cgroup */
@@ -1864,8 +2606,9 @@ static void ioc_rqos_merge(struct rq_qos rqos, struct request rq, · 5 unmodified lines
1864	return;	2606	return;
1865		2607
1866	ioc_now(ioc, &now);	2608	ioc_now(ioc, &now);
1867	current_hweight(iocg, NULL, &hw_inuse);	2609
1868	cost = abs_cost_to_cost(abs_cost, hw_inuse);	2610	vtime = atomic64_read(&iocg->vtime);
		2611	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
1869		2612
1870	/* update cursor if backmerging into the request at the cursor */	2613	/* update cursor if backmerging into the request at the cursor */
1871	if (blk_rq_pos(rq) < bio_end &&	2614	if (blk_rq_pos(rq) < bio_end &&
@@ -1878,7 +2621,7 @@ static void ioc_rqos_merge(struct rq_qos rqos, struct request rq, · 6 unmodified lines
1878	*/	2621	*/
1879	if (rq->bio && rq->bio->bi_iocost_cost &&	2622	if (rq->bio && rq->bio->bi_iocost_cost &&
1880	time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {	2623	time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {
1881	iocg_commit_bio(iocg, bio, cost);	2624	iocg_commit_bio(iocg, bio, abs_cost, cost);
1882	return;	2625	return;
1883	}	2626	}
1884		2627
@@ -1887,14 +2630,20 @@ static void ioc_rqos_merge(struct rq_qos rqos, struct request rq, · 2 unmodified lines
1887	* be for the vast majority of cases. See debt handling in	2630	* be for the vast majority of cases. See debt handling in
1888	* ioc_rqos_throttle() for details.	2631	* ioc_rqos_throttle() for details.
1889	*/	2632	*/
1890	spin_lock_irqsave(&iocg->waitq.lock, flags);	2633	spin_lock_irqsave(&ioc->lock, flags);
		2634	spin_lock(&iocg->waitq.lock);
		2635
1891	if (likely(!list_empty(&iocg->active_list))) {	2636	if (likely(!list_empty(&iocg->active_list))) {
1892	iocg->abs_vdebt += abs_cost;	2637	iocg_incur_debt(iocg, abs_cost, &now);
1893	iocg_kick_delay(iocg, &now);	2638	if (iocg_kick_delay(iocg, &now))
		2639	blkcg_schedule_throttle(rqos->q,
		2640	(bio->bi_opf & REQ_SWAP) == REQ_SWAP);
1894	} else {	2641	} else {
1895	iocg_commit_bio(iocg, bio, cost);	2642	iocg_commit_bio(iocg, bio, abs_cost, cost);
1896	}	2643	}
1897	spin_unlock_irqrestore(&iocg->waitq.lock, flags);	2644
		2645	spin_unlock(&iocg->waitq.lock);
		2646	spin_unlock_irqrestore(&ioc->lock, flags);
1898	}	2647	}
1899		2648
1900	static void ioc_rqos_done_bio(struct rq_qos rqos, struct bio bio)	2649	static void ioc_rqos_done_bio(struct rq_qos rqos, struct bio bio)
@@ -1908,6 +2657,7 @@ static void ioc_rqos_done_bio(struct rq_qos rqos, struct bio bio) · 7 unmodified lines
1908	static void ioc_rqos_done(struct rq_qos rqos, struct request rq)	2657	static void ioc_rqos_done(struct rq_qos rqos, struct request rq)
1909	{	2658	{
1910	struct ioc *ioc = rqos_to_ioc(rqos);	2659	struct ioc *ioc = rqos_to_ioc(rqos);
		2660	struct ioc_pcpu_stat *ccs;
1911	u64 on_q_ns, rq_wait_ns, size_nsec;	2661	u64 on_q_ns, rq_wait_ns, size_nsec;
1912	int pidx, rw;	2662	int pidx, rw;
1913		2663
@@ -1931,13 +2681,17 @@ static void ioc_rqos_done(struct rq_qos rqos, struct request rq) · 17 unmodified lines
1931	rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;	2681	rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;
1932	size_nsec = div64_u64(calc_size_vtime_cost(rq, ioc), VTIME_PER_NSEC);	2682	size_nsec = div64_u64(calc_size_vtime_cost(rq, ioc), VTIME_PER_NSEC);
1933		2683
		2684	ccs = get_cpu_ptr(ioc->pcpu_stat);
		2685
1934	if (on_q_ns <= size_nsec \|\|	2686	if (on_q_ns <= size_nsec \|\|
1935	on_q_ns - size_nsec <= ioc->params.qos[pidx] * NSEC_PER_USEC)	2687	on_q_ns - size_nsec <= ioc->params.qos[pidx] * NSEC_PER_USEC)
1936	this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);	2688	local_inc(&ccs->missed[rw].nr_met);
1937	else	2689	else
1938	this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);	2690	local_inc(&ccs->missed[rw].nr_missed);
		2691
		2692	local64_add(rq_wait_ns, &ccs->rq_wait_ns);
1939		2693
1940	this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);	2694	put_cpu_ptr(ccs);
1941	}	2695	}
1942		2696
1943	static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)	2697	static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)
@@ -1977,7 +2731,7 @@ static int blk_iocost_init(struct request_queue *q) · 33 unmodified lines
1977	{	2731	{
1978	struct ioc *ioc;	2732	struct ioc *ioc;
1979	struct rq_qos *rqos;	2733	struct rq_qos *rqos;
1980	int ret;	2734	int i, cpu, ret;
1981		2735
1982	ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);	2736	ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
1983	if (!ioc)	2737	if (!ioc)
@@ -1989,6 +2743,16 @@ static int blk_iocost_init(struct request_queue *q) · 5 unmodified lines
1989	return -ENOMEM;	2743	return -ENOMEM;
1990	}	2744	}
1991		2745
		2746	for_each_possible_cpu(cpu) {
		2747	struct ioc_pcpu_stat *ccs = per_cpu_ptr(ioc->pcpu_stat, cpu);
		2748
		2749	for (i = 0; i < ARRAY_SIZE(ccs->missed); i++) {
		2750	local_set(&ccs->missed[i].nr_met, 0);
		2751	local_set(&ccs->missed[i].nr_missed, 0);
		2752	}
		2753	local64_set(&ccs->rq_wait_ns, 0);
		2754	}
		2755
1992	rqos = &ioc->rqos;	2756	rqos = &ioc->rqos;
1993	rqos->id = RQ_QOS_COST;	2757	rqos->id = RQ_QOS_COST;
1994	rqos->ops = &ioc_rqos_ops;	2758	rqos->ops = &ioc_rqos_ops;
@@ -1999,6 +2763,7 @@ static int blk_iocost_init(struct request_queue *q) · 4 unmodified lines
1999	INIT_LIST_HEAD(&ioc->active_iocgs);	2763	INIT_LIST_HEAD(&ioc->active_iocgs);
2000		2764
2001	ioc->running = IOC_IDLE;	2765	ioc->running = IOC_IDLE;
		2766	ioc->vtime_base_rate = VTIME_PER_USEC;
2002	atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);	2767	atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);
2003	seqcount_spinlock_init(&ioc->period_seqcount, &ioc->lock);	2768	seqcount_spinlock_init(&ioc->period_seqcount, &ioc->lock);
2004	ioc->period_at = ktime_to_us(ktime_get());	2769	ioc->period_at = ktime_to_us(ktime_get());
@@ -2029,7 +2794,7 @@ static struct blkcg_policy_data *ioc_cpd_alloc(gfp_t gfp) · 24 unmodified lines
2029	if (!iocc)	2794	if (!iocc)
2030	return NULL;	2795	return NULL;
2031		2796
2032	iocc->dfl_weight = CGROUP_WEIGHT_DFL;	2797	iocc->dfl_weight = CGROUP_WEIGHT_DFL * WEIGHT_ONE;
2033	return &iocc->cpd;	2798	return &iocc->cpd;
2034	}	2799	}
2035		2800
@@ -2048,6 +2813,12 @@ static struct blkg_policy_data ioc_pd_alloc(gfp_t gfp, struct request_queue q, · 12 unmodified lines
2048	if (!iocg)	2813	if (!iocg)
2049	return NULL;	2814	return NULL;
2050		2815
		2816	iocg->pcpu_stat = alloc_percpu_gfp(struct iocg_pcpu_stat, gfp);
		2817	if (!iocg->pcpu_stat) {
		2818	kfree(iocg);
		2819	return NULL;
		2820	}
		2821
2051	return &iocg->pd;	2822	return &iocg->pd;
2052	}	2823	}
2053		2824
@@ -2067,14 +2838,14 @@ static void ioc_pd_init(struct blkg_policy_data *pd) · 13 unmodified lines
2067	atomic64_set(&iocg->done_vtime, now.vnow);	2838	atomic64_set(&iocg->done_vtime, now.vnow);
2068	atomic64_set(&iocg->active_period, atomic64_read(&ioc->cur_period));	2839	atomic64_set(&iocg->active_period, atomic64_read(&ioc->cur_period));
2069	INIT_LIST_HEAD(&iocg->active_list);	2840	INIT_LIST_HEAD(&iocg->active_list);
2070	iocg->hweight_active = HWEIGHT_WHOLE;	2841	INIT_LIST_HEAD(&iocg->walk_list);
2071	iocg->hweight_inuse = HWEIGHT_WHOLE;	2842	INIT_LIST_HEAD(&iocg->surplus_list);
		2843	iocg->hweight_active = WEIGHT_ONE;
		2844	iocg->hweight_inuse = WEIGHT_ONE;
2072		2845
2073	init_waitqueue_head(&iocg->waitq);	2846	init_waitqueue_head(&iocg->waitq);
2074	hrtimer_init(&iocg->waitq_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);	2847	hrtimer_init(&iocg->waitq_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2075	iocg->waitq_timer.function = iocg_waitq_timer_fn;	2848	iocg->waitq_timer.function = iocg_waitq_timer_fn;
2076	hrtimer_init(&iocg->delay_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2077	iocg->delay_timer.function = iocg_delay_timer_fn;
2078		2849
2079	iocg->level = blkg->blkcg->css.cgroup->level;	2850	iocg->level = blkg->blkcg->css.cgroup->level;
2080		2851
@@ -2084,7 +2855,7 @@ static void ioc_pd_init(struct blkg_policy_data *pd) · 3 unmodified lines
2084	}	2855	}
2085		2856
2086	spin_lock_irqsave(&ioc->lock, flags);	2857	spin_lock_irqsave(&ioc->lock, flags);
2087	weight_updated(iocg);	2858	weight_updated(iocg, &now);
2088	spin_unlock_irqrestore(&ioc->lock, flags);	2859	spin_unlock_irqrestore(&ioc->lock, flags);
2089	}	2860	}
2090		2861
@@ -2096,18 +2867,56 @@ static void ioc_pd_free(struct blkg_policy_data *pd) · 5 unmodified lines
2096		2867
2097	if (ioc) {	2868	if (ioc) {
2098	spin_lock_irqsave(&ioc->lock, flags);	2869	spin_lock_irqsave(&ioc->lock, flags);
		2870
2099	if (!list_empty(&iocg->active_list)) {	2871	if (!list_empty(&iocg->active_list)) {
2100	propagate_active_weight(iocg, 0, 0);	2872	struct ioc_now now;
		2873
		2874	ioc_now(ioc, &now);
		2875	propagate_weights(iocg, 0, 0, false, &now);
2101	list_del_init(&iocg->active_list);	2876	list_del_init(&iocg->active_list);
2102	}	2877	}
		2878
		2879	WARN_ON_ONCE(!list_empty(&iocg->walk_list));
		2880	WARN_ON_ONCE(!list_empty(&iocg->surplus_list));
		2881
2103	spin_unlock_irqrestore(&ioc->lock, flags);	2882	spin_unlock_irqrestore(&ioc->lock, flags);
2104		2883
2105	hrtimer_cancel(&iocg->waitq_timer);	2884	hrtimer_cancel(&iocg->waitq_timer);
2106	hrtimer_cancel(&iocg->delay_timer);
2107	}	2885	}
		2886	free_percpu(iocg->pcpu_stat);
2108	kfree(iocg);	2887	kfree(iocg);
2109	}	2888	}
2110		2889
		2890	static size_t ioc_pd_stat(struct blkg_policy_data pd, char buf, size_t size)
		2891	{
		2892	struct ioc_gq *iocg = pd_to_iocg(pd);
		2893	struct ioc *ioc = iocg->ioc;
		2894	size_t pos = 0;
		2895
		2896	if (!ioc->enabled)
		2897	return 0;
		2898
		2899	if (iocg->level == 0) {
		2900	unsigned vp10k = DIV64_U64_ROUND_CLOSEST(
		2901	ioc->vtime_base_rate * 10000,
		2902	VTIME_PER_USEC);
		2903	pos += scnprintf(buf + pos, size - pos, " cost.vrate=%u.%02u",
		2904	vp10k / 100, vp10k % 100);
		2905	}
		2906
		2907	pos += scnprintf(buf + pos, size - pos, " cost.usage=%llu",
		2908	iocg->last_stat.usage_us);
		2909
		2910	if (blkcg_debug_stats)
		2911	pos += scnprintf(buf + pos, size - pos,
		2912	" cost.wait=%llu cost.indebt=%llu cost.indelay=%llu",
		2913	iocg->last_stat.wait_us,
		2914	iocg->last_stat.indebt_us,
		2915	iocg->last_stat.indelay_us);
		2916
		2917	return pos;
		2918	}
		2919
2111	static u64 ioc_weight_prfill(struct seq_file sf, struct blkg_policy_data pd,	2920	static u64 ioc_weight_prfill(struct seq_file sf, struct blkg_policy_data pd,
2112	int off)	2921	int off)
2113	{	2922	{
@@ -2115,7 +2924,7 @@ static u64 ioc_weight_prfill(struct seq_file sf, struct blkg_policy_data pd, · 1 unmodified line
2115	struct ioc_gq *iocg = pd_to_iocg(pd);	2924	struct ioc_gq *iocg = pd_to_iocg(pd);
2116		2925
2117	if (dname && iocg->cfg_weight)	2926	if (dname && iocg->cfg_weight)
2118	seq_printf(sf, "%s %u\n", dname, iocg->cfg_weight);	2927	seq_printf(sf, "%s %u\n", dname, iocg->cfg_weight / WEIGHT_ONE);
2119	return 0;	2928	return 0;
2120	}	2929	}
2121		2930
@@ -2125,7 +2934,7 @@ static int ioc_weight_show(struct seq_file sf, void v) · 3 unmodified lines
2125	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));	2934	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
2126	struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);	2935	struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
2127		2936
2128	seq_printf(sf, "default %u\n", iocc->dfl_weight);	2937	seq_printf(sf, "default %u\n", iocc->dfl_weight / WEIGHT_ONE);
2129	blkcg_print_blkgs(sf, blkcg, ioc_weight_prfill,	2938	blkcg_print_blkgs(sf, blkcg, ioc_weight_prfill,
2130	&blkcg_policy_iocost, seq_cft(sf)->private, false);	2939	&blkcg_policy_iocost, seq_cft(sf)->private, false);
2131	return 0;	2940	return 0;
@@ -2137,6 +2946,7 @@ static ssize_t ioc_weight_write(struct kernfs_open_file of, char buf, · 5 unmodified lines
2137	struct blkcg *blkcg = css_to_blkcg(of_css(of));	2946	struct blkcg *blkcg = css_to_blkcg(of_css(of));
2138	struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);	2947	struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
2139	struct blkg_conf_ctx ctx;	2948	struct blkg_conf_ctx ctx;
		2949	struct ioc_now now;
2140	struct ioc_gq *iocg;	2950	struct ioc_gq *iocg;
2141	u32 v;	2951	u32 v;
2142	int ret;	2952	int ret;
@@ -2151,13 +2961,14 @@ static ssize_t ioc_weight_write(struct kernfs_open_file of, char buf, · 8 unmodified lines
2151	return -EINVAL;	2961	return -EINVAL;
2152		2962
2153	spin_lock(&blkcg->lock);	2963	spin_lock(&blkcg->lock);
2154	iocc->dfl_weight = v;	2964	iocc->dfl_weight = v * WEIGHT_ONE;
2155	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {	2965	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
2156	struct ioc_gq *iocg = blkg_to_iocg(blkg);	2966	struct ioc_gq *iocg = blkg_to_iocg(blkg);
2157		2967
2158	if (iocg) {	2968	if (iocg) {
2159	spin_lock_irq(&iocg->ioc->lock);	2969	spin_lock_irq(&iocg->ioc->lock);
2160	weight_updated(iocg);	2970	ioc_now(iocg->ioc, &now);
		2971	weight_updated(iocg, &now);
2161	spin_unlock_irq(&iocg->ioc->lock);	2972	spin_unlock_irq(&iocg->ioc->lock);
2162	}	2973	}
2163	}	2974	}
@@ -2182,8 +2993,9 @@ static ssize_t ioc_weight_write(struct kernfs_open_file of, char buf, · 18 unmodified lines
2182	}	2993	}
2183		2994
2184	spin_lock(&iocg->ioc->lock);	2995	spin_lock(&iocg->ioc->lock);
2185	iocg->cfg_weight = v;	2996	iocg->cfg_weight = v * WEIGHT_ONE;
2186	weight_updated(iocg);	2997	ioc_now(iocg->ioc, &now);
		2998	weight_updated(iocg, &now);
2187	spin_unlock(&iocg->ioc->lock);	2999	spin_unlock(&iocg->ioc->lock);
2188		3000
2189	blkg_conf_finish(&ctx);	3001	blkg_conf_finish(&ctx);
@@ -2521,6 +3333,7 @@ static struct blkcg_policy blkcg_policy_iocost = { · 331 unmodified lines
2521	.pd_alloc_fn = ioc_pd_alloc,	3333	.pd_alloc_fn = ioc_pd_alloc,
2522	.pd_init_fn = ioc_pd_init,	3334	.pd_init_fn = ioc_pd_init,
2523	.pd_free_fn = ioc_pd_free,	3335	.pd_free_fn = ioc_pd_free,
		3336	.pd_stat_fn = ioc_pd_stat,
2524	};	3337	};
2525		3338
2526	static int __init ioc_init(void)	3339	static int __init ioc_init(void)

block/blk-map.c+66−111

@@ -12,7 +12,8 @@ · 11 unmodified lines
12	#include "blk.h"	12	#include "blk.h"
13		13
14	struct bio_map_data {	14	struct bio_map_data {
15	int is_our_pages;	15	bool is_our_pages : 1;
		16	bool is_null_mapped : 1;
16	struct iov_iter iter;	17	struct iov_iter iter;
17	struct iovec iov[];	18	struct iovec iov[];
18	};	19	};
@@ -108,7 +109,7 @@ static int bio_uncopy_user(struct bio *bio) · 89 unmodified lines
108	struct bio_map_data *bmd = bio->bi_private;	109	struct bio_map_data *bmd = bio->bi_private;
109	int ret = 0;	110	int ret = 0;
110		111
111	if (!bio_flagged(bio, BIO_NULL_MAPPED)) {	112	if (!bmd->is_null_mapped) {
112	/*	113	/*
113	* if we're in a workqueue, the request is orphaned, so	114	* if we're in a workqueue, the request is orphaned, so
114	* don't copy into a random user address space, just free	115	* don't copy into a random user address space, just free
@@ -126,24 +127,12 @@ static int bio_uncopy_user(struct bio *bio) · 11 unmodified lines
126	return ret;	127	return ret;
127	}	128	}
128		129
129	/**	130	static int bio_copy_user_iov(struct request rq, struct rq_map_data map_data,
130	* bio_copy_user_iov - copy user data to bio	131	struct iov_iter *iter, gfp_t gfp_mask)
131	* @q: destination block queue
132	* @map_data: pointer to the rq_map_data holding pages (if necessary)
133	* @iter: iovec iterator
134	* @gfp_mask: memory allocation flags
135	*
136	* Prepares and returns a bio for indirect user io, bouncing data
137	* to/from kernel pages as necessary. Must be paired with
138	* call bio_uncopy_user() on io completion.
139	*/
140	static struct bio bio_copy_user_iov(struct request_queue q,
141	struct rq_map_data map_data, struct iov_iter iter,
142	gfp_t gfp_mask)
143	{	132	{
144	struct bio_map_data *bmd;	133	struct bio_map_data *bmd;
145	struct page *page;	134	struct page *page;
146	struct bio *bio;	135	struct bio bio, bounce_bio;
147	int i = 0, ret;	136	int i = 0, ret;
148	int nr_pages;	137	int nr_pages;
149	unsigned int len = iter->count;	138	unsigned int len = iter->count;
@@ -151,14 +140,15 @@ static struct bio bio_copy_user_iov(struct request_queue q, · 1 unmodified line
151		140
152	bmd = bio_alloc_map_data(iter, gfp_mask);	141	bmd = bio_alloc_map_data(iter, gfp_mask);
153	if (!bmd)	142	if (!bmd)
154	return ERR_PTR(-ENOMEM);	143	return -ENOMEM;
155		144
156	/*	145	/*
157	* We need to do a deep copy of the iov_iter including the iovecs.	146	* We need to do a deep copy of the iov_iter including the iovecs.
158	* The caller provided iov might point to an on-stack or otherwise	147	* The caller provided iov might point to an on-stack or otherwise
159	* shortlived one.	148	* shortlived one.
160	*/	149	*/
161	bmd->is_our_pages = map_data ? 0 : 1;	150	bmd->is_our_pages = !map_data;
		151	bmd->is_null_mapped = (map_data && map_data->null_mapped);
162		152
163	nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);	153	nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);
164	if (nr_pages > BIO_MAX_PAGES)	154	if (nr_pages > BIO_MAX_PAGES)
@@ -168,8 +158,7 @@ static struct bio bio_copy_user_iov(struct request_queue q, · 3 unmodified lines
168	bio = bio_kmalloc(gfp_mask, nr_pages);	158	bio = bio_kmalloc(gfp_mask, nr_pages);
169	if (!bio)	159	if (!bio)
170	goto out_bmd;	160	goto out_bmd;
171		161	bio->bi_opf \|= req_op(rq);
172	ret = 0;
173		162
174	if (map_data) {	163	if (map_data) {
175	nr_pages = 1 << map_data->page_order;	164	nr_pages = 1 << map_data->page_order;
@@ -186,7 +175,7 @@ static struct bio bio_copy_user_iov(struct request_queue q, · 10 unmodified lines
186	if (map_data) {	175	if (map_data) {
187	if (i == map_data->nr_entries * nr_pages) {	176	if (i == map_data->nr_entries * nr_pages) {
188	ret = -ENOMEM;	177	ret = -ENOMEM;
189	break;	178	goto cleanup;
190	}	179	}
191		180
192	page = map_data->pages[i / nr_pages];	181	page = map_data->pages[i / nr_pages];
@@ -194,14 +183,14 @@ static struct bio bio_copy_user_iov(struct request_queue q, · 1 unmodified line
194		183
195	i++;	184	i++;
196	} else {	185	} else {
197	page = alloc_page(q->bounce_gfp \| gfp_mask);	186	page = alloc_page(rq->q->bounce_gfp \| gfp_mask);
198	if (!page) {	187	if (!page) {
199	ret = -ENOMEM;	188	ret = -ENOMEM;
200	break;	189	goto cleanup;
201	}	190	}
202	}	191	}
203		192
204	if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) {	193	if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) {
205	if (!map_data)	194	if (!map_data)
206	__free_page(page);	195	__free_page(page);
207	break;	196	break;
@@ -211,9 +200,6 @@ static struct bio bio_copy_user_iov(struct request_queue q, · 3 unmodified lines
211	offset = 0;	200	offset = 0;
212	}	201	}
213		202
214	if (ret)
215	goto cleanup;
216
217	if (map_data)	203	if (map_data)
218	map_data->offset += bio->bi_iter.bi_size;	204	map_data->offset += bio->bi_iter.bi_size;
219		205
@@ -233,41 +219,42 @@ static struct bio bio_copy_user_iov(struct request_queue q, · 13 unmodified lines
233	}	219	}
234		220
235	bio->bi_private = bmd;	221	bio->bi_private = bmd;
236	if (map_data && map_data->null_mapped)	222
237	bio_set_flag(bio, BIO_NULL_MAPPED);	223	bounce_bio = bio;
238	return bio;	224	ret = blk_rq_append_bio(rq, &bounce_bio);
		225	if (ret)
		226	goto cleanup;
		227
		228	/*
		229	* We link the bounce buffer in and could have to traverse it later, so
		230	* we have to get a ref to prevent it from being freed
		231	*/
		232	bio_get(bounce_bio);
		233	return 0;
239	cleanup:	234	cleanup:
240	if (!map_data)	235	if (!map_data)
241	bio_free_pages(bio);	236	bio_free_pages(bio);
242	bio_put(bio);	237	bio_put(bio);
243	out_bmd:	238	out_bmd:
244	kfree(bmd);	239	kfree(bmd);
245	return ERR_PTR(ret);	240	return ret;
246	}	241	}
247		242
248	/**	243	static int bio_map_user_iov(struct request rq, struct iov_iter iter,
249	* bio_map_user_iov - map user iovec into bio	244	gfp_t gfp_mask)
250	* @q: the struct request_queue for the bio
251	* @iter: iovec iterator
252	* @gfp_mask: memory allocation flags
253	*
254	* Map the user space address into a bio suitable for io to a block
255	* device. Returns an error pointer in case of error.
256	*/
257	static struct bio bio_map_user_iov(struct request_queue q,
258	struct iov_iter *iter, gfp_t gfp_mask)
259	{	245	{
260	unsigned int max_sectors = queue_max_hw_sectors(q);	246	unsigned int max_sectors = queue_max_hw_sectors(rq->q);
261	int j;	247	struct bio bio, bounce_bio;
262	struct bio *bio;
263	int ret;	248	int ret;
		249	int j;
264		250
265	if (!iov_iter_count(iter))	251	if (!iov_iter_count(iter))
266	return ERR_PTR(-EINVAL);	252	return -EINVAL;
267		253
268	bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES));	254	bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES));
269	if (!bio)	255	if (!bio)
270	return ERR_PTR(-ENOMEM);	256	return -ENOMEM;
		257	bio->bi_opf \|= req_op(rq);
271		258
272	while (iov_iter_count(iter)) {	259	while (iov_iter_count(iter)) {
273	struct page **pages;	260	struct page **pages;
@@ -283,7 +270,7 @@ static struct bio bio_map_user_iov(struct request_queue q, · 9 unmodified lines
283		270
284	npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE);	271	npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE);
285		272
286	if (unlikely(offs & queue_dma_alignment(q))) {	273	if (unlikely(offs & queue_dma_alignment(rq->q))) {
287	ret = -EINVAL;	274	ret = -EINVAL;
288	j = 0;	275	j = 0;
289	} else {	276	} else {
@@ -295,7 +282,7 @@ static struct bio bio_map_user_iov(struct request_queue q, · 5 unmodified lines
295	if (n > bytes)	282	if (n > bytes)
296	n = bytes;	283	n = bytes;
297		284
298	if (!bio_add_hw_page(q, bio, page, n, offs,	285	if (!bio_add_hw_page(rq->q, bio, page, n, offs,
299	max_sectors, &same_page)) {	286	max_sectors, &same_page)) {
300	if (same_page)	287	if (same_page)
301	put_page(page);	288	put_page(page);
@@ -319,21 +306,31 @@ static struct bio bio_map_user_iov(struct request_queue q, · 17 unmodified lines
319	break;	306	break;
320	}	307	}
321		308
322	bio_set_flag(bio, BIO_USER_MAPPED);
323
324	/*	309	/*
325	* subtle -- if bio_map_user_iov() ended up bouncing a bio,	310	* Subtle: if we end up needing to bounce a bio, it would normally
326	* it would normally disappear when its bi_end_io is run.	311	* disappear when its bi_end_io is run. However, we need the original
327	* however, we need it for the unmap, so grab an extra	312	* bio for the unmap, so grab an extra reference to it
328	* reference to it
329	*/	313	*/
330	bio_get(bio);	314	bio_get(bio);
331	return bio;
332		315
		316	bounce_bio = bio;
		317	ret = blk_rq_append_bio(rq, &bounce_bio);
		318	if (ret)
		319	goto out_put_orig;
		320
		321	/*
		322	* We link the bounce buffer in and could have to traverse it
		323	* later, so we have to get a ref to prevent it from being freed
		324	*/
		325	bio_get(bounce_bio);
		326	return 0;
		327
		328	out_put_orig:
		329	bio_put(bio);
333	out_unmap:	330	out_unmap:
334	bio_release_pages(bio, false);	331	bio_release_pages(bio, false);
335	bio_put(bio);	332	bio_put(bio);
336	return ERR_PTR(ret);	333	return ret;
337	}	334	}
338		335
339	/**	336	/**
@@ -557,55 +554,6 @@ int blk_rq_append_bio(struct request rq, struct bio *bio) · 217 unmodified lines
557	}	554	}
558	EXPORT_SYMBOL(blk_rq_append_bio);	555	EXPORT_SYMBOL(blk_rq_append_bio);
559		556
560	static int __blk_rq_unmap_user(struct bio *bio)
561	{
562	int ret = 0;
563
564	if (bio) {
565	if (bio_flagged(bio, BIO_USER_MAPPED))
566	bio_unmap_user(bio);
567	else
568	ret = bio_uncopy_user(bio);
569	}
570
571	return ret;
572	}
573
574	static int __blk_rq_map_user_iov(struct request *rq,
575	struct rq_map_data map_data, struct iov_iter iter,
576	gfp_t gfp_mask, bool copy)
577	{
578	struct request_queue *q = rq->q;
579	struct bio bio, orig_bio;
580	int ret;
581
582	if (copy)
583	bio = bio_copy_user_iov(q, map_data, iter, gfp_mask);
584	else
585	bio = bio_map_user_iov(q, iter, gfp_mask);
586
587	if (IS_ERR(bio))
588	return PTR_ERR(bio);
589
590	bio->bi_opf &= ~REQ_OP_MASK;
591	bio->bi_opf \|= req_op(rq);
592
593	orig_bio = bio;
594
595	/*
596	* We link the bounce buffer in and could have to traverse it
597	* later so we have to get a ref to prevent it from being freed
598	*/
599	ret = blk_rq_append_bio(rq, &bio);
600	if (ret) {
601	__blk_rq_unmap_user(orig_bio);
602	return ret;
603	}
604	bio_get(bio);
605
606	return 0;
607	}
608
609	/**	557	/**
610	* blk_rq_map_user_iov - map user data to a request, for passthrough requests	558	* blk_rq_map_user_iov - map user data to a request, for passthrough requests
611	* @q: request queue where request should be inserted	559	* @q: request queue where request should be inserted
@@ -649,7 +597,10 @@ int blk_rq_map_user_iov(struct request_queue q, struct request rq, · 37 unmodified lines
649		597
650	i = *iter;	598	i = *iter;
651	do {	599	do {
652	ret =__blk_rq_map_user_iov(rq, map_data, &i, gfp_mask, copy);	600	if (copy)
		601	ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask);
		602	else
		603	ret = bio_map_user_iov(rq, &i, gfp_mask);
653	if (ret)	604	if (ret)
654	goto unmap_rq;	605	goto unmap_rq;
655	if (!bio)	606	if (!bio)
@@ -700,9 +651,13 @@ int blk_rq_unmap_user(struct bio *bio) · 44 unmodified lines
700	if (unlikely(bio_flagged(bio, BIO_BOUNCED)))	651	if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
701	mapped_bio = bio->bi_private;	652	mapped_bio = bio->bi_private;
702		653
703	ret2 = __blk_rq_unmap_user(mapped_bio);	654	if (bio->bi_private) {
704	if (ret2 && !ret)	655	ret2 = bio_uncopy_user(mapped_bio);
705	ret = ret2;	656	if (ret2 && !ret)
		657	ret = ret2;
		658	} else {
		659	bio_unmap_user(mapped_bio);
		660	}
706		661
707	mapped_bio = bio;	662	mapped_bio = bio;
708	bio = bio->bi_next;	663	bio = bio->bi_next;

block/blk-merge.c+201−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	#include <trace/events/block.h>	11	#include <trace/events/block.h>
12		12
13	#include "blk.h"	13	#include "blk.h"
		14	#include "blk-rq-qos.h"
14		15
15	static inline bool bio_will_gap(struct request_queue *q,	16	static inline bool bio_will_gap(struct request_queue *q,
16	struct request prev_rq, struct bio prev, struct bio *next)	17	struct request prev_rq, struct bio prev, struct bio *next)
@@ -895,3 +896,203 @@ enum elv_merge blk_try_merge(struct request rq, struct bio bio) · 878 unmodified lines
895	return ELEVATOR_FRONT_MERGE;	896	return ELEVATOR_FRONT_MERGE;
896	return ELEVATOR_NO_MERGE;	897	return ELEVATOR_NO_MERGE;
897	}	898	}
		899
		900	static void blk_account_io_merge_bio(struct request *req)
		901	{
		902	if (!blk_do_io_stat(req))
		903	return;
		904
		905	part_stat_lock();
		906	part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
		907	part_stat_unlock();
		908	}
		909
		910	enum bio_merge_status bio_attempt_back_merge(struct request *req,
		911	struct bio *bio,
		912	unsigned int nr_segs)
		913	{
		914	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
		915
		916	if (!ll_back_merge_fn(req, bio, nr_segs))
		917	return BIO_MERGE_FAILED;
		918
		919	trace_block_bio_backmerge(req->q, req, bio);
		920	rq_qos_merge(req->q, req, bio);
		921
		922	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
		923	blk_rq_set_mixed_merge(req);
		924
		925	req->biotail->bi_next = bio;
		926	req->biotail = bio;
		927	req->__data_len += bio->bi_iter.bi_size;
		928
		929	bio_crypt_free_ctx(bio);
		930
		931	blk_account_io_merge_bio(req);
		932	return BIO_MERGE_OK;
		933	}
		934
		935	enum bio_merge_status bio_attempt_front_merge(struct request *req,
		936	struct bio *bio,
		937	unsigned int nr_segs)
		938	{
		939	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
		940
		941	if (!ll_front_merge_fn(req, bio, nr_segs))
		942	return BIO_MERGE_FAILED;
		943
		944	trace_block_bio_frontmerge(req->q, req, bio);
		945	rq_qos_merge(req->q, req, bio);
		946
		947	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
		948	blk_rq_set_mixed_merge(req);
		949
		950	bio->bi_next = req->bio;
		951	req->bio = bio;
		952
		953	req->__sector = bio->bi_iter.bi_sector;
		954	req->__data_len += bio->bi_iter.bi_size;
		955
		956	bio_crypt_do_front_merge(req, bio);
		957
		958	blk_account_io_merge_bio(req);
		959	return BIO_MERGE_OK;
		960	}
		961
		962	enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
		963	struct request *req,
		964	struct bio *bio)
		965	{
		966	unsigned short segments = blk_rq_nr_discard_segments(req);
		967
		968	if (segments >= queue_max_discard_segments(q))
		969	goto no_merge;
		970	if (blk_rq_sectors(req) + bio_sectors(bio) >
		971	blk_rq_get_max_sectors(req, blk_rq_pos(req)))
		972	goto no_merge;
		973
		974	rq_qos_merge(q, req, bio);
		975
		976	req->biotail->bi_next = bio;
		977	req->biotail = bio;
		978	req->__data_len += bio->bi_iter.bi_size;
		979	req->nr_phys_segments = segments + 1;
		980
		981	blk_account_io_merge_bio(req);
		982	return BIO_MERGE_OK;
		983	no_merge:
		984	req_set_nomerge(q, req);
		985	return BIO_MERGE_FAILED;
		986	}
		987
		988	static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
		989	struct request *rq,
		990	struct bio *bio,
		991	unsigned int nr_segs,
		992	bool sched_allow_merge)
		993	{
		994	if (!blk_rq_merge_ok(rq, bio))
		995	return BIO_MERGE_NONE;
		996
		997	switch (blk_try_merge(rq, bio)) {
		998	case ELEVATOR_BACK_MERGE:
		999	if (!sched_allow_merge \|\| blk_mq_sched_allow_merge(q, rq, bio))
		1000	return bio_attempt_back_merge(rq, bio, nr_segs);
		1001	break;
		1002	case ELEVATOR_FRONT_MERGE:
		1003	if (!sched_allow_merge \|\| blk_mq_sched_allow_merge(q, rq, bio))
		1004	return bio_attempt_front_merge(rq, bio, nr_segs);
		1005	break;
		1006	case ELEVATOR_DISCARD_MERGE:
		1007	return bio_attempt_discard_merge(q, rq, bio);
		1008	default:
		1009	return BIO_MERGE_NONE;
		1010	}
		1011
		1012	return BIO_MERGE_FAILED;
		1013	}
		1014
		1015	/**
		1016	* blk_attempt_plug_merge - try to merge with %current's plugged list
		1017	* @q: request_queue new bio is being queued at
		1018	* @bio: new bio being queued
		1019	* @nr_segs: number of segments in @bio
		1020	* @same_queue_rq: pointer to &struct request that gets filled in when
		1021	* another request associated with @q is found on the plug list
		1022	* (optional, may be %NULL)
		1023	*
		1024	* Determine whether @bio being queued on @q can be merged with a request
		1025	* on %current's plugged list. Returns %true if merge was successful,
		1026	* otherwise %false.
		1027	*
		1028	* Plugging coalesces IOs from the same issuer for the same purpose without
		1029	* going through @q->queue_lock. As such it's more of an issuing mechanism
		1030	* than scheduling, and the request, while may have elvpriv data, is not
		1031	* added on the elevator at this point. In addition, we don't have
		1032	* reliable access to the elevator outside queue lock. Only check basic
		1033	* merging parameters without querying the elevator.
		1034	*
		1035	* Caller must ensure !blk_queue_nomerges(q) beforehand.
		1036	*/
		1037	bool blk_attempt_plug_merge(struct request_queue q, struct bio bio,
		1038	unsigned int nr_segs, struct request **same_queue_rq)
		1039	{
		1040	struct blk_plug *plug;
		1041	struct request *rq;
		1042	struct list_head *plug_list;
		1043
		1044	plug = blk_mq_plug(q, bio);
		1045	if (!plug)
		1046	return false;
		1047
		1048	plug_list = &plug->mq_list;
		1049
		1050	list_for_each_entry_reverse(rq, plug_list, queuelist) {
		1051	if (rq->q == q && same_queue_rq) {
		1052	/*
		1053	* Only blk-mq multiple hardware queues case checks the
		1054	* rq in the same queue, there should be only one such
		1055	* rq in a queue
		1056	**/
		1057	*same_queue_rq = rq;
		1058	}
		1059
		1060	if (rq->q != q)
		1061	continue;
		1062
		1063	if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
		1064	BIO_MERGE_OK)
		1065	return true;
		1066	}
		1067
		1068	return false;
		1069	}
		1070
		1071	/*
		1072	* Iterate list of requests and see if we can merge this bio with any
		1073	* of them.
		1074	*/
		1075	bool blk_bio_list_merge(struct request_queue q, struct list_head list,
		1076	struct bio *bio, unsigned int nr_segs)
		1077	{
		1078	struct request *rq;
		1079	int checked = 8;
		1080
		1081	list_for_each_entry_reverse(rq, list, queuelist) {
		1082	if (!checked--)
		1083	break;
		1084
		1085	switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) {
		1086	case BIO_MERGE_NONE:
		1087	continue;
		1088	case BIO_MERGE_OK:
		1089	return true;
		1090	case BIO_MERGE_FAILED:
		1091	return false;
		1092	}
		1093
		1094	}
		1095
		1096	return false;
		1097	}
		1098	EXPORT_SYMBOL_GPL(blk_bio_list_merge);

block/blk-mq-debugfs.c+6−5

 	QUEUE_FLAG_NAME(SAME_FORCE),
 	QUEUE_FLAG_NAME(DEAD),
 	QUEUE_FLAG_NAME(INIT_DONE),
+	QUEUE_FLAG_NAME(STABLE_WRITES),
 	QUEUE_FLAG_NAME(POLL),
 	QUEUE_FLAG_NAME(WC),
 	QUEUE_FLAG_NAME(FUA),
 #define HCTX_FLAG_NAME(name) [ilog2(BLK_MQ_F_##name)] = #name
 static const char *const hctx_flag_name[] = {
 	HCTX_FLAG_NAME(SHOULD_MERGE),
-	HCTX_FLAG_NAME(TAG_SHARED),
+	HCTX_FLAG_NAME(TAG_QUEUE_SHARED),
 	HCTX_FLAG_NAME(BLOCKING),
 	HCTX_FLAG_NAME(NO_SCHED),
 	HCTX_FLAG_NAME(STACKING),
 		   atomic_read(&tags->active_queues));
 	seq_puts(m, "\nbitmap_tags:\n");
-	sbitmap_queue_show(&tags->bitmap_tags, m);
+	sbitmap_queue_show(tags->bitmap_tags, m);
 	if (tags->nr_reserved_tags) {
 		seq_puts(m, "\nbreserved_tags:\n");
-		sbitmap_queue_show(&tags->breserved_tags, m);
+		sbitmap_queue_show(tags->breserved_tags, m);
 	}
 }
 	if (res)
 		goto out;
 	if (hctx->tags)
-		sbitmap_bitmap_show(&hctx->tags->bitmap_tags.sb, m);
+		sbitmap_bitmap_show(&hctx->tags->bitmap_tags->sb, m);
 	mutex_unlock(&q->sysfs_lock);
 out:
 	if (res)
 		goto out;
 	if (hctx->sched_tags)
-		sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags.sb, m);
+		sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags->sb, m);
 	mutex_unlock(&q->sysfs_lock);
 out:

block/blk-mq-sched.c+30−94

@@ -18,21 +18,6 @@ · 17 unmodified lines
18	#include "blk-mq-tag.h"	18	#include "blk-mq-tag.h"
19	#include "blk-wbt.h"	19	#include "blk-wbt.h"
20		20
21	void blk_mq_sched_free_hctx_data(struct request_queue *q,
22	void (exit)(struct blk_mq_hw_ctx ))
23	{
24	struct blk_mq_hw_ctx *hctx;
25	int i;
26
27	queue_for_each_hw_ctx(q, hctx, i) {
28	if (exit && hctx->sched_data)
29	exit(hctx);
30	kfree(hctx->sched_data);
31	hctx->sched_data = NULL;
32	}
33	}
34	EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);
35
36	void blk_mq_sched_assign_ioc(struct request *rq)	21	void blk_mq_sched_assign_ioc(struct request *rq)
37	{	22	{
38	struct request_queue *q = rq->q;	23	struct request_queue *q = rq->q;
@@ -368,7 +353,7 @@ bool blk_mq_sched_try_merge(struct request_queue q, struct bio bio, · 329 unmodified lines
368	case ELEVATOR_BACK_MERGE:	353	case ELEVATOR_BACK_MERGE:
369	if (!blk_mq_sched_allow_merge(q, rq, bio))	354	if (!blk_mq_sched_allow_merge(q, rq, bio))
370	return false;	355	return false;
371	if (!bio_attempt_back_merge(rq, bio, nr_segs))	356	if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
372	return false;	357	return false;
373	*merged_request = attempt_back_merge(q, rq);	358	*merged_request = attempt_back_merge(q, rq);
374	if (!*merged_request)	359	if (!*merged_request)
@@ -377,86 +362,20 @@ bool blk_mq_sched_try_merge(struct request_queue q, struct bio bio, · 2 unmodified lines
377	case ELEVATOR_FRONT_MERGE:	362	case ELEVATOR_FRONT_MERGE:
378	if (!blk_mq_sched_allow_merge(q, rq, bio))	363	if (!blk_mq_sched_allow_merge(q, rq, bio))
379	return false;	364	return false;
380	if (!bio_attempt_front_merge(rq, bio, nr_segs))	365	if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
381	return false;	366	return false;
382	*merged_request = attempt_front_merge(q, rq);	367	*merged_request = attempt_front_merge(q, rq);
383	if (!*merged_request)	368	if (!*merged_request)
384	elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);	369	elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
385	return true;	370	return true;
386	case ELEVATOR_DISCARD_MERGE:	371	case ELEVATOR_DISCARD_MERGE:
387	return bio_attempt_discard_merge(q, rq, bio);	372	return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
388	default:	373	default:
389	return false;	374	return false;
390	}	375	}
391	}	376	}
392	EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);	377	EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
393		378
394	/*
395	* Iterate list of requests and see if we can merge this bio with any
396	* of them.
397	*/
398	bool blk_mq_bio_list_merge(struct request_queue q, struct list_head list,
399	struct bio *bio, unsigned int nr_segs)
400	{
401	struct request *rq;
402	int checked = 8;
403
404	list_for_each_entry_reverse(rq, list, queuelist) {
405	bool merged = false;
406
407	if (!checked--)
408	break;
409
410	if (!blk_rq_merge_ok(rq, bio))
411	continue;
412
413	switch (blk_try_merge(rq, bio)) {
414	case ELEVATOR_BACK_MERGE:
415	if (blk_mq_sched_allow_merge(q, rq, bio))
416	merged = bio_attempt_back_merge(rq, bio,
417	nr_segs);
418	break;
419	case ELEVATOR_FRONT_MERGE:
420	if (blk_mq_sched_allow_merge(q, rq, bio))
421	merged = bio_attempt_front_merge(rq, bio,
422	nr_segs);
423	break;
424	case ELEVATOR_DISCARD_MERGE:
425	merged = bio_attempt_discard_merge(q, rq, bio);
426	break;
427	default:
428	continue;
429	}
430
431	return merged;
432	}
433
434	return false;
435	}
436	EXPORT_SYMBOL_GPL(blk_mq_bio_list_merge);
437
438	/*
439	* Reverse check our software queue for entries that we could potentially
440	* merge with. Currently includes a hand-wavy stop count of 8, to not spend
441	* too much time checking for merges.
442	*/
443	static bool blk_mq_attempt_merge(struct request_queue *q,
444	struct blk_mq_hw_ctx *hctx,
445	struct blk_mq_ctx ctx, struct bio bio,
446	unsigned int nr_segs)
447	{
448	enum hctx_type type = hctx->type;
449
450	lockdep_assert_held(&ctx->lock);
451
452	if (blk_mq_bio_list_merge(q, &ctx->rq_lists[type], bio, nr_segs)) {
453	ctx->rq_merged++;
454	return true;
455	}
456
457	return false;
458	}
459
460	bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio,	379	bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio,
461	unsigned int nr_segs)	380	unsigned int nr_segs)
462	{	381	{
@@ -470,14 +389,24 @@ bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio, · 7 unmodified lines
470	return e->type->ops.bio_merge(hctx, bio, nr_segs);	389	return e->type->ops.bio_merge(hctx, bio, nr_segs);
471		390
472	type = hctx->type;	391	type = hctx->type;
473	if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&	392	if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE) \|\|
474	!list_empty_careful(&ctx->rq_lists[type])) {	393	list_empty_careful(&ctx->rq_lists[type]))
475	/* default per sw-queue merge */	394	return false;
476	spin_lock(&ctx->lock);	395
477	ret = blk_mq_attempt_merge(q, hctx, ctx, bio, nr_segs);	396	/* default per sw-queue merge */
478	spin_unlock(&ctx->lock);	397	spin_lock(&ctx->lock);
		398	/*
		399	* Reverse check our software queue for entries that we could
		400	* potentially merge with. Currently includes a hand-wavy stop
		401	* count of 8, to not spend too much time checking for merges.
		402	*/
		403	if (blk_bio_list_merge(q, &ctx->rq_lists[type], bio, nr_segs)) {
		404	ctx->rq_merged++;
		405	ret = true;
479	}	406	}
480		407
		408	spin_unlock(&ctx->lock);
		409
481	return ret;	410	return ret;
482	}	411	}
483		412
@@ -531,7 +460,7 @@ void blk_mq_sched_insert_request(struct request *rq, bool at_head, · 47 unmodified lines
531	goto run;	460	goto run;
532	}	461	}
533		462
534	WARN_ON(e && (rq->tag != -1));	463	WARN_ON(e && (rq->tag != BLK_MQ_NO_TAG));
535		464
536	if (blk_mq_sched_bypass_insert(hctx, !!e, rq)) {	465	if (blk_mq_sched_bypass_insert(hctx, !!e, rq)) {
537	/*	466	/*
@@ -616,9 +545,11 @@ static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set, · 78 unmodified lines
616	struct blk_mq_hw_ctx *hctx,	545	struct blk_mq_hw_ctx *hctx,
617	unsigned int hctx_idx)	546	unsigned int hctx_idx)
618	{	547	{
		548	unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;
		549
619	if (hctx->sched_tags) {	550	if (hctx->sched_tags) {
620	blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);	551	blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
621	blk_mq_free_rq_map(hctx->sched_tags);	552	blk_mq_free_rq_map(hctx->sched_tags, flags);
622	hctx->sched_tags = NULL;	553	hctx->sched_tags = NULL;
623	}	554	}
624	}	555	}
@@ -628,10 +559,12 @@ static int blk_mq_sched_alloc_tags(struct request_queue *q, · 3 unmodified lines
628	unsigned int hctx_idx)	559	unsigned int hctx_idx)
629	{	560	{
630	struct blk_mq_tag_set *set = q->tag_set;	561	struct blk_mq_tag_set *set = q->tag_set;
		562	/* Clear HCTX_SHARED so tags are init'ed */
		563	unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;
631	int ret;	564	int ret;
632		565
633	hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,	566	hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
634	set->reserved_tags);	567	set->reserved_tags, flags);
635	if (!hctx->sched_tags)	568	if (!hctx->sched_tags)
636	return -ENOMEM;	569	return -ENOMEM;
637		570
@@ -649,8 +582,11 @@ static void blk_mq_sched_tags_teardown(struct request_queue *q) · 11 unmodified lines
649	int i;	582	int i;
650		583
651	queue_for_each_hw_ctx(q, hctx, i) {	584	queue_for_each_hw_ctx(q, hctx, i) {
		585	/* Clear HCTX_SHARED so tags are freed */
		586	unsigned int flags = hctx->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;
		587
652	if (hctx->sched_tags) {	588	if (hctx->sched_tags) {
653	blk_mq_free_rq_map(hctx->sched_tags);	589	blk_mq_free_rq_map(hctx->sched_tags, flags);
654	hctx->sched_tags = NULL;	590	hctx->sched_tags = NULL;
655	}	591	}
656	}	592	}

block/blk-mq-sched.h+0−3

@@ -5,9 +5,6 @@ · 4 unmodified lines
5	#include "blk-mq.h"	5	#include "blk-mq.h"
6	#include "blk-mq-tag.h"	6	#include "blk-mq-tag.h"
7		7
8	void blk_mq_sched_free_hctx_data(struct request_queue *q,
9	void (exit)(struct blk_mq_hw_ctx ));
10
11	void blk_mq_sched_assign_ioc(struct request *rq);	8	void blk_mq_sched_assign_ioc(struct request *rq);
12		9
13	void blk_mq_sched_request_inserted(struct request *rq);	10	void blk_mq_sched_request_inserted(struct request *rq);

block/blk-mq-tag.c+110−42

@@ -23,9 +23,18 @@ · 22 unmodified lines
23	*/	23	*/
24	bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)	24	bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
25	{	25	{
26	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&	26	if (blk_mq_is_sbitmap_shared(hctx->flags)) {
27	!test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))	27	struct request_queue *q = hctx->queue;
28	atomic_inc(&hctx->tags->active_queues);	28	struct blk_mq_tag_set *set = q->tag_set;
		29
		30	if (!test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags) &&
		31	!test_and_set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags))
		32	atomic_inc(&set->active_queues_shared_sbitmap);
		33	} else {
		34	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&
		35	!test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		36	atomic_inc(&hctx->tags->active_queues);
		37	}
29		38
30	return true;	39	return true;
31	}	40	}
@@ -35,9 +44,9 @@ bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) · 3 unmodified lines
35	*/	44	*/
36	void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)	45	void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
37	{	46	{
38	sbitmap_queue_wake_all(&tags->bitmap_tags);	47	sbitmap_queue_wake_all(tags->bitmap_tags);
39	if (include_reserve)	48	if (include_reserve)
40	sbitmap_queue_wake_all(&tags->breserved_tags);	49	sbitmap_queue_wake_all(tags->breserved_tags);
41	}	50	}
42		51
43	/*	52	/*
@@ -47,11 +56,19 @@ void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve) · 3 unmodified lines
47	void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)	56	void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
48	{	57	{
49	struct blk_mq_tags *tags = hctx->tags;	58	struct blk_mq_tags *tags = hctx->tags;
50		59	struct request_queue *q = hctx->queue;
51	if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))	60	struct blk_mq_tag_set *set = q->tag_set;
52	return;	61
53		62	if (blk_mq_is_sbitmap_shared(hctx->flags)) {
54	atomic_dec(&tags->active_queues);	63	if (!test_and_clear_bit(QUEUE_FLAG_HCTX_ACTIVE,
		64	&q->queue_flags))
		65	return;
		66	atomic_dec(&set->active_queues_shared_sbitmap);
		67	} else {
		68	if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		69	return;
		70	atomic_dec(&tags->active_queues);
		71	}
55		72
56	blk_mq_tag_wakeup_all(tags, false);	73	blk_mq_tag_wakeup_all(tags, false);
57	}	74	}
@@ -59,7 +76,8 @@ void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) · 1 unmodified line
59	static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,	76	static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
60	struct sbitmap_queue *bt)	77	struct sbitmap_queue *bt)
61	{	78	{
62	if (!data->q->elevator && !hctx_may_queue(data->hctx, bt))	79	if (!data->q->elevator && !(data->flags & BLK_MQ_REQ_RESERVED) &&
		80	!hctx_may_queue(data->hctx, bt))
63	return BLK_MQ_NO_TAG;	81	return BLK_MQ_NO_TAG;
64		82
65	if (data->shallow_depth)	83	if (data->shallow_depth)
@@ -82,10 +100,10 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) · 16 unmodified lines
82	WARN_ON_ONCE(1);	100	WARN_ON_ONCE(1);
83	return BLK_MQ_NO_TAG;	101	return BLK_MQ_NO_TAG;
84	}	102	}
85	bt = &tags->breserved_tags;	103	bt = tags->breserved_tags;
86	tag_offset = 0;	104	tag_offset = 0;
87	} else {	105	} else {
88	bt = &tags->bitmap_tags;	106	bt = tags->bitmap_tags;
89	tag_offset = tags->nr_reserved_tags;	107	tag_offset = tags->nr_reserved_tags;
90	}	108	}
91		109
@@ -131,9 +149,9 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) · 39 unmodified lines
131	data->ctx);	149	data->ctx);
132	tags = blk_mq_tags_from_data(data);	150	tags = blk_mq_tags_from_data(data);
133	if (data->flags & BLK_MQ_REQ_RESERVED)	151	if (data->flags & BLK_MQ_REQ_RESERVED)
134	bt = &tags->breserved_tags;	152	bt = tags->breserved_tags;
135	else	153	else
136	bt = &tags->bitmap_tags;	154	bt = tags->bitmap_tags;
137		155
138	/*	156	/*
139	* If destination hw queue is changed, fake wake up on	157	* If destination hw queue is changed, fake wake up on
@@ -167,10 +185,10 @@ void blk_mq_put_tag(struct blk_mq_tags tags, struct blk_mq_ctx ctx, · 27 unmodified lines
167	const int real_tag = tag - tags->nr_reserved_tags;	185	const int real_tag = tag - tags->nr_reserved_tags;
168		186
169	BUG_ON(real_tag >= tags->nr_tags);	187	BUG_ON(real_tag >= tags->nr_tags);
170	sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);	188	sbitmap_queue_clear(tags->bitmap_tags, real_tag, ctx->cpu);
171	} else {	189	} else {
172	BUG_ON(tag >= tags->nr_reserved_tags);	190	BUG_ON(tag >= tags->nr_reserved_tags);
173	sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);	191	sbitmap_queue_clear(tags->breserved_tags, tag, ctx->cpu);
174	}	192	}
175	}	193	}
176		194
@@ -197,7 +215,7 @@ static bool bt_iter(struct sbitmap bitmap, unsigned int bitnr, void data) · 20 unmodified lines
197	* We can hit rq == NULL here, because the tagging functions	215	* We can hit rq == NULL here, because the tagging functions
198	* test and set the bit before assigning ->rqs[].	216	* test and set the bit before assigning ->rqs[].
199	*/	217	*/
200	if (rq && rq->q == hctx->queue)	218	if (rq && rq->q == hctx->queue && rq->mq_hctx == hctx)
201	return iter_data->fn(hctx, rq, iter_data->data, reserved);	219	return iter_data->fn(hctx, rq, iter_data->data, reserved);
202	return true;	220	return true;
203	}	221	}
@@ -298,9 +316,9 @@ static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags, · 94 unmodified lines
298	WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED);	316	WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED);
299		317
300	if (tags->nr_reserved_tags)	318	if (tags->nr_reserved_tags)
301	bt_tags_for_each(tags, &tags->breserved_tags, fn, priv,	319	bt_tags_for_each(tags, tags->breserved_tags, fn, priv,
302	flags \| BT_TAG_ITER_RESERVED);	320	flags \| BT_TAG_ITER_RESERVED);
303	bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, flags);	321	bt_tags_for_each(tags, tags->bitmap_tags, fn, priv, flags);
304	}	322	}
305		323
306	/**	324	/**
@@ -416,8 +434,8 @@ void blk_mq_queue_tag_busy_iter(struct request_queue q, busy_iter_fn fn, · 109 unmodified lines
416	continue;	434	continue;
417		435
418	if (tags->nr_reserved_tags)	436	if (tags->nr_reserved_tags)
419	bt_for_each(hctx, &tags->breserved_tags, fn, priv, true);	437	bt_for_each(hctx, tags->breserved_tags, fn, priv, true);
420	bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false);	438	bt_for_each(hctx, tags->bitmap_tags, fn, priv, false);
421	}	439	}
422	blk_queue_exit(q);	440	blk_queue_exit(q);
423	}	441	}
@@ -429,30 +447,64 @@ static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth, · 5 unmodified lines
429	node);	447	node);
430	}	448	}
431		449
432	static struct blk_mq_tags blk_mq_init_bitmap_tags(struct blk_mq_tags tags,	450	static int blk_mq_init_bitmap_tags(struct blk_mq_tags *tags,
433	int node, int alloc_policy)	451	int node, int alloc_policy)
434	{	452	{
435	unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;	453	unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
436	bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;	454	bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
437		455
438	if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node))	456	if (bt_alloc(&tags->__bitmap_tags, depth, round_robin, node))
439	goto free_tags;	457	return -ENOMEM;
440	if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin,	458	if (bt_alloc(&tags->__breserved_tags, tags->nr_reserved_tags,
441	node))	459	round_robin, node))
442	goto free_bitmap_tags;	460	goto free_bitmap_tags;
443		461
444	return tags;	462	tags->bitmap_tags = &tags->__bitmap_tags;
		463	tags->breserved_tags = &tags->__breserved_tags;
		464
		465	return 0;
445	free_bitmap_tags:	466	free_bitmap_tags:
446	sbitmap_queue_free(&tags->bitmap_tags);	467	sbitmap_queue_free(&tags->__bitmap_tags);
447	free_tags:	468	return -ENOMEM;
448	kfree(tags);	469	}
449	return NULL;	470
		471	int blk_mq_init_shared_sbitmap(struct blk_mq_tag_set *set, unsigned int flags)
		472	{
		473	unsigned int depth = set->queue_depth - set->reserved_tags;
		474	int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags);
		475	bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
		476	int i, node = set->numa_node;
		477
		478	if (bt_alloc(&set->__bitmap_tags, depth, round_robin, node))
		479	return -ENOMEM;
		480	if (bt_alloc(&set->__breserved_tags, set->reserved_tags,
		481	round_robin, node))
		482	goto free_bitmap_tags;
		483
		484	for (i = 0; i < set->nr_hw_queues; i++) {
		485	struct blk_mq_tags *tags = set->tags[i];
		486
		487	tags->bitmap_tags = &set->__bitmap_tags;
		488	tags->breserved_tags = &set->__breserved_tags;
		489	}
		490
		491	return 0;
		492	free_bitmap_tags:
		493	sbitmap_queue_free(&set->__bitmap_tags);
		494	return -ENOMEM;
		495	}
		496
		497	void blk_mq_exit_shared_sbitmap(struct blk_mq_tag_set *set)
		498	{
		499	sbitmap_queue_free(&set->__bitmap_tags);
		500	sbitmap_queue_free(&set->__breserved_tags);
450	}	501	}
451		502
452	struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,	503	struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
453	unsigned int reserved_tags,	504	unsigned int reserved_tags,
454	int node, int alloc_policy)	505	int node, unsigned int flags)
455	{	506	{
		507	int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(flags);
456	struct blk_mq_tags *tags;	508	struct blk_mq_tags *tags;
457		509
458	if (total_tags > BLK_MQ_TAG_MAX) {	510	if (total_tags > BLK_MQ_TAG_MAX) {
@@ -467,13 +519,22 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, · 8 unmodified lines
467	tags->nr_tags = total_tags;	519	tags->nr_tags = total_tags;
468	tags->nr_reserved_tags = reserved_tags;	520	tags->nr_reserved_tags = reserved_tags;
469		521
470	return blk_mq_init_bitmap_tags(tags, node, alloc_policy);	522	if (flags & BLK_MQ_F_TAG_HCTX_SHARED)
		523	return tags;
		524
		525	if (blk_mq_init_bitmap_tags(tags, node, alloc_policy) < 0) {
		526	kfree(tags);
		527	return NULL;
		528	}
		529	return tags;
471	}	530	}
472		531
473	void blk_mq_free_tags(struct blk_mq_tags *tags)	532	void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags)
474	{	533	{
475	sbitmap_queue_free(&tags->bitmap_tags);	534	if (!(flags & BLK_MQ_F_TAG_HCTX_SHARED)) {
476	sbitmap_queue_free(&tags->breserved_tags);	535	sbitmap_queue_free(tags->bitmap_tags);
		536	sbitmap_queue_free(tags->breserved_tags);
		537	}
477	kfree(tags);	538	kfree(tags);
478	}	539	}
479		540
@@ -492,6 +553,8 @@ int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, · 12 unmodified lines
492	*/	553	*/
493	if (tdepth > tags->nr_tags) {	554	if (tdepth > tags->nr_tags) {
494	struct blk_mq_tag_set *set = hctx->queue->tag_set;	555	struct blk_mq_tag_set *set = hctx->queue->tag_set;
		556	/* Only sched tags can grow, so clear HCTX_SHARED flag */
		557	unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;
495	struct blk_mq_tags *new;	558	struct blk_mq_tags *new;
496	bool ret;	559	bool ret;
497		560
@@ -506,30 +569,35 @@ int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, · 8 unmodified lines
506	return -EINVAL;	569	return -EINVAL;
507		570
508	new = blk_mq_alloc_rq_map(set, hctx->queue_num, tdepth,	571	new = blk_mq_alloc_rq_map(set, hctx->queue_num, tdepth,
509	tags->nr_reserved_tags);	572	tags->nr_reserved_tags, flags);
510	if (!new)	573	if (!new)
511	return -ENOMEM;	574	return -ENOMEM;
512	ret = blk_mq_alloc_rqs(set, new, hctx->queue_num, tdepth);	575	ret = blk_mq_alloc_rqs(set, new, hctx->queue_num, tdepth);
513	if (ret) {	576	if (ret) {
514	blk_mq_free_rq_map(new);	577	blk_mq_free_rq_map(new, flags);
515	return -ENOMEM;	578	return -ENOMEM;
516	}	579	}
517		580
518	blk_mq_free_rqs(set, *tagsptr, hctx->queue_num);	581	blk_mq_free_rqs(set, *tagsptr, hctx->queue_num);
519	blk_mq_free_rq_map(*tagsptr);	582	blk_mq_free_rq_map(*tagsptr, flags);
520	*tagsptr = new;	583	*tagsptr = new;
521	} else {	584	} else {
522	/*	585	/*
523	* Don't need (or can't) update reserved tags here, they	586	* Don't need (or can't) update reserved tags here, they
524	* remain static and should never need resizing.	587	* remain static and should never need resizing.
525	*/	588	*/
526	sbitmap_queue_resize(&tags->bitmap_tags,	589	sbitmap_queue_resize(tags->bitmap_tags,
527	tdepth - tags->nr_reserved_tags);	590	tdepth - tags->nr_reserved_tags);
528	}	591	}
529		592
530	return 0;	593	return 0;
531	}	594	}
532		595
		596	void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set, unsigned int size)
		597	{
		598	sbitmap_queue_resize(&set->__bitmap_tags, size - set->reserved_tags);
		599	}
		600
533	/**	601	/**
534	* blk_mq_unique_tag() - return a tag that is unique queue-wide	602	* blk_mq_unique_tag() - return a tag that is unique queue-wide
535	* @rq: request for which to compute a unique tag	603	* @rq: request for which to compute a unique tag

block/blk-mq-tag.h+17−39

 #ifndef INT_BLK_MQ_TAG_H
 #define INT_BLK_MQ_TAG_H
-#include "blk-mq.h"
 /*
  * Tag address space map.
  */
 	atomic_t active_queues;
-	struct sbitmap_queue bitmap_tags;
+	struct sbitmap_queue *bitmap_tags;
-	struct sbitmap_queue breserved_tags;
+	struct sbitmap_queue *breserved_tags;
+	struct sbitmap_queue __bitmap_tags;
+	struct sbitmap_queue __breserved_tags;
 	struct request **rqs;
 	struct request **static_rqs;
 	struct list_head page_list;
 };
+extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags,
+					unsigned int reserved_tags,
+					int node, unsigned int flags);
+extern void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags);
-extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node, int alloc_policy);
+extern int blk_mq_init_shared_sbitmap(struct blk_mq_tag_set *set,
-extern void blk_mq_free_tags(struct blk_mq_tags *tags);
+				      unsigned int flags);
+extern void blk_mq_exit_shared_sbitmap(struct blk_mq_tag_set *set);
 extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);
 extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
 extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
 					struct blk_mq_tags **tags,
 					unsigned int depth, bool can_grow);
+extern void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set,
+					     unsigned int size);
 extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
 void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
 		void *priv);
 static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
 {
-	if (!(hctx->flags & BLK_MQ_F_TAG_SHARED))
+	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
 		return false;
 	return __blk_mq_tag_busy(hctx);
 static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
 {
-	if (!(hctx->flags & BLK_MQ_F_TAG_SHARED))
+	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
 		return;
 	__blk_mq_tag_idle(hctx);
 }
-/*
- * For shared tag users, we track the number of currently active users
- * and attempt to provide a fair share of the tag depth for each of them.
- */
-static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
-				  struct sbitmap_queue *bt)
-{
-	unsigned int depth, users;
-	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))
-		return true;
-	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
-		return true;
-	/*
-	 * Don't try dividing an ant
-	 */
-	if (bt->sb.depth == 1)
-		return true;
-	users = atomic_read(&hctx->tags->active_queues);
-	if (!users)
-		return true;
-	/*
-	 * Allow at least some tags
-	 */
-	depth = max((bt->sb.depth + users - 1) / users, 4U);
-	return atomic_read(&hctx->nr_active) < depth;
-}
 static inline bool blk_mq_tag_is_reserved(struct blk_mq_tags *tags,
 					  unsigned int tag)
 {

block/blk-mq.c+54−32

 	ctx->rq_completed[rq_is_sync(rq)]++;
 	if (rq->rq_flags & RQF_MQ_INFLIGHT)
-		atomic_dec(&hctx->nr_active);
+		__blk_mq_dec_active_requests(hctx);
 	if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
 		laptop_io_completion(q->backing_dev_info);
 static bool __blk_mq_get_driver_tag(struct request *rq)
 {
-	struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
+	struct sbitmap_queue *bt = rq->mq_hctx->tags->bitmap_tags;
 	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
 	int tag;
 	blk_mq_tag_busy(rq->mq_hctx);
 	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
-		bt = &rq->mq_hctx->tags->breserved_tags;
+		bt = rq->mq_hctx->tags->breserved_tags;
 		tag_offset = 0;
+	} else {
+		if (!hctx_may_queue(rq->mq_hctx, bt))
+			return false;
 	}
-	if (!hctx_may_queue(rq->mq_hctx, bt))
-		return false;
 	tag = __sbitmap_queue_get(bt);
 	if (tag == BLK_MQ_NO_TAG)
 		return false;
 	if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
 		return false;
-	if ((hctx->flags & BLK_MQ_F_TAG_SHARED) &&
+	if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
 			!(rq->rq_flags & RQF_MQ_INFLIGHT)) {
 		rq->rq_flags |= RQF_MQ_INFLIGHT;
-		atomic_inc(&hctx->nr_active);
+		__blk_mq_inc_active_requests(hctx);
 	}
 	hctx->tags->rqs[rq->tag] = rq;
 	return true;
 		struct sbitmap_queue *sbq;
 		list_del_init(&wait->entry);
-		sbq = &hctx->tags->bitmap_tags;
+		sbq = hctx->tags->bitmap_tags;
 		atomic_dec(&sbq->ws_active);
 	}
 	spin_unlock(&hctx->dispatch_wait_lock);
 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
 				 struct request *rq)
 {
-	struct sbitmap_queue *sbq = &hctx->tags->bitmap_tags;
+	struct sbitmap_queue *sbq = hctx->tags->bitmap_tags;
 	struct wait_queue_head *wq;
 	wait_queue_entry_t *wait;
 	bool ret;
-	if (!(hctx->flags & BLK_MQ_F_TAG_SHARED)) {
+	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
 		blk_mq_sched_mark_restart_hctx(hctx);
 		/*
 		bool needs_restart;
 		/* For non-shared tags, the RESTART check will suffice */
 		bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
-                        (hctx->flags & BLK_MQ_F_TAG_SHARED);
+			(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
 		bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
 		blk_mq_release_budgets(q, nr_budgets);
 	}
 }
-void blk_mq_free_rq_map(struct blk_mq_tags *tags)
+void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags)
 {
 	kfree(tags->rqs);
 	tags->rqs = NULL;
 	kfree(tags->static_rqs);
 	tags->static_rqs = NULL;
-	blk_mq_free_tags(tags);
+	blk_mq_free_tags(tags, flags);
 }
 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
 					unsigned int hctx_idx,
 					unsigned int nr_tags,
-					unsigned int reserved_tags)
+					unsigned int reserved_tags,
+					unsigned int flags)
 {
 	struct blk_mq_tags *tags;
 	int node;
 	if (node == NUMA_NO_NODE)
 		node = set->numa_node;
-	tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
+	tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags);
-				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
 	if (!tags)
 		return NULL;
 				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
 				 node);
 	if (!tags->rqs) {
-		blk_mq_free_tags(tags);
+		blk_mq_free_tags(tags, flags);
 		return NULL;
 	}
 					node);
 	if (!tags->static_rqs) {
 		kfree(tags->rqs);
-		blk_mq_free_tags(tags);
+		blk_mq_free_tags(tags, flags);
 		return NULL;
 	}
 		goto free_hctx;
 	atomic_set(&hctx->nr_active, 0);
+	atomic_set(&hctx->elevator_queued, 0);
 	if (node == NUMA_NO_NODE)
 		node = set->numa_node;
 	hctx->numa_node = node;
 	spin_lock_init(&hctx->lock);
 	INIT_LIST_HEAD(&hctx->dispatch);
 	hctx->queue = q;
-	hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
+	hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
 	INIT_LIST_HEAD(&hctx->hctx_list);
 static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set,
 					int hctx_idx)
 {
+	unsigned int flags = set->flags;
 	int ret = 0;
 	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
-					set->queue_depth, set->reserved_tags);
+					set->queue_depth, set->reserved_tags, flags);
 	if (!set->tags[hctx_idx])
 		return false;
 	if (!ret)
 		return true;
-	blk_mq_free_rq_map(set->tags[hctx_idx]);
+	blk_mq_free_rq_map(set->tags[hctx_idx], flags);
 	set->tags[hctx_idx] = NULL;
 	return false;
 }
 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
 					 unsigned int hctx_idx)
 {
+	unsigned int flags = set->flags;
 	if (set->tags && set->tags[hctx_idx]) {
 		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
-		blk_mq_free_rq_map(set->tags[hctx_idx]);
+		blk_mq_free_rq_map(set->tags[hctx_idx], flags);
 		set->tags[hctx_idx] = NULL;
 	}
 }
 	queue_for_each_hw_ctx(q, hctx, i) {
 		if (shared)
-			hctx->flags |= BLK_MQ_F_TAG_SHARED;
+			hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
 		else
-			hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
+			hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
 	}
 }
-static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set,
+static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
-					bool shared)
+					 bool shared)
 {
 	struct request_queue *q;
 	list_del(&q->tag_set_list);
 	if (list_is_singular(&set->tag_list)) {
 		/* just transitioned to unshared */
-		set->flags &= ~BLK_MQ_F_TAG_SHARED;
+		set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
 		/* update existing queue */
-		blk_mq_update_tag_set_depth(set, false);
+		blk_mq_update_tag_set_shared(set, false);
 	}
 	mutex_unlock(&set->tag_list_lock);
 	INIT_LIST_HEAD(&q->tag_set_list);
 	 * Check to see if we're transitioning to shared (from 1 to 2 queues).
 	 */
 	if (!list_empty(&set->tag_list) &&
-	    !(set->flags & BLK_MQ_F_TAG_SHARED)) {
+	    !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
-		set->flags |= BLK_MQ_F_TAG_SHARED;
+		set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
 		/* update existing queue */
-		blk_mq_update_tag_set_depth(set, true);
+		blk_mq_update_tag_set_shared(set, true);
 	}
-	if (set->flags & BLK_MQ_F_TAG_SHARED)
+	if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
 		queue_set_hctx_shared(q, true);
 	list_add_tail(&q->tag_set_list, &set->tag_list);
 	if (ret)
 		goto out_free_mq_map;
+	if (blk_mq_is_sbitmap_shared(set->flags)) {
+		atomic_set(&set->active_queues_shared_sbitmap, 0);
+		if (blk_mq_init_shared_sbitmap(set, set->flags)) {
+			ret = -ENOMEM;
+			goto out_free_mq_rq_maps;
+		}
+	}
 	mutex_init(&set->tag_list_lock);
 	INIT_LIST_HEAD(&set->tag_list);
 	return 0;
+out_free_mq_rq_maps:
+	for (i = 0; i < set->nr_hw_queues; i++)
+		blk_mq_free_map_and_requests(set, i);
 out_free_mq_map:
 	for (i = 0; i < set->nr_maps; i++) {
 		kfree(set->map[i].mq_map);
 	for (i = 0; i < set->nr_hw_queues; i++)
 		blk_mq_free_map_and_requests(set, i);
+	if (blk_mq_is_sbitmap_shared(set->flags))
+		blk_mq_exit_shared_sbitmap(set);
 	for (j = 0; j < set->nr_maps; j++) {
 		kfree(set->map[j].mq_map);
 		set->map[j].mq_map = NULL;
 		if (!hctx->sched_tags) {
 			ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
 							false);
+			if (!ret && blk_mq_is_sbitmap_shared(set->flags))
+				blk_mq_tag_resize_shared_sbitmap(set, nr);
 		} else {
 			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
 							nr, true);

block/blk-mq.h+73−3

@@ -53,11 +53,12 @@ struct request blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx hctx, · 52 unmodified lines
53	*/	53	*/
54	void blk_mq_free_rqs(struct blk_mq_tag_set set, struct blk_mq_tags tags,	54	void blk_mq_free_rqs(struct blk_mq_tag_set set, struct blk_mq_tags tags,
55	unsigned int hctx_idx);	55	unsigned int hctx_idx);
56	void blk_mq_free_rq_map(struct blk_mq_tags *tags);	56	void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags);
57	struct blk_mq_tags blk_mq_alloc_rq_map(struct blk_mq_tag_set set,	57	struct blk_mq_tags blk_mq_alloc_rq_map(struct blk_mq_tag_set set,
58	unsigned int hctx_idx,	58	unsigned int hctx_idx,
59	unsigned int nr_tags,	59	unsigned int nr_tags,
60	unsigned int reserved_tags);	60	unsigned int reserved_tags,
		61	unsigned int flags);
61	int blk_mq_alloc_rqs(struct blk_mq_tag_set set, struct blk_mq_tags tags,	62	int blk_mq_alloc_rqs(struct blk_mq_tag_set set, struct blk_mq_tags tags,
62	unsigned int hctx_idx, unsigned int depth);	63	unsigned int hctx_idx, unsigned int depth);
63		64
@@ -158,6 +159,11 @@ struct blk_mq_alloc_data { · 94 unmodified lines
158	struct blk_mq_hw_ctx *hctx;	159	struct blk_mq_hw_ctx *hctx;
159	};	160	};
160		161
		162	static inline bool blk_mq_is_sbitmap_shared(unsigned int flags)
		163	{
		164	return flags & BLK_MQ_F_TAG_HCTX_SHARED;
		165	}
		166
161	static inline struct blk_mq_tags blk_mq_tags_from_data(struct blk_mq_alloc_data data)	167	static inline struct blk_mq_tags blk_mq_tags_from_data(struct blk_mq_alloc_data data)
162	{	168	{
163	if (data->q->elevator)	169	if (data->q->elevator)
@@ -193,6 +199,28 @@ static inline bool blk_mq_get_dispatch_budget(struct request_queue *q) · 29 unmodified lines
193	return true;	199	return true;
194	}	200	}
195		201
		202	static inline void __blk_mq_inc_active_requests(struct blk_mq_hw_ctx *hctx)
		203	{
		204	if (blk_mq_is_sbitmap_shared(hctx->flags))
		205	atomic_inc(&hctx->queue->nr_active_requests_shared_sbitmap);
		206	else
		207	atomic_inc(&hctx->nr_active);
		208	}
		209
		210	static inline void __blk_mq_dec_active_requests(struct blk_mq_hw_ctx *hctx)
		211	{
		212	if (blk_mq_is_sbitmap_shared(hctx->flags))
		213	atomic_dec(&hctx->queue->nr_active_requests_shared_sbitmap);
		214	else
		215	atomic_dec(&hctx->nr_active);
		216	}
		217
		218	static inline int __blk_mq_active_requests(struct blk_mq_hw_ctx *hctx)
		219	{
		220	if (blk_mq_is_sbitmap_shared(hctx->flags))
		221	return atomic_read(&hctx->queue->nr_active_requests_shared_sbitmap);
		222	return atomic_read(&hctx->nr_active);
		223	}
196	static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,	224	static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
197	struct request *rq)	225	struct request *rq)
198	{	226	{
@@ -201,7 +229,7 @@ static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx, · 2 unmodified lines
201		229
202	if (rq->rq_flags & RQF_MQ_INFLIGHT) {	230	if (rq->rq_flags & RQF_MQ_INFLIGHT) {
203	rq->rq_flags &= ~RQF_MQ_INFLIGHT;	231	rq->rq_flags &= ~RQF_MQ_INFLIGHT;
204	atomic_dec(&hctx->nr_active);	232	__blk_mq_dec_active_requests(hctx);
205	}	233	}
206	}	234	}
207		235
@@ -253,4 +281,46 @@ static inline struct blk_plug blk_mq_plug(struct request_queue q, · 45 unmodified lines
253	return NULL;	281	return NULL;
254	}	282	}
255		283
		284	/*
		285	* For shared tag users, we track the number of currently active users
		286	* and attempt to provide a fair share of the tag depth for each of them.
		287	*/
		288	static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
		289	struct sbitmap_queue *bt)
		290	{
		291	unsigned int depth, users;
		292
		293	if (!hctx \|\| !(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
		294	return true;
		295
		296	/*
		297	* Don't try dividing an ant
		298	*/
		299	if (bt->sb.depth == 1)
		300	return true;
		301
		302	if (blk_mq_is_sbitmap_shared(hctx->flags)) {
		303	struct request_queue *q = hctx->queue;
		304	struct blk_mq_tag_set *set = q->tag_set;
		305
		306	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &q->queue_flags))
		307	return true;
		308	users = atomic_read(&set->active_queues_shared_sbitmap);
		309	} else {
		310	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		311	return true;
		312	users = atomic_read(&hctx->tags->active_queues);
		313	}
		314
		315	if (!users)
		316	return true;
		317
		318	/*
		319	* Allow at least some tags
		320	*/
		321	depth = max((bt->sb.depth + users - 1) / users, 4U);
		322	return __blk_mq_active_requests(hctx) < depth;
		323	}
		324
		325
256	#endif	326	#endif

block/blk-settings.c+28−12

  *
  * Description:
  *    If a driver doesn't want IOs to cross a given chunk size, it can set
- *    this limit and prevent merging across chunks. Note that the chunk size
+ *    this limit and prevent merging across chunks. Note that the block layer
- *    must currently be a power-of-2 in sectors. Also note that the block
+ *    must accept a page worth of data at any offset. So if the crossing of
- *    layer must accept a page worth of data at any offset. So if the
+ *    chunks is a hard limitation in the driver, it must still be prepared
- *    crossing of chunks is a hard limitation in the driver, it must still be
+ *    to split single page bios.
- *    prepared to split single page bios.
  **/
 void blk_queue_chunk_sectors(struct request_queue *q, unsigned int chunk_sectors)
 {
-	BUG_ON(!is_power_of_2(chunk_sectors));
 	q->limits.chunk_sectors = chunk_sectors;
 }
 EXPORT_SYMBOL(blk_queue_chunk_sectors);
 }
 EXPORT_SYMBOL(blk_queue_alignment_offset);
+void blk_queue_update_readahead(struct request_queue *q)
+{
+	/*
+	 * For read-ahead of large files to be effective, we need to read ahead
+	 * at least twice the optimal I/O size.
+	 */
+	q->backing_dev_info->ra_pages =
+		max(queue_io_opt(q) * 2 / PAGE_SIZE, VM_READAHEAD_PAGES);
+	q->backing_dev_info->io_pages =
+		queue_max_sectors(q) >> (PAGE_SHIFT - 9);
+}
+EXPORT_SYMBOL_GPL(blk_queue_update_readahead);
 /**
  * blk_limits_io_min - set minimum request size for a device
  * @limits: the queue limits
 void blk_queue_io_opt(struct request_queue *q, unsigned int opt)
 {
 	blk_limits_io_opt(&q->limits, opt);
+	q->backing_dev_info->ra_pages =
+		max(queue_io_opt(q) * 2 / PAGE_SIZE, VM_READAHEAD_PAGES);
 }
 EXPORT_SYMBOL(blk_queue_io_opt);
 	t->io_min = max(t->io_min, b->io_min);
 	t->io_opt = lcm_not_zero(t->io_opt, b->io_opt);
+	t->chunk_sectors = lcm_not_zero(t->chunk_sectors, b->chunk_sectors);
 	/* Physical block size a multiple of the logical block size? */
 	if (t->physical_block_size & (t->logical_block_size - 1)) {
 		ret = -1;
 	}
+	/* chunk_sectors a multiple of the physical block size? */
+	if ((t->chunk_sectors << 9) & (t->physical_block_size - 1)) {
+		t->chunk_sectors = 0;
+		t->misaligned = 1;
+		ret = -1;
+	}
 	t->raid_partial_stripes_expensive =
 		max(t->raid_partial_stripes_expensive,
 		    b->raid_partial_stripes_expensive);
 			t->discard_granularity;
 	}
-	if (b->chunk_sectors)
-		t->chunk_sectors = min_not_zero(t->chunk_sectors,
-						b->chunk_sectors);
 	t->zoned = max(t->zoned, b->zoned);
 	return ret;
 }
 		       top, bottom);
 	}
-	t->backing_dev_info->io_pages =
+	blk_queue_update_readahead(disk->queue);
-		t->limits.max_sectors >> (PAGE_SHIFT - 9);
 }
 EXPORT_SYMBOL(disk_stack_limits);

block/blk-sysfs.c+68−209

@@ -260,14 +260,14 @@ static ssize_t queue_max_hw_sectors_show(struct request_queue q, char page) · 259 unmodified lines
260		260
261	#define QUEUE_SYSFS_BIT_FNS(name, flag, neg) \	261	#define QUEUE_SYSFS_BIT_FNS(name, flag, neg) \
262	static ssize_t \	262	static ssize_t \
263	queue_show_##name(struct request_queue q, char page) \	263	queue_##name##_show(struct request_queue q, char page) \
264	{ \	264	{ \
265	int bit; \	265	int bit; \
266	bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags); \	266	bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags); \
267	return queue_var_show(neg ? !bit : bit, page); \	267	return queue_var_show(neg ? !bit : bit, page); \
268	} \	268	} \
269	static ssize_t \	269	static ssize_t \
270	queue_store_##name(struct request_queue q, const char page, size_t count) \	270	queue_##name##_store(struct request_queue q, const char page, size_t count) \
271	{ \	271	{ \
272	unsigned long val; \	272	unsigned long val; \
273	ssize_t ret; \	273	ssize_t ret; \
@@ -287,6 +287,7 @@ queue_store_##name(struct request_queue q, const char page, size_t count) \ · 13 unmodified lines
287	QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);	287	QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
288	QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);	288	QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
289	QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);	289	QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
		290	QUEUE_SYSFS_BIT_FNS(stable_writes, STABLE_WRITES, 0);
290	#undef QUEUE_SYSFS_BIT_FNS	291	#undef QUEUE_SYSFS_BIT_FNS
291		292
292	static ssize_t queue_zoned_show(struct request_queue q, char page)	293	static ssize_t queue_zoned_show(struct request_queue q, char page)
@@ -547,218 +548,73 @@ static ssize_t queue_dax_show(struct request_queue q, char page) · 254 unmodified lines
547	return queue_var_show(blk_queue_dax(q), page);	548	return queue_var_show(blk_queue_dax(q), page);
548	}	549	}
549		550
550	static struct queue_sysfs_entry queue_requests_entry = {	551	#define QUEUE_RO_ENTRY(_prefix, _name) \
551	.attr = {.name = "nr_requests", .mode = 0644 },	552	static struct queue_sysfs_entry _prefix##_entry = { \
552	.show = queue_requests_show,	553	.attr = { .name = _name, .mode = 0444 }, \
553	.store = queue_requests_store,	554	.show = _prefix##_show, \
554	};	555	};
555		556
556	static struct queue_sysfs_entry queue_ra_entry = {	557	#define QUEUE_RW_ENTRY(_prefix, _name) \
557	.attr = {.name = "read_ahead_kb", .mode = 0644 },	558	static struct queue_sysfs_entry _prefix##_entry = { \
558	.show = queue_ra_show,	559	.attr = { .name = _name, .mode = 0644 }, \
559	.store = queue_ra_store,	560	.show = _prefix##_show, \
560	};	561	.store = _prefix##_store, \
561		562	};
562	static struct queue_sysfs_entry queue_max_sectors_entry = {	563
563	.attr = {.name = "max_sectors_kb", .mode = 0644 },	564	QUEUE_RW_ENTRY(queue_requests, "nr_requests");
564	.show = queue_max_sectors_show,	565	QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
565	.store = queue_max_sectors_store,	566	QUEUE_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
566	};	567	QUEUE_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
567		568	QUEUE_RO_ENTRY(queue_max_segments, "max_segments");
568	static struct queue_sysfs_entry queue_max_hw_sectors_entry = {	569	QUEUE_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
569	.attr = {.name = "max_hw_sectors_kb", .mode = 0444 },	570	QUEUE_RO_ENTRY(queue_max_segment_size, "max_segment_size");
570	.show = queue_max_hw_sectors_show,	571	QUEUE_RW_ENTRY(elv_iosched, "scheduler");
571	};	572
572		573	QUEUE_RO_ENTRY(queue_logical_block_size, "logical_block_size");
573	static struct queue_sysfs_entry queue_max_segments_entry = {	574	QUEUE_RO_ENTRY(queue_physical_block_size, "physical_block_size");
574	.attr = {.name = "max_segments", .mode = 0444 },	575	QUEUE_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
575	.show = queue_max_segments_show,	576	QUEUE_RO_ENTRY(queue_io_min, "minimum_io_size");
576	};	577	QUEUE_RO_ENTRY(queue_io_opt, "optimal_io_size");
577		578
578	static struct queue_sysfs_entry queue_max_discard_segments_entry = {	579	QUEUE_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
579	.attr = {.name = "max_discard_segments", .mode = 0444 },	580	QUEUE_RO_ENTRY(queue_discard_granularity, "discard_granularity");
580	.show = queue_max_discard_segments_show,	581	QUEUE_RO_ENTRY(queue_discard_max_hw, "discard_max_hw_bytes");
581	};	582	QUEUE_RW_ENTRY(queue_discard_max, "discard_max_bytes");
582		583	QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");
583	static struct queue_sysfs_entry queue_max_integrity_segments_entry = {	584
584	.attr = {.name = "max_integrity_segments", .mode = 0444 },	585	QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
585	.show = queue_max_integrity_segments_show,	586	QUEUE_RO_ENTRY(queue_write_zeroes_max, "write_zeroes_max_bytes");
586	};	587	QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");
587		588
588	static struct queue_sysfs_entry queue_max_segment_size_entry = {	589	QUEUE_RO_ENTRY(queue_zoned, "zoned");
589	.attr = {.name = "max_segment_size", .mode = 0444 },	590	QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
590	.show = queue_max_segment_size_show,	591	QUEUE_RO_ENTRY(queue_max_open_zones, "max_open_zones");
591	};	592	QUEUE_RO_ENTRY(queue_max_active_zones, "max_active_zones");
		593
		594	QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
		595	QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
		596	QUEUE_RW_ENTRY(queue_poll, "io_poll");
		597	QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
		598	QUEUE_RW_ENTRY(queue_wc, "write_cache");
		599	QUEUE_RO_ENTRY(queue_fua, "fua");
		600	QUEUE_RO_ENTRY(queue_dax, "dax");
		601	QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
		602	QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
592		603
593	static struct queue_sysfs_entry queue_iosched_entry = {	604	#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
594	.attr = {.name = "scheduler", .mode = 0644 },	605	QUEUE_RW_ENTRY(blk_throtl_sample_time, "throttle_sample_time");
595	.show = elv_iosched_show,	606	#endif
596	.store = elv_iosched_store,
597	};
598		607
		608	/* legacy alias for logical_block_size: */
599	static struct queue_sysfs_entry queue_hw_sector_size_entry = {	609	static struct queue_sysfs_entry queue_hw_sector_size_entry = {
600	.attr = {.name = "hw_sector_size", .mode = 0444 },	610	.attr = {.name = "hw_sector_size", .mode = 0444 },
601	.show = queue_logical_block_size_show,	611	.show = queue_logical_block_size_show,
602	};	612	};
603		613
604	static struct queue_sysfs_entry queue_logical_block_size_entry = {	614	QUEUE_RW_ENTRY(queue_nonrot, "rotational");
605	.attr = {.name = "logical_block_size", .mode = 0444 },	615	QUEUE_RW_ENTRY(queue_iostats, "iostats");
606	.show = queue_logical_block_size_show,	616	QUEUE_RW_ENTRY(queue_random, "add_random");
607	};	617	QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");
608
609	static struct queue_sysfs_entry queue_physical_block_size_entry = {
610	.attr = {.name = "physical_block_size", .mode = 0444 },
611	.show = queue_physical_block_size_show,
612	};
613
614	static struct queue_sysfs_entry queue_chunk_sectors_entry = {
615	.attr = {.name = "chunk_sectors", .mode = 0444 },
616	.show = queue_chunk_sectors_show,
617	};
618
619	static struct queue_sysfs_entry queue_io_min_entry = {
620	.attr = {.name = "minimum_io_size", .mode = 0444 },
621	.show = queue_io_min_show,
622	};
623
624	static struct queue_sysfs_entry queue_io_opt_entry = {
625	.attr = {.name = "optimal_io_size", .mode = 0444 },
626	.show = queue_io_opt_show,
627	};
628
629	static struct queue_sysfs_entry queue_discard_granularity_entry = {
630	.attr = {.name = "discard_granularity", .mode = 0444 },
631	.show = queue_discard_granularity_show,
632	};
633
634	static struct queue_sysfs_entry queue_discard_max_hw_entry = {
635	.attr = {.name = "discard_max_hw_bytes", .mode = 0444 },
636	.show = queue_discard_max_hw_show,
637	};
638
639	static struct queue_sysfs_entry queue_discard_max_entry = {
640	.attr = {.name = "discard_max_bytes", .mode = 0644 },
641	.show = queue_discard_max_show,
642	.store = queue_discard_max_store,
643	};
644
645	static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
646	.attr = {.name = "discard_zeroes_data", .mode = 0444 },
647	.show = queue_discard_zeroes_data_show,
648	};
649
650	static struct queue_sysfs_entry queue_write_same_max_entry = {
651	.attr = {.name = "write_same_max_bytes", .mode = 0444 },
652	.show = queue_write_same_max_show,
653	};
654
655	static struct queue_sysfs_entry queue_write_zeroes_max_entry = {
656	.attr = {.name = "write_zeroes_max_bytes", .mode = 0444 },
657	.show = queue_write_zeroes_max_show,
658	};
659
660	static struct queue_sysfs_entry queue_zone_append_max_entry = {
661	.attr = {.name = "zone_append_max_bytes", .mode = 0444 },
662	.show = queue_zone_append_max_show,
663	};
664
665	static struct queue_sysfs_entry queue_nonrot_entry = {
666	.attr = {.name = "rotational", .mode = 0644 },
667	.show = queue_show_nonrot,
668	.store = queue_store_nonrot,
669	};
670
671	static struct queue_sysfs_entry queue_zoned_entry = {
672	.attr = {.name = "zoned", .mode = 0444 },
673	.show = queue_zoned_show,
674	};
675
676	static struct queue_sysfs_entry queue_nr_zones_entry = {
677	.attr = {.name = "nr_zones", .mode = 0444 },
678	.show = queue_nr_zones_show,
679	};
680
681	static struct queue_sysfs_entry queue_max_open_zones_entry = {
682	.attr = {.name = "max_open_zones", .mode = 0444 },
683	.show = queue_max_open_zones_show,
684	};
685
686	static struct queue_sysfs_entry queue_max_active_zones_entry = {
687	.attr = {.name = "max_active_zones", .mode = 0444 },
688	.show = queue_max_active_zones_show,
689	};
690
691	static struct queue_sysfs_entry queue_nomerges_entry = {
692	.attr = {.name = "nomerges", .mode = 0644 },
693	.show = queue_nomerges_show,
694	.store = queue_nomerges_store,
695	};
696
697	static struct queue_sysfs_entry queue_rq_affinity_entry = {
698	.attr = {.name = "rq_affinity", .mode = 0644 },
699	.show = queue_rq_affinity_show,
700	.store = queue_rq_affinity_store,
701	};
702
703	static struct queue_sysfs_entry queue_iostats_entry = {
704	.attr = {.name = "iostats", .mode = 0644 },
705	.show = queue_show_iostats,
706	.store = queue_store_iostats,
707	};
708
709	static struct queue_sysfs_entry queue_random_entry = {
710	.attr = {.name = "add_random", .mode = 0644 },
711	.show = queue_show_random,
712	.store = queue_store_random,
713	};
714
715	static struct queue_sysfs_entry queue_poll_entry = {
716	.attr = {.name = "io_poll", .mode = 0644 },
717	.show = queue_poll_show,
718	.store = queue_poll_store,
719	};
720
721	static struct queue_sysfs_entry queue_poll_delay_entry = {
722	.attr = {.name = "io_poll_delay", .mode = 0644 },
723	.show = queue_poll_delay_show,
724	.store = queue_poll_delay_store,
725	};
726
727	static struct queue_sysfs_entry queue_wc_entry = {
728	.attr = {.name = "write_cache", .mode = 0644 },
729	.show = queue_wc_show,
730	.store = queue_wc_store,
731	};
732
733	static struct queue_sysfs_entry queue_fua_entry = {
734	.attr = {.name = "fua", .mode = 0444 },
735	.show = queue_fua_show,
736	};
737
738	static struct queue_sysfs_entry queue_dax_entry = {
739	.attr = {.name = "dax", .mode = 0444 },
740	.show = queue_dax_show,
741	};
742
743	static struct queue_sysfs_entry queue_io_timeout_entry = {
744	.attr = {.name = "io_timeout", .mode = 0644 },
745	.show = queue_io_timeout_show,
746	.store = queue_io_timeout_store,
747	};
748
749	static struct queue_sysfs_entry queue_wb_lat_entry = {
750	.attr = {.name = "wbt_lat_usec", .mode = 0644 },
751	.show = queue_wb_lat_show,
752	.store = queue_wb_lat_store,
753	};
754
755	#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
756	static struct queue_sysfs_entry throtl_sample_time_entry = {
757	.attr = {.name = "throttle_sample_time", .mode = 0644 },
758	.show = blk_throtl_sample_time_show,
759	.store = blk_throtl_sample_time_store,
760	};
761	#endif
762		618
763	static struct attribute *queue_attrs[] = {	619	static struct attribute *queue_attrs[] = {
764	&queue_requests_entry.attr,	620	&queue_requests_entry.attr,
@@ -769,7 +625,7 @@ static struct attribute *queue_attrs[] = { · 4 unmodified lines
769	&queue_max_discard_segments_entry.attr,	625	&queue_max_discard_segments_entry.attr,
770	&queue_max_integrity_segments_entry.attr,	626	&queue_max_integrity_segments_entry.attr,
771	&queue_max_segment_size_entry.attr,	627	&queue_max_segment_size_entry.attr,
772	&queue_iosched_entry.attr,	628	&elv_iosched_entry.attr,
773	&queue_hw_sector_size_entry.attr,	629	&queue_hw_sector_size_entry.attr,
774	&queue_logical_block_size_entry.attr,	630	&queue_logical_block_size_entry.attr,
775	&queue_physical_block_size_entry.attr,	631	&queue_physical_block_size_entry.attr,
@@ -791,6 +647,7 @@ static struct attribute *queue_attrs[] = { · 15 unmodified lines
791	&queue_nomerges_entry.attr,	647	&queue_nomerges_entry.attr,
792	&queue_rq_affinity_entry.attr,	648	&queue_rq_affinity_entry.attr,
793	&queue_iostats_entry.attr,	649	&queue_iostats_entry.attr,
		650	&queue_stable_writes_entry.attr,
794	&queue_random_entry.attr,	651	&queue_random_entry.attr,
795	&queue_poll_entry.attr,	652	&queue_poll_entry.attr,
796	&queue_wc_entry.attr,	653	&queue_wc_entry.attr,
@@ -800,7 +657,7 @@ static struct attribute *queue_attrs[] = { · 3 unmodified lines
800	&queue_poll_delay_entry.attr,	657	&queue_poll_delay_entry.attr,
801	&queue_io_timeout_entry.attr,	658	&queue_io_timeout_entry.attr,
802	#ifdef CONFIG_BLK_DEV_THROTTLING_LOW	659	#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
803	&throtl_sample_time_entry.attr,	660	&blk_throtl_sample_time_entry.attr,
804	#endif	661	#endif
805	NULL,	662	NULL,
806	};	663	};
@@ -1000,6 +857,8 @@ int blk_register_queue(struct gendisk *disk) · 193 unmodified lines
1000	percpu_ref_switch_to_percpu(&q->q_usage_counter);	857	percpu_ref_switch_to_percpu(&q->q_usage_counter);
1001	}	858	}
1002		859
		860	blk_queue_update_readahead(q);
		861
1003	ret = blk_trace_init_sysfs(dev);	862	ret = blk_trace_init_sysfs(dev);
1004	if (ret)	863	if (ret)
1005	return ret;	864	return ret;

block/blk-throttle.c+36−23

 #include "blk-cgroup-rwstat.h"
 /* Max dispatch from a group in 1 round */
-static int throtl_grp_quantum = 8;
+#define THROTL_GRP_QUANTUM 8
 /* Total max dispatch from all groups in one round */
-static int throtl_quantum = 32;
+#define THROTL_QUANTUM 32
 /* Throttling is performed over a slice and after that slice is renewed */
 #define DFL_THROTL_SLICE_HD (HZ / 10)
 	/* user configured IOPS limits */
 	unsigned int iops_conf[2][LIMIT_CNT];
-	/* Number of bytes disptached in current slice */
+	/* Number of bytes dispatched in current slice */
 	uint64_t bytes_disp[2];
 	/* Number of bio's dispatched in current slice */
 	unsigned int io_disp[2];
 	/*
 	 * A bio has been dispatched. Also adjust slice_end. It might happen
 	 * that initially cgroup limit was very low resulting in high
-	 * slice_end, but later limit was bumped up and bio was dispached
+	 * slice_end, but later limit was bumped up and bio was dispatched
 	 * sooner, then we need to reduce slice_end. A high bogus slice_end
 	 * is bad because it does not allow new slice to start.
 	 */
 }
 static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,
-				  unsigned long *wait)
+				  u32 iops_limit, unsigned long *wait)
 {
 	bool rw = bio_data_dir(bio);
 	unsigned int io_allowed;
 	unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
 	u64 tmp;
+	if (iops_limit == UINT_MAX) {
+		if (wait)
+			*wait = 0;
+		return true;
+	}
 	jiffy_elapsed = jiffies - tg->slice_start[rw];
 	/* Round up to the next throttle slice, wait time must be nonzero */
 	 * have been trimmed.
 	 */
-	tmp = (u64)tg_iops_limit(tg, rw) * jiffy_elapsed_rnd;
+	tmp = (u64)iops_limit * jiffy_elapsed_rnd;
 	do_div(tmp, HZ);
 	if (tmp > UINT_MAX)
 }
 static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
-				 unsigned long *wait)
+				 u64 bps_limit, unsigned long *wait)
 {
 	bool rw = bio_data_dir(bio);
 	u64 bytes_allowed, extra_bytes, tmp;
 	unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
 	unsigned int bio_size = throtl_bio_data_size(bio);
+	if (bps_limit == U64_MAX) {
+		if (wait)
+			*wait = 0;
+		return true;
+	}
 	jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];
 	/* Slice has just started. Consider one slice interval */
 	jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice);
-	tmp = tg_bps_limit(tg, rw) * jiffy_elapsed_rnd;
+	tmp = bps_limit * jiffy_elapsed_rnd;
 	do_div(tmp, HZ);
 	bytes_allowed = tmp;
 	/* Calc approx time to dispatch */
 	extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed;
-	jiffy_wait = div64_u64(extra_bytes * HZ, tg_bps_limit(tg, rw));
+	jiffy_wait = div64_u64(extra_bytes * HZ, bps_limit);
 	if (!jiffy_wait)
 		jiffy_wait = 1;
 {
 	bool rw = bio_data_dir(bio);
 	unsigned long bps_wait = 0, iops_wait = 0, max_wait = 0;
+	u64 bps_limit = tg_bps_limit(tg, rw);
+	u32 iops_limit = tg_iops_limit(tg, rw);
 	/*
  	 * Currently whole state machine of group depends on first bio
 	       bio != throtl_peek_queued(&tg->service_queue.queued[rw]));
 	/* If tg->bps = -1, then BW is unlimited */
-	if (tg_bps_limit(tg, rw) == U64_MAX &&
+	if (bps_limit == U64_MAX && iops_limit == UINT_MAX) {
-	    tg_iops_limit(tg, rw) == UINT_MAX) {
 		if (wait)
 			*wait = 0;
 		return true;
 				jiffies + tg->td->throtl_slice);
 	}
-	if (tg_with_in_bps_limit(tg, bio, &bps_wait) &&
+	if (tg_with_in_bps_limit(tg, bio, bps_limit, &bps_wait) &&
-	    tg_with_in_iops_limit(tg, bio, &iops_wait)) {
+	    tg_with_in_iops_limit(tg, bio, iops_limit, &iops_wait)) {
 		if (wait)
 			*wait = 0;
 		return true;
 	 * If @tg doesn't currently have any bios queued in the same
 	 * direction, queueing @bio can change when @tg should be
 	 * dispatched.  Mark that @tg was empty.  This is automatically
-	 * cleaered on the next tg_update_disptime().
+	 * cleared on the next tg_update_disptime().
 	 */
 	if (!sq->nr_queued[rw])
 		tg->flags |= THROTL_TG_WAS_EMPTY;
 {
 	struct throtl_service_queue *sq = &tg->service_queue;
 	unsigned int nr_reads = 0, nr_writes = 0;
-	unsigned int max_nr_reads = throtl_grp_quantum*3/4;
+	unsigned int max_nr_reads = THROTL_GRP_QUANTUM * 3 / 4;
-	unsigned int max_nr_writes = throtl_grp_quantum - max_nr_reads;
+	unsigned int max_nr_writes = THROTL_GRP_QUANTUM - max_nr_reads;
 	struct bio *bio;
 	/* Try to dispatch 75% READS and 25% WRITES */
 		if (sq->nr_queued[0] || sq->nr_queued[1])
 			tg_update_disptime(tg);
-		if (nr_disp >= throtl_quantum)
+		if (nr_disp >= THROTL_QUANTUM)
 			break;
 	}
 			}
 		}
 	} else {
-		/* reached the top-level, queue issueing */
+		/* reached the top-level, queue issuing */
 		queue_work(kthrotld_workqueue, &td->dispatch_work);
 	}
 out_unlock:
  * blk_throtl_dispatch_work_fn - work function for throtl_data->dispatch_work
  * @work: work item being executed
  *
- * This function is queued for execution when bio's reach the bio_lists[]
+ * This function is queued for execution when bios reach the bio_lists[]
- * of throtl_data->service_queue.  Those bio's are ready and issued by this
+ * of throtl_data->service_queue.  Those bios are ready and issued by this
  * function.
  */
 static void blk_throtl_dispatch_work_fn(struct work_struct *work)
 	 * that a group's limit are dropped suddenly and we don't want to
 	 * account recently dispatched IO with new low rate.
 	 */
-	throtl_start_new_slice(tg, 0);
+	throtl_start_new_slice(tg, READ);
-	throtl_start_new_slice(tg, 1);
+	throtl_start_new_slice(tg, WRITE);
 	if (tg->flags & THROTL_TG_PENDING) {
 		tg_update_disptime(tg);
 		/*
 		 * @bio passed through this layer without being throttled.
-		 * Climb up the ladder.  If we''re already at the top, it
+		 * Climb up the ladder.  If we're already at the top, it
 		 * can be executed directly.
 		 */
 		qn = &tg->qnode_on_parent[rw];

block/blk.h+18−7

 	spinlock_t		mq_flush_lock;
 };
+enum bio_merge_status {
+	BIO_MERGE_OK,
+	BIO_MERGE_NONE,
+	BIO_MERGE_FAILED,
+};
 extern struct kmem_cache *blk_requestq_cachep;
 extern struct kobj_type blk_queue_ktype;
 extern struct ida blk_queue_ida;
 unsigned long blk_rq_timeout(unsigned long timeout);
 void blk_add_timer(struct request *req);
-bool bio_attempt_front_merge(struct request *req, struct bio *bio,
+enum bio_merge_status bio_attempt_front_merge(struct request *req,
-		unsigned int nr_segs);
+					      struct bio *bio,
-bool bio_attempt_back_merge(struct request *req, struct bio *bio,
+					      unsigned int nr_segs);
-		unsigned int nr_segs);
+enum bio_merge_status bio_attempt_back_merge(struct request *req,
-bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
+					     struct bio *bio,
-		struct bio *bio);
+					     unsigned int nr_segs);
+enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
+						struct request *req,
+						struct bio *bio);
 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
 		unsigned int nr_segs, struct request **same_queue_rq);
+bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
+			struct bio *bio, unsigned int nr_segs);
 void blk_account_io_start(struct request *req);
 void blk_account_io_done(struct request *req, u64 now);
 #define ADDPART_FLAG_NONE	0
 #define ADDPART_FLAG_RAID	1
 #define ADDPART_FLAG_WHOLEDISK	2
-void delete_partition(struct gendisk *disk, struct hd_struct *part);
+void delete_partition(struct hd_struct *part);
 int bdev_add_partition(struct block_device *bdev, int partno,
 		sector_t start, sector_t length);
 int bdev_del_partition(struct block_device *bdev, int partno);

block/bsg-lib.c+1−1

@@ -207,7 +207,7 @@ static int bsg_map_buffer(struct bsg_buffer buf, struct request req) · 206 unmodified lines
207		207
208	BUG_ON(!req->nr_phys_segments);	208	BUG_ON(!req->nr_phys_segments);
209		209
210	buf->sg_list = kzalloc(sz, GFP_KERNEL);	210	buf->sg_list = kmalloc(sz, GFP_KERNEL);
211	if (!buf->sg_list)	211	if (!buf->sg_list)
212	return -ENOMEM;	212	return -ENOMEM;
213	sg_init_table(buf->sg_list, req->nr_phys_segments);	213	sg_init_table(buf->sg_list, req->nr_phys_segments);

block/genhd.c+89−67

@@ -50,14 +50,13 @@ static void disk_release_events(struct gendisk *disk); · 49 unmodified lines
50	* zero and will not be set to zero	50	* zero and will not be set to zero
51	*/	51	*/
52	void set_capacity_revalidate_and_notify(struct gendisk *disk, sector_t size,	52	void set_capacity_revalidate_and_notify(struct gendisk *disk, sector_t size,
53	bool revalidate)	53	bool update_bdev)
54	{	54	{
55	sector_t capacity = get_capacity(disk);	55	sector_t capacity = get_capacity(disk);
56		56
57	set_capacity(disk, size);	57	set_capacity(disk, size);
58		58	if (update_bdev)
59	if (revalidate)	59	revalidate_disk_size(disk, true);
60	revalidate_disk(disk);
61		60
62	if (capacity != size && capacity != 0 && size != 0) {	61	if (capacity != size && capacity != 0 && size != 0) {
63	char *envp[] = { "RESIZE=1", NULL };	62	char *envp[] = { "RESIZE=1", NULL };
@@ -110,8 +109,7 @@ static void part_stat_read_all(struct hd_struct part, struct disk_stats stat) · 46 unmodified lines
110	}	109	}
111	}	110	}
112		111
113	static unsigned int part_in_flight(struct request_queue *q,	112	static unsigned int part_in_flight(struct hd_struct *part)
114	struct hd_struct *part)
115	{	113	{
116	unsigned int inflight = 0;	114	unsigned int inflight = 0;
117	int cpu;	115	int cpu;
@@ -126,8 +124,7 @@ static unsigned int part_in_flight(struct request_queue *q, · 8 unmodified lines
126	return inflight;	124	return inflight;
127	}	125	}
128		126
129	static void part_in_flight_rw(struct request_queue q, struct hd_struct part,	127	static void part_in_flight_rw(struct hd_struct *part, unsigned int inflight[2])
130	unsigned int inflight[2])
131	{	128	{
132	int cpu;	129	int cpu;
133		130
@@ -676,11 +673,23 @@ static int exact_lock(dev_t devt, void *data) · 542 unmodified lines
676	return 0;	673	return 0;
677	}	674	}
678		675
		676	static void disk_scan_partitions(struct gendisk *disk)
		677	{
		678	struct block_device *bdev;
		679
		680	if (!get_capacity(disk) \|\| !disk_part_scan_enabled(disk))
		681	return;
		682
		683	set_bit(GD_NEED_PART_SCAN, &disk->state);
		684	bdev = blkdev_get_by_dev(disk_devt(disk), FMODE_READ, NULL);
		685	if (!IS_ERR(bdev))
		686	blkdev_put(bdev, FMODE_READ);
		687	}
		688
679	static void register_disk(struct device parent, struct gendisk disk,	689	static void register_disk(struct device parent, struct gendisk disk,
680	const struct attribute_group **groups)	690	const struct attribute_group **groups)
681	{	691	{
682	struct device *ddev = disk_to_dev(disk);	692	struct device *ddev = disk_to_dev(disk);
683	struct block_device *bdev;
684	struct disk_part_iter piter;	693	struct disk_part_iter piter;
685	struct hd_struct *part;	694	struct hd_struct *part;
686	int err;	695	int err;
@@ -722,25 +731,8 @@ static void register_disk(struct device parent, struct gendisk disk, · 35 unmodified lines
722	return;	731	return;
723	}	732	}
724		733
725	/* No minors to use for partitions */	734	disk_scan_partitions(disk);
726	if (!disk_part_scan_enabled(disk))
727	goto exit;
728		735
729	/* No such device (e.g., media were just removed) */
730	if (!get_capacity(disk))
731	goto exit;
732
733	bdev = bdget_disk(disk, 0);
734	if (!bdev)
735	goto exit;
736
737	bdev->bd_invalidated = 1;
738	err = blkdev_get(bdev, FMODE_READ, NULL);
739	if (err < 0)
740	goto exit;
741	blkdev_put(bdev, FMODE_READ);
742
743	exit:
744	/* announce disk after possible partitions are created */	736	/* announce disk after possible partitions are created */
745	dev_set_uevent_suppress(ddev, 0);	737	dev_set_uevent_suppress(ddev, 0);
746	kobject_uevent(&ddev->kobj, KOBJ_ADD);	738	kobject_uevent(&ddev->kobj, KOBJ_ADD);
@@ -913,7 +905,7 @@ void del_gendisk(struct gendisk *disk) · 166 unmodified lines
913	DISK_PITER_INCL_EMPTY \| DISK_PITER_REVERSE);	905	DISK_PITER_INCL_EMPTY \| DISK_PITER_REVERSE);
914	while ((part = disk_part_iter_next(&piter))) {	906	while ((part = disk_part_iter_next(&piter))) {
915	invalidate_partition(disk, part->partno);	907	invalidate_partition(disk, part->partno);
916	delete_partition(disk, part);	908	delete_partition(part);
917	}	909	}
918	disk_part_iter_exit(&piter);	910	disk_part_iter_exit(&piter);
919		911
@@ -1301,7 +1293,7 @@ ssize_t part_stat_show(struct device *dev, · 381 unmodified lines
1301	if (queue_is_mq(q))	1293	if (queue_is_mq(q))
1302	inflight = blk_mq_in_flight(q, p);	1294	inflight = blk_mq_in_flight(q, p);
1303	else	1295	else
1304	inflight = part_in_flight(q, p);	1296	inflight = part_in_flight(p);
1305		1297
1306	return sprintf(buf,	1298	return sprintf(buf,
1307	"%8lu %8lu %8llu %8u "	1299	"%8lu %8lu %8llu %8u "
@@ -1343,7 +1335,7 @@ ssize_t part_inflight_show(struct device dev, struct device_attribute attr, · 35 unmodified lines
1343	if (queue_is_mq(q))	1335	if (queue_is_mq(q))
1344	blk_mq_in_flight_rw(q, p, inflight);	1336	blk_mq_in_flight_rw(q, p, inflight);
1345	else	1337	else
1346	part_in_flight_rw(q, p, inflight);	1338	part_in_flight_rw(p, inflight);
1347		1339
1348	return sprintf(buf, "%8u %8u\n", inflight[0], inflight[1]);	1340	return sprintf(buf, "%8u %8u\n", inflight[0], inflight[1]);
1349	}	1341	}
@@ -1623,7 +1615,7 @@ static int diskstats_show(struct seq_file seqf, void v) · 273 unmodified lines
1623	if (queue_is_mq(gp->queue))	1615	if (queue_is_mq(gp->queue))
1624	inflight = blk_mq_in_flight(gp->queue, hd);	1616	inflight = blk_mq_in_flight(gp->queue, hd);
1625	else	1617	else
1626	inflight = part_in_flight(gp->queue, hd);	1618	inflight = part_in_flight(hd);
1627		1619
1628	seq_printf(seqf, "%4d %7d %s "	1620	seq_printf(seqf, "%4d %7d %s "
1629	"%lu %lu %lu %u "	1621	"%lu %lu %lu %u "
@@ -1729,45 +1721,48 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) · 99 unmodified lines
1729	}	1721	}
1730		1722
1731	disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);	1723	disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
1732	if (disk) {	1724	if (!disk)
1733	disk->part0.dkstats = alloc_percpu(struct disk_stats);	1725	return NULL;
1734	if (!disk->part0.dkstats) {
1735	kfree(disk);
1736	return NULL;
1737	}
1738	init_rwsem(&disk->lookup_sem);
1739	disk->node_id = node_id;
1740	if (disk_expand_part_tbl(disk, 0)) {
1741	free_percpu(disk->part0.dkstats);
1742	kfree(disk);
1743	return NULL;
1744	}
1745	ptbl = rcu_dereference_protected(disk->part_tbl, 1);
1746	rcu_assign_pointer(ptbl->part[0], &disk->part0);
1747		1726
1748	/*	1727	disk->part0.dkstats = alloc_percpu(struct disk_stats);
1749	* set_capacity() and get_capacity() currently don't use	1728	if (!disk->part0.dkstats)
1750	* seqcounter to read/update the part0->nr_sects. Still init	1729	goto out_free_disk;
1751	* the counter as we can read the sectors in IO submission
1752	* patch using seqence counters.
1753	*
1754	* TODO: Ideally set_capacity() and get_capacity() should be
1755	* converted to make use of bd_mutex and sequence counters.
1756	*/
1757	hd_sects_seq_init(&disk->part0);
1758	if (hd_ref_init(&disk->part0)) {
1759	hd_free_part(&disk->part0);
1760	kfree(disk);
1761	return NULL;
1762	}
1763		1730
1764	disk->minors = minors;	1731	init_rwsem(&disk->lookup_sem);
1765	rand_initialize_disk(disk);	1732	disk->node_id = node_id;
1766	disk_to_dev(disk)->class = &block_class;	1733	if (disk_expand_part_tbl(disk, 0)) {
1767	disk_to_dev(disk)->type = &disk_type;	1734	free_percpu(disk->part0.dkstats);
1768	device_initialize(disk_to_dev(disk));	1735	goto out_free_disk;
1769	}	1736	}
		1737
		1738	ptbl = rcu_dereference_protected(disk->part_tbl, 1);
		1739	rcu_assign_pointer(ptbl->part[0], &disk->part0);
		1740
		1741	/*
		1742	* set_capacity() and get_capacity() currently don't use
		1743	* seqcounter to read/update the part0->nr_sects. Still init
		1744	* the counter as we can read the sectors in IO submission
		1745	* patch using seqence counters.
		1746	*
		1747	* TODO: Ideally set_capacity() and get_capacity() should be
		1748	* converted to make use of bd_mutex and sequence counters.
		1749	*/
		1750	hd_sects_seq_init(&disk->part0);
		1751	if (hd_ref_init(&disk->part0))
		1752	goto out_free_part0;
		1753
		1754	disk->minors = minors;
		1755	rand_initialize_disk(disk);
		1756	disk_to_dev(disk)->class = &block_class;
		1757	disk_to_dev(disk)->type = &disk_type;
		1758	device_initialize(disk_to_dev(disk));
1770	return disk;	1759	return disk;
		1760
		1761	out_free_part0:
		1762	hd_free_part(&disk->part0);
		1763	out_free_disk:
		1764	kfree(disk);
		1765	return NULL;
1771	}	1766	}
1772	EXPORT_SYMBOL(__alloc_disk_node);	1767	EXPORT_SYMBOL(__alloc_disk_node);
1773		1768
@@ -2052,7 +2047,7 @@ void disk_flush_events(struct gendisk *disk, unsigned int mask) · 278 unmodified lines
2052	* CONTEXT:	2047	* CONTEXT:
2053	* Might sleep.	2048	* Might sleep.
2054	*/	2049	*/
2055	unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask)	2050	static unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask)
2056	{	2051	{
2057	struct disk_events *ev = disk->ev;	2052	struct disk_events *ev = disk->ev;
2058	unsigned int pending;	2053	unsigned int pending;
@@ -2090,6 +2085,33 @@ unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask) · 31 unmodified lines
2090	return pending;	2085	return pending;
2091	}	2086	}
2092		2087
		2088	/**
		2089	* bdev_check_media_change - check if a removable media has been changed
		2090	* @bdev: block device to check
		2091	*
		2092	* Check whether a removable media has been changed, and attempt to free all
		2093	* dentries and inodes and invalidates all block device page cache entries in
		2094	* that case.
		2095	*
		2096	* Returns %true if the block device changed, or %false if not.
		2097	*/
		2098	bool bdev_check_media_change(struct block_device *bdev)
		2099	{
		2100	unsigned int events;
		2101
		2102	events = disk_clear_events(bdev->bd_disk, DISK_EVENT_MEDIA_CHANGE \|
		2103	DISK_EVENT_EJECT_REQUEST);
		2104	if (!(events & DISK_EVENT_MEDIA_CHANGE))
		2105	return false;
		2106
		2107	if (__invalidate_device(bdev, true))
		2108	pr_warn("VFS: busy inodes on changed media %s\n",
		2109	bdev->bd_disk->disk_name);
		2110	set_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state);
		2111	return true;
		2112	}
		2113	EXPORT_SYMBOL(bdev_check_media_change);
		2114
2093	/*	2115	/*
2094	* Separate this part out so that a different pointer for clearing_ptr can be	2116	* Separate this part out so that a different pointer for clearing_ptr can be
2095	* passed in for disk_clear_events.	2117	* passed in for disk_clear_events.

block/ioctl.c+16−13

@@ -112,8 +112,7 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, · 111 unmodified lines
112	uint64_t range[2];	112	uint64_t range[2];
113	uint64_t start, len;	113	uint64_t start, len;
114	struct request_queue *q = bdev_get_queue(bdev);	114	struct request_queue *q = bdev_get_queue(bdev);
115	struct address_space *mapping = bdev->bd_inode->i_mapping;	115	int err;
116
117		116
118	if (!(mode & FMODE_WRITE))	117	if (!(mode & FMODE_WRITE))
119	return -EBADF;	118	return -EBADF;
@@ -134,7 +133,11 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, · 14 unmodified lines
134		133
135	if (start + len > i_size_read(bdev->bd_inode))	134	if (start + len > i_size_read(bdev->bd_inode))
136	return -EINVAL;	135	return -EINVAL;
137	truncate_inode_pages_range(mapping, start, start + len - 1);	136
		137	err = truncate_bdev_range(bdev, mode, start, start + len - 1);
		138	if (err)
		139	return err;
		140
138	return blkdev_issue_discard(bdev, start >> 9, len >> 9,	141	return blkdev_issue_discard(bdev, start >> 9, len >> 9,
139	GFP_KERNEL, flags);	142	GFP_KERNEL, flags);
140	}	143	}
@@ -143,8 +146,8 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode, · 2 unmodified lines
143	unsigned long arg)	146	unsigned long arg)
144	{	147	{
145	uint64_t range[2];	148	uint64_t range[2];
146	struct address_space *mapping;
147	uint64_t start, end, len;	149	uint64_t start, end, len;
		150	int err;
148		151
149	if (!(mode & FMODE_WRITE))	152	if (!(mode & FMODE_WRITE))
150	return -EBADF;	153	return -EBADF;
@@ -166,8 +169,9 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode, · 15 unmodified lines
166	return -EINVAL;	169	return -EINVAL;
167		170
168	/* Invalidate the page cache, including dirty pages */	171	/* Invalidate the page cache, including dirty pages */
169	mapping = bdev->bd_inode->i_mapping;	172	err = truncate_bdev_range(bdev, mode, start, end);
170	truncate_inode_pages_range(mapping, start, end);	173	if (err)
		174	return err;
171		175
172	return blkdev_issue_zeroout(bdev, start >> 9, len >> 9, GFP_KERNEL,	176	return blkdev_issue_zeroout(bdev, start >> 9, len >> 9, GFP_KERNEL,
173	BLKDEV_ZERO_NOUNMAP);	177	BLKDEV_ZERO_NOUNMAP);
@@ -474,15 +478,14 @@ static int blkdev_bszset(struct block_device *bdev, fmode_t mode, · 300 unmodified lines
474	if (get_user(n, argp))	478	if (get_user(n, argp))
475	return -EFAULT;	479	return -EFAULT;
476		480
477	if (!(mode & FMODE_EXCL)) {	481	if (mode & FMODE_EXCL)
478	bdgrab(bdev);	482	return set_blocksize(bdev, n);
479	if (blkdev_get(bdev, mode \| FMODE_EXCL, &bdev) < 0)
480	return -EBUSY;
481	}
482		483
		484	if (IS_ERR(blkdev_get_by_dev(bdev->bd_dev, mode \| FMODE_EXCL, &bdev)))
		485	return -EBUSY;
483	ret = set_blocksize(bdev, n);	486	ret = set_blocksize(bdev, n);
484	if (!(mode & FMODE_EXCL))	487	blkdev_put(bdev, mode \| FMODE_EXCL);
485	blkdev_put(bdev, mode \| FMODE_EXCL);	488
486	return ret;	489	return ret;
487	}	490	}
488		491

block/ioprio.c+1−1

@@ -69,7 +69,7 @@ int ioprio_check_cap(int ioprio) · 68 unmodified lines
69		69
70	switch (class) {	70	switch (class) {
71	case IOPRIO_CLASS_RT:	71	case IOPRIO_CLASS_RT:
72	if (!capable(CAP_SYS_ADMIN))	72	if (!capable(CAP_SYS_NICE) && !capable(CAP_SYS_ADMIN))
73	return -EPERM;	73	return -EPERM;
74	fallthrough;	74	fallthrough;
75	/* rt has prio field too */	75	/* rt has prio field too */

block/kyber-iosched.c+3−3

@@ -359,7 +359,7 @@ static unsigned int kyber_sched_tags_shift(struct request_queue *q) · 358 unmodified lines
359	* All of the hardware queues have the same depth, so we can just grab	359	* All of the hardware queues have the same depth, so we can just grab
360	* the shift of the first one.	360	* the shift of the first one.
361	*/	361	*/
362	return q->queue_hw_ctx[0]->sched_tags->bitmap_tags.sb.shift;	362	return q->queue_hw_ctx[0]->sched_tags->bitmap_tags->sb.shift;
363	}	363	}
364		364
365	static struct kyber_queue_data kyber_queue_data_alloc(struct request_queue q)	365	static struct kyber_queue_data kyber_queue_data_alloc(struct request_queue q)
@@ -502,7 +502,7 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) · 136 unmodified lines
502	khd->batching = 0;	502	khd->batching = 0;
503		503
504	hctx->sched_data = khd;	504	hctx->sched_data = khd;
505	sbitmap_queue_min_shallow_depth(&hctx->sched_tags->bitmap_tags,	505	sbitmap_queue_min_shallow_depth(hctx->sched_tags->bitmap_tags,
506	kqd->async_depth);	506	kqd->async_depth);
507		507
508	return 0;	508	return 0;
@@ -573,7 +573,7 @@ static bool kyber_bio_merge(struct blk_mq_hw_ctx hctx, struct bio bio, · 64 unmodified lines
573	bool merged;	573	bool merged;
574		574
575	spin_lock(&kcq->lock);	575	spin_lock(&kcq->lock);
576	merged = blk_mq_bio_list_merge(hctx->queue, rq_list, bio, nr_segs);	576	merged = blk_bio_list_merge(hctx->queue, rq_list, bio, nr_segs);
577	spin_unlock(&kcq->lock);	577	spin_unlock(&kcq->lock);
578		578
579	return merged;	579	return merged;

block/mq-deadline.c+6−0

 	spin_lock(&dd->lock);
 	rq = __dd_dispatch_request(dd);
 	spin_unlock(&dd->lock);
+	if (rq)
+		atomic_dec(&rq->mq_hctx->elevator_queued);
 	return rq;
 }
 		rq = list_first_entry(list, struct request, queuelist);
 		list_del_init(&rq->queuelist);
 		dd_insert_request(hctx, rq, at_head);
+		atomic_inc(&hctx->elevator_queued);
 	}
 	spin_unlock(&dd->lock);
 }
 {
 	struct deadline_data *dd = hctx->queue->elevator->elevator_data;
+	if (!atomic_read(&hctx->elevator_queued))
+		return false;
 	return !list_empty_careful(&dd->dispatch) ||
 		!list_empty_careful(&dd->fifo_list[0]) ||
 		!list_empty_careful(&dd->fifo_list[1]);

block/partitions/core.c+15−12

@@ -199,14 +199,20 @@ static ssize_t part_alignment_offset_show(struct device *dev, · 198 unmodified lines
199	struct device_attribute attr, char buf)	199	struct device_attribute attr, char buf)
200	{	200	{
201	struct hd_struct *p = dev_to_part(dev);	201	struct hd_struct *p = dev_to_part(dev);
202	return sprintf(buf, "%llu\n", (unsigned long long)p->alignment_offset);	202
		203	return sprintf(buf, "%u\n",
		204	queue_limit_alignment_offset(&part_to_disk(p)->queue->limits,
		205	p->start_sect));
203	}	206	}
204		207
205	static ssize_t part_discard_alignment_show(struct device *dev,	208	static ssize_t part_discard_alignment_show(struct device *dev,
206	struct device_attribute attr, char buf)	209	struct device_attribute attr, char buf)
207	{	210	{
208	struct hd_struct *p = dev_to_part(dev);	211	struct hd_struct *p = dev_to_part(dev);
209	return sprintf(buf, "%u\n", p->discard_alignment);	212
		213	return sprintf(buf, "%u\n",
		214	queue_limit_discard_alignment(&part_to_disk(p)->queue->limits,
		215	p->start_sect));
210	}	216	}
211		217
212	static DEVICE_ATTR(partition, 0444, part_partition_show, NULL);	218	static DEVICE_ATTR(partition, 0444, part_partition_show, NULL);
@@ -318,8 +324,9 @@ int hd_ref_init(struct hd_struct *part) · 105 unmodified lines
318	* Must be called either with bd_mutex held, before a disk can be opened or	324	* Must be called either with bd_mutex held, before a disk can be opened or
319	* after all disk users are gone.	325	* after all disk users are gone.
320	*/	326	*/
321	void delete_partition(struct gendisk disk, struct hd_struct part)	327	void delete_partition(struct hd_struct *part)
322	{	328	{
		329	struct gendisk *disk = part_to_disk(part);
323	struct disk_part_tbl *ptbl =	330	struct disk_part_tbl *ptbl =
324	rcu_dereference_protected(disk->part_tbl, 1);	331	rcu_dereference_protected(disk->part_tbl, 1);
325		332
@@ -327,7 +334,7 @@ void delete_partition(struct gendisk disk, struct hd_struct part) · 1 unmodified line
327	* ->part_tbl is referenced in this part's release handler, so	334	* ->part_tbl is referenced in this part's release handler, so
328	* we have to hold the disk device	335	* we have to hold the disk device
329	*/	336	*/
330	get_device(disk_to_dev(part_to_disk(part)));	337	get_device(disk_to_dev(disk));
331	rcu_assign_pointer(ptbl->part[part->partno], NULL);	338	rcu_assign_pointer(ptbl->part[part->partno], NULL);
332	kobject_put(part->holder_dir);	339	kobject_put(part->holder_dir);
333	device_del(part_to_dev(part));	340	device_del(part_to_dev(part));
@@ -405,10 +412,6 @@ static struct hd_struct add_partition(struct gendisk disk, int partno, · 71 unmodified lines
405	pdev = part_to_dev(p);	412	pdev = part_to_dev(p);
406		413
407	p->start_sect = start;	414	p->start_sect = start;
408	p->alignment_offset =
409	queue_limit_alignment_offset(&disk->queue->limits, start);
410	p->discard_alignment =
411	queue_limit_discard_alignment(&disk->queue->limits, start);
412	p->nr_sects = len;	415	p->nr_sects = len;
413	p->partno = partno;	416	p->partno = partno;
414	p->policy = get_disk_ro(disk);	417	p->policy = get_disk_ro(disk);
@@ -554,7 +557,7 @@ int bdev_del_partition(struct block_device *bdev, int partno) · 139 unmodified lines
554	sync_blockdev(bdevp);	557	sync_blockdev(bdevp);
555	invalidate_bdev(bdevp);	558	invalidate_bdev(bdevp);
556		559
557	delete_partition(bdev->bd_disk, part);	560	delete_partition(part);
558	ret = 0;	561	ret = 0;
559	out_unlock:	562	out_unlock:
560	mutex_unlock(&bdev->bd_mutex);	563	mutex_unlock(&bdev->bd_mutex);
@@ -592,8 +595,8 @@ int bdev_resize_partition(struct block_device *bdev, int partno, · 31 unmodified lines
592	if (partition_overlaps(bdev->bd_disk, start, length, partno))	595	if (partition_overlaps(bdev->bd_disk, start, length, partno))
593	goto out_unlock;	596	goto out_unlock;
594		597
595	part_nr_sects_write(part, (sector_t)length);	598	part_nr_sects_write(part, length);
596	i_size_write(bdevp->bd_inode, length << SECTOR_SHIFT);	599	bd_set_nr_sectors(bdevp, length);
597		600
598	ret = 0;	601	ret = 0;
599	out_unlock:	602	out_unlock:
@@ -634,7 +637,7 @@ int blk_drop_partitions(struct block_device *bdev) · 34 unmodified lines
634		637
635	disk_part_iter_init(&piter, bdev->bd_disk, DISK_PITER_INCL_EMPTY);	638	disk_part_iter_init(&piter, bdev->bd_disk, DISK_PITER_INCL_EMPTY);
636	while ((part = disk_part_iter_next(&piter)))	639	while ((part = disk_part_iter_next(&piter)))
637	delete_partition(bdev->bd_disk, part);	640	delete_partition(part);
638	disk_part_iter_exit(&piter);	641	disk_part_iter_exit(&piter);
639		642
640	return 0;	643	return 0;

block/scsi_ioctl.c+0−2

@@ -37,8 +37,6 @@ const unsigned char scsi_command_size_tbl[8] = · 36 unmodified lines
37	};	37	};
38	EXPORT_SYMBOL(scsi_command_size_tbl);	38	EXPORT_SYMBOL(scsi_command_size_tbl);
39		39
40	#include <scsi/sg.h>
41
42	static int sg_get_version(int __user *p)	40	static int sg_get_version(int __user *p)
43	{	41	{
44	static const int sg_version_num = 30527;	42	static const int sg_version_num = 30527;

drivers/block/amiflop.c+1−1

@@ -1670,7 +1670,7 @@ static int floppy_open(struct block_device *bdev, fmode_t mode) · 1669 unmodified lines
1670	}	1670	}
1671		1671
1672	if (mode & (FMODE_READ\|FMODE_WRITE)) {	1672	if (mode & (FMODE_READ\|FMODE_WRITE)) {
1673	check_disk_change(bdev);	1673	bdev_check_media_change(bdev);
1674	if (mode & FMODE_WRITE) {	1674	if (mode & FMODE_WRITE) {
1675	int wrprot;	1675	int wrprot;
1676		1676

drivers/block/aoe/aoeblk.c+1−2

 	mempool_t *mp;
 	struct request_queue *q;
 	struct blk_mq_tag_set *set;
-	enum { KB = 1024, MB = KB * KB, READ_AHEAD = 2 * MB, };
 	ulong flags;
 	int late = 0;
 	int err;
 	WARN_ON(d->gd);
 	WARN_ON(d->flags & DEVFL_UP);
 	blk_queue_max_hw_sectors(q, BLK_DEF_MAX_SECTORS);
-	q->backing_dev_info->ra_pages = READ_AHEAD / PAGE_SIZE;
+	blk_queue_io_opt(q, SZ_2M);
 	d->bufpool = mp;
 	d->blkq = gd->queue = q;
 	q->queuedata = d;

drivers/block/aoe/aoecmd.c+1−3

 		ssize = get_capacity(d->gd);
 		bd = bdget_disk(d->gd, 0);
 		if (bd) {
-			inode_lock(bd->bd_inode);
+			bd_set_nr_sectors(bd, ssize);
-			i_size_write(bd->bd_inode, (loff_t)ssize<<9);
-			inode_unlock(bd->bd_inode);
 			bdput(bd);
 		}
 		spin_lock_irq(&d->lock);

drivers/block/ataflop.c+4−3

 		/* invalidate the buffer track to force a reread */
 		BufferDrive = -1;
 		set_bit(drive, &fake_change);
-		check_disk_change(bdev);
+		if (bdev_check_media_change(bdev))
+			floppy_revalidate(bdev->bd_disk);
 		return 0;
 	default:
 		return -EINVAL;
 		return 0;
 	if (mode & (FMODE_READ|FMODE_WRITE)) {
-		check_disk_change(bdev);
+		if (bdev_check_media_change(bdev))
+			floppy_revalidate(bdev->bd_disk);
 		if (mode & FMODE_WRITE) {
 			if (p->wpstat) {
 				if (p->ref < 0)
 	.release	= floppy_release,
 	.ioctl		= fd_ioctl,
 	.check_events	= floppy_check_events,
-	.revalidate_disk= floppy_revalidate,
 };
 static const struct blk_mq_ops ataflop_mq_ops = {

drivers/block/brd.c+0−1

@@ -403,7 +403,6 @@ static struct brd_device *brd_alloc(int i) · 402 unmodified lines
403	disk->flags = GENHD_FL_EXT_DEVT;	403	disk->flags = GENHD_FL_EXT_DEVT;
404	sprintf(disk->disk_name, "ram%d", i);	404	sprintf(disk->disk_name, "ram%d", i);
405	set_capacity(disk, rd_size * 2);	405	set_capacity(disk, rd_size * 2);
406	brd->brd_queue->backing_dev_info->capabilities \|= BDI_CAP_SYNCHRONOUS_IO;
407		406
408	/* Tell the block layer that this is not a rotational device */	407	/* Tell the block layer that this is not a rotational device */
409	blk_queue_flag_set(QUEUE_FLAG_NONROT, brd->brd_queue);	408	blk_queue_flag_set(QUEUE_FLAG_NONROT, brd->brd_queue);

drivers/block/drbd/drbd_nl.c+1−15

@@ -1362,15 +1362,7 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi · 1361 unmodified lines
1362		1362
1363	if (b) {	1363	if (b) {
1364	blk_stack_limits(&q->limits, &b->limits, 0);	1364	blk_stack_limits(&q->limits, &b->limits, 0);
1365		1365	blk_queue_update_readahead(q);
1366	if (q->backing_dev_info->ra_pages !=
1367	b->backing_dev_info->ra_pages) {
1368	drbd_info(device, "Adjusting my ra_pages to backing device's (%lu -> %lu)\n",
1369	q->backing_dev_info->ra_pages,
1370	b->backing_dev_info->ra_pages);
1371	q->backing_dev_info->ra_pages =
1372	b->backing_dev_info->ra_pages;
1373	}
1374	}	1366	}
1375	fixup_discard_if_not_supported(q);	1367	fixup_discard_if_not_supported(q);
1376	fixup_write_zeroes(device, q);	1368	fixup_write_zeroes(device, q);
@@ -3370,7 +3362,6 @@ static void device_to_statistics(struct device_statistics *s, · 1993 unmodified lines
3370	if (get_ldev(device)) {	3362	if (get_ldev(device)) {
3371	struct drbd_md *md = &device->ldev->md;	3363	struct drbd_md *md = &device->ldev->md;
3372	u64 history_uuids = (u64 )s->history_uuids;	3364	u64 history_uuids = (u64 )s->history_uuids;
3373	struct request_queue *q;
3374	int n;	3365	int n;
3375		3366
3376	spin_lock_irq(&md->uuid_lock);	3367	spin_lock_irq(&md->uuid_lock);
@@ -3384,11 +3375,6 @@ static void device_to_statistics(struct device_statistics *s, · 7 unmodified lines
3384	spin_unlock_irq(&md->uuid_lock);	3375	spin_unlock_irq(&md->uuid_lock);
3385		3376
3386	s->dev_disk_flags = md->flags;	3377	s->dev_disk_flags = md->flags;
3387	q = bdev_get_queue(device->ldev->backing_bdev);
3388	s->dev_lower_blocked =
3389	bdi_congested(q->backing_dev_info,
3390	(1 << WB_async_congested) \|
3391	(1 << WB_sync_congested));
3392	put_ldev(device);	3378	put_ldev(device);
3393	}	3379	}
3394	s->dev_size = drbd_get_capacity(device->this_bdev);	3380	s->dev_size = drbd_get_capacity(device->this_bdev);

drivers/block/floppy.c+5−3

  * output_byte is automatically disabled when reset is set.
  */
 static void reset_fdc(void);
+static int floppy_revalidate(struct gendisk *disk);
 /*
  * These are global variables, as that's the easiest way to give
 	/* invalidate the buffer track to force a reread */
 	set_bit((long)bdev->bd_disk->private_data, &fake_change);
 	process_fd_request();
-	check_disk_change(bdev);
+	if (bdev_check_media_change(bdev))
+		floppy_revalidate(bdev->bd_disk);
 	return 0;
 }
 			drive_state[drive].last_checked = 0;
 			clear_bit(FD_OPEN_SHOULD_FAIL_BIT,
 				  &drive_state[drive].flags);
-			check_disk_change(bdev);
+			if (bdev_check_media_change(bdev))
+				floppy_revalidate(bdev->bd_disk);
 			if (test_bit(FD_DISK_CHANGED_BIT, &drive_state[drive].flags))
 				goto out;
 			if (test_bit(FD_OPEN_SHOULD_FAIL_BIT, &drive_state[drive].flags))
 	.ioctl			= fd_ioctl,
 	.getgeo			= fd_getgeo,
 	.check_events		= floppy_check_events,
-	.revalidate_disk	= floppy_revalidate,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl		= fd_compat_ioctl,
 #endif

drivers/block/loop.c+2−2

@@ -253,7 +253,7 @@ static void loop_set_size(struct loop_device *lo, loff_t size) · 252 unmodified lines
253	{	253	{
254	struct block_device *bdev = lo->lo_device;	254	struct block_device *bdev = lo->lo_device;
255		255
256	bd_set_size(bdev, size << SECTOR_SHIFT);	256	bd_set_nr_sectors(bdev, size);
257		257
258	set_capacity_revalidate_and_notify(lo->lo_disk, size, false);	258	set_capacity_revalidate_and_notify(lo->lo_disk, size, false);
259	}	259	}
@@ -1251,7 +1251,7 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) · 991 unmodified lines
1251	set_capacity(lo->lo_disk, 0);	1251	set_capacity(lo->lo_disk, 0);
1252	loop_sysfs_exit(lo);	1252	loop_sysfs_exit(lo);
1253	if (bdev) {	1253	if (bdev) {
1254	bd_set_size(bdev, 0);	1254	bd_set_nr_sectors(bdev, 0);
1255	/* let user-space know about this change */	1255	/* let user-space know about this change */
1256	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);	1256	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
1257	}	1257	}

drivers/block/nbd.c+8−7

 {
 	struct nbd_config *config = nbd->config;
 	struct block_device *bdev = bdget_disk(nbd->disk, 0);
+	sector_t nr_sectors = config->bytesize >> 9;
 	if (config->flags & NBD_FLAG_SEND_TRIM) {
 		nbd->disk->queue->limits.discard_granularity = config->blksize;
 	}
 	blk_queue_logical_block_size(nbd->disk->queue, config->blksize);
 	blk_queue_physical_block_size(nbd->disk->queue, config->blksize);
-	set_capacity(nbd->disk, config->bytesize >> 9);
+	set_capacity(nbd->disk, nr_sectors);
 	if (bdev) {
 		if (bdev->bd_disk) {
-			bd_set_size(bdev, config->bytesize);
+			bd_set_nr_sectors(bdev, nr_sectors);
 			set_blocksize(bdev, config->blksize);
 		} else
-			bdev->bd_invalidated = 1;
+			set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);
 		bdput(bdev);
 	}
 	kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);
 {
 	if (bdev->bd_openers > 1)
 		return;
-	bd_set_size(bdev, 0);
+	bd_set_nr_sectors(bdev, 0);
 }
 static void nbd_parse_flags(struct nbd_device *nbd)
 		return ret;
 	if (max_part)
-		bdev->bd_invalidated = 1;
+		set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);
 	mutex_unlock(&nbd->config_lock);
 	ret = wait_event_interruptible(config->recv_wq,
 					 atomic_read(&config->recv_threads) == 0);
 		refcount_set(&nbd->config_refs, 1);
 		refcount_inc(&nbd->refs);
 		mutex_unlock(&nbd->config_lock);
-		bdev->bd_invalidated = 1;
+		set_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state);
 	} else if (nbd_disconnected(nbd->config)) {
-		bdev->bd_invalidated = 1;
+		set_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state);
 	}
 out:
 	mutex_unlock(&nbd_index_mutex);

drivers/block/paride/pcd.c+1−1

@@ -233,7 +233,7 @@ static int pcd_block_open(struct block_device *bdev, fmode_t mode) · 232 unmodified lines
233	struct pcd_unit *cd = bdev->bd_disk->private_data;	233	struct pcd_unit *cd = bdev->bd_disk->private_data;
234	int ret;	234	int ret;
235		235
236	check_disk_change(bdev);	236	bdev_check_media_change(bdev);
237		237
238	mutex_lock(&pcd_mutex);	238	mutex_lock(&pcd_mutex);
239	ret = cdrom_open(&cd->info, bdev, mode);	239	ret = cdrom_open(&cd->info, bdev, mode);

drivers/block/pktcdvd.c+14−80

 	}
 }
-/*
- * recover a failed write, query for relocation if possible
- *
- * returns 1 if recovery is possible, or 0 if not
- *
- */
-static int pkt_start_recovery(struct packet_data *pkt)
-{
-	/*
-	 * FIXME. We need help from the file system to implement
-	 * recovery handling.
-	 */
-	return 0;
-#if 0
-	struct request *rq = pkt->rq;
-	struct pktcdvd_device *pd = rq->rq_disk->private_data;
-	struct block_device *pkt_bdev;
-	struct super_block *sb = NULL;
-	unsigned long old_block, new_block;
-	sector_t new_sector;
-	pkt_bdev = bdget(kdev_t_to_nr(pd->pkt_dev));
-	if (pkt_bdev) {
-		sb = get_super(pkt_bdev);
-		bdput(pkt_bdev);
-	}
-	if (!sb)
-		return 0;
-	if (!sb->s_op->relocate_blocks)
-		goto out;
-	old_block = pkt->sector / (CD_FRAMESIZE >> 9);
-	if (sb->s_op->relocate_blocks(sb, old_block, &new_block))
-		goto out;
-	new_sector = new_block * (CD_FRAMESIZE >> 9);
-	pkt->sector = new_sector;
-	bio_reset(pkt->bio);
-	bio_set_dev(pkt->bio, pd->bdev);
-	bio_set_op_attrs(pkt->bio, REQ_OP_WRITE, 0);
-	pkt->bio->bi_iter.bi_sector = new_sector;
-	pkt->bio->bi_iter.bi_size = pkt->frames * CD_FRAMESIZE;
-	pkt->bio->bi_vcnt = pkt->frames;
-	pkt->bio->bi_end_io = pkt_end_io_packet_write;
-	pkt->bio->bi_private = pkt;
-	drop_super(sb);
-	return 1;
-out:
-	drop_super(sb);
-	return 0;
-#endif
-}
 static inline void pkt_set_state(struct packet_data *pkt, enum packet_data_state state)
 {
 #if PACKET_DEBUG > 1
 			break;
 		case PACKET_RECOVERY_STATE:
-			if (pkt_start_recovery(pkt)) {
+			pkt_dbg(2, pd, "No recovery possible\n");
-				pkt_start_write(pd, pkt);
+			pkt_set_state(pkt, PACKET_FINISHED_STATE);
-			} else {
-				pkt_dbg(2, pd, "No recovery possible\n");
-				pkt_set_state(pkt, PACKET_FINISHED_STATE);
-			}
 			break;
 		case PACKET_FINISHED_STATE:
 	int ret;
 	long lba;
 	struct request_queue *q;
+	struct block_device *bdev;
 	/*
 	 * We need to re-open the cdrom device without O_NONBLOCK to be able
 	 * to read/write from/to it. It is already opened in O_NONBLOCK mode
-	 * so bdget() can't fail.
+	 * so open should not fail.
 	 */
-	bdget(pd->bdev->bd_dev);
+	bdev = blkdev_get_by_dev(pd->bdev->bd_dev, FMODE_READ | FMODE_EXCL, pd);
-	ret = blkdev_get(pd->bdev, FMODE_READ | FMODE_EXCL, pd);
+	if (IS_ERR(bdev)) {
-	if (ret)
+		ret = PTR_ERR(bdev);
 		goto out;
+	}
 	ret = pkt_get_last_written(pd, &lba);
 	if (ret) {
 	set_capacity(pd->disk, lba << 2);
 	set_capacity(pd->bdev->bd_disk, lba << 2);
-	bd_set_size(pd->bdev, (loff_t)lba << 11);
+	bd_set_nr_sectors(pd->bdev, lba << 2);
 	q = bdev_get_queue(pd->bdev);
 	if (write) {
 	return 0;
 out_putdev:
-	blkdev_put(pd->bdev, FMODE_READ | FMODE_EXCL);
+	blkdev_put(bdev, FMODE_READ | FMODE_EXCL);
 out:
 	return ret;
 }
 static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev)
 {
 	int i;
-	int ret = 0;
 	char b[BDEVNAME_SIZE];
 	struct block_device *bdev;
 		}
 	}
-	bdev = bdget(dev);
+	bdev = blkdev_get_by_dev(dev, FMODE_READ | FMODE_NDELAY, NULL);
-	if (!bdev)
+	if (IS_ERR(bdev))
-		return -ENOMEM;
+		return PTR_ERR(bdev);
-	ret = blkdev_get(bdev, FMODE_READ | FMODE_NDELAY, NULL);
-	if (ret)
-		return ret;
 	if (!blk_queue_scsi_passthrough(bdev_get_queue(bdev))) {
 		blkdev_put(bdev, FMODE_READ | FMODE_NDELAY);
 		return -EINVAL;
 	pd->cdrw.thread = kthread_run(kcdrwd, pd, "%s", pd->name);
 	if (IS_ERR(pd->cdrw.thread)) {
 		pkt_err(pd, "can't start kernel thread\n");
-		ret = -ENOMEM;
 		goto out_mem;
 	}
 	blkdev_put(bdev, FMODE_READ | FMODE_NDELAY);
 	/* This is safe: open() is still holding a reference. */
 	module_put(THIS_MODULE);
-	return ret;
+	return -ENOMEM;
 }
 static int pkt_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)

drivers/block/rbd.c+2−2

@@ -4921,7 +4921,7 @@ static void rbd_dev_update_size(struct rbd_device *rbd_dev) · 4920 unmodified lines
4921	size = (sector_t)rbd_dev->mapping.size / SECTOR_SIZE;	4921	size = (sector_t)rbd_dev->mapping.size / SECTOR_SIZE;
4922	dout("setting size to %llu sectors", (unsigned long long)size);	4922	dout("setting size to %llu sectors", (unsigned long long)size);
4923	set_capacity(rbd_dev->disk, size);	4923	set_capacity(rbd_dev->disk, size);
4924	revalidate_disk(rbd_dev->disk);	4924	revalidate_disk_size(rbd_dev->disk, true);
4925	}	4925	}
4926	}	4926	}
4927		4927
@@ -5022,7 +5022,7 @@ static int rbd_init_disk(struct rbd_device *rbd_dev) · 94 unmodified lines
5022	}	5022	}
5023		5023
5024	if (!ceph_test_opt(rbd_dev->rbd_client->client, NOCRC))	5024	if (!ceph_test_opt(rbd_dev->rbd_client->client, NOCRC))
5025	q->backing_dev_info->capabilities \|= BDI_CAP_STABLE_WRITES;	5025	blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
5026		5026
5027	/*	5027	/*
5028	* disk_release() expects a queue ref from add_disk() and will	5028	* disk_release() expects a queue ref from add_disk() and will

drivers/block/rnbd/rnbd-clt.c+3−9

 static int rnbd_clt_change_capacity(struct rnbd_clt_dev *dev,
 				    size_t new_nsectors)
 {
-	int err = 0;
 	rnbd_clt_info(dev, "Device size changed from %zu to %zu sectors\n",
 		       dev->nsectors, new_nsectors);
 	dev->nsectors = new_nsectors;
 	set_capacity(dev->gd, dev->nsectors);
-	err = revalidate_disk(dev->gd);
+	revalidate_disk_size(dev->gd, true);
-	if (err)
+	return 0;
-		rnbd_clt_err(dev,
-			      "Failed to change device size from %zu to %zu, err: %d\n",
-			      dev->nsectors, new_nsectors, err);
-	return err;
 }
 static int process_msg_open_rsp(struct rnbd_clt_dev *dev,
 	tag_set->queue_depth	= sess->queue_depth;
 	tag_set->numa_node		= NUMA_NO_NODE;
 	tag_set->flags		= BLK_MQ_F_SHOULD_MERGE |
-				  BLK_MQ_F_TAG_SHARED;
+				  BLK_MQ_F_TAG_QUEUE_SHARED;
 	tag_set->cmd_size		= sizeof(struct rnbd_iu);
 	tag_set->nr_hw_queues	= num_online_cpus();

drivers/block/swim.c+2−20

 		return 0;
 	if (mode & (FMODE_READ|FMODE_WRITE)) {
-		check_disk_change(bdev);
+		if (bdev_check_media_change(bdev) && fs->disk_in)
+			fs->ejected = 0;
 		if ((mode & FMODE_WRITE) && fs->write_protected) {
 			err = -EROFS;
 			goto out;
 	return fs->ejected ? DISK_EVENT_MEDIA_CHANGE : 0;
 }
-static int floppy_revalidate(struct gendisk *disk)
-{
-	struct floppy_state *fs = disk->private_data;
-	struct swim __iomem *base = fs->swd->base;
-	swim_drive(base, fs->location);
-	if (fs->ejected)
-		setup_medium(fs);
-	if (!fs->disk_in)
-		swim_motor(base, OFF);
-	else
-		fs->ejected = 0;
-	return !fs->disk_in;
-}
 static const struct block_device_operations floppy_fops = {
 	.owner		 = THIS_MODULE,
 	.open		 = floppy_unlocked_open,
 	.ioctl		 = floppy_ioctl,
 	.getgeo		 = floppy_getgeo,
 	.check_events	 = floppy_check_events,
-	.revalidate_disk = floppy_revalidate,
 };
 static struct kobject *floppy_find(dev_t dev, int *part, void *data)

drivers/block/swim3.c+2−2

 	if (err == 0 && (mode & FMODE_NDELAY) == 0
 	    && (mode & (FMODE_READ|FMODE_WRITE))) {
-		check_disk_change(bdev);
+		if (bdev_check_media_change(bdev))
+			floppy_revalidate(bdev->bd_disk);
 		if (fs->ejected)
 			err = -ENXIO;
 	}
 	.release	= floppy_release,
 	.ioctl		= floppy_ioctl,
 	.check_events	= floppy_check_events,
-	.revalidate_disk= floppy_revalidate,
 };
 static const struct blk_mq_ops swim3_mq_ops = {

drivers/block/virtio_blk.c+2−2

@@ -598,7 +598,7 @@ static void virtblk_update_cache_mode(struct virtio_device *vdev) · 597 unmodified lines
598	struct virtio_blk *vblk = vdev->priv;	598	struct virtio_blk *vblk = vdev->priv;
599		599
600	blk_queue_write_cache(vblk->disk->queue, writeback, false);	600	blk_queue_write_cache(vblk->disk->queue, writeback, false);
601	revalidate_disk(vblk->disk);	601	revalidate_disk_size(vblk->disk, true);
602	}	602	}
603		603
604	static const char *const virtblk_cache_types[] = {	604	static const char *const virtblk_cache_types[] = {
@@ -646,7 +646,7 @@ static struct attribute *virtblk_attrs[] = { · 41 unmodified lines
646	static umode_t virtblk_attrs_are_visible(struct kobject *kobj,	646	static umode_t virtblk_attrs_are_visible(struct kobject *kobj,
647	struct attribute *a, int n)	647	struct attribute *a, int n)
648	{	648	{
649	struct device *dev = container_of(kobj, struct device, kobj);	649	struct device *dev = kobj_to_dev(kobj);
650	struct gendisk *disk = dev_to_disk(dev);	650	struct gendisk *disk = dev_to_disk(dev);
651	struct virtio_blk *vblk = disk->private_data;	651	struct virtio_blk *vblk = disk->private_data;
652	struct virtio_device *vdev = vblk->vdev;	652	struct virtio_device *vdev = vblk->vdev;

drivers/block/xsysace.c+10−16

 	return ace->media_change ? DISK_EVENT_MEDIA_CHANGE : 0;
 }
-static int ace_revalidate_disk(struct gendisk *gd)
+static void ace_media_changed(struct ace_device *ace)
 {
-	struct ace_device *ace = gd->private_data;
 	unsigned long flags;
-	dev_dbg(ace->dev, "ace_revalidate_disk()\n");
+	dev_dbg(ace->dev, "requesting cf id and scheduling tasklet\n");
-	if (ace->media_change) {
-		dev_dbg(ace->dev, "requesting cf id and scheduling tasklet\n");
-		spin_lock_irqsave(&ace->lock, flags);
+	spin_lock_irqsave(&ace->lock, flags);
-		ace->id_req_count++;
+	ace->id_req_count++;
-		spin_unlock_irqrestore(&ace->lock, flags);
+	spin_unlock_irqrestore(&ace->lock, flags);
-		tasklet_schedule(&ace->fsm_tasklet);
+	tasklet_schedule(&ace->fsm_tasklet);
-		wait_for_completion(&ace->id_completion);
+	wait_for_completion(&ace->id_completion);
-	}
 	dev_dbg(ace->dev, "revalidate complete\n");
-	return ace->id_result;
 }
 static int ace_open(struct block_device *bdev, fmode_t mode)
 	ace->users++;
 	spin_unlock_irqrestore(&ace->lock, flags);
-	check_disk_change(bdev);
+	if (bdev_check_media_change(bdev) && ace->media_change)
+		ace_media_changed(ace);
 	mutex_unlock(&xsysace_mutex);
 	return 0;
 	.open = ace_open,
 	.release = ace_release,
 	.check_events = ace_check_events,
-	.revalidate_disk = ace_revalidate_disk,
 	.getgeo = ace_getgeo,
 };
 		(unsigned long long) ace->physaddr, ace->baseaddr, ace->irq);
 	ace->media_change = 1;
-	ace_revalidate_disk(ace->gd);
+	ace_media_changed(ace);
 	/* Make the sysace device 'live' */
 	add_disk(ace->gd);

drivers/block/zram/zram_drv.c+19−11

  */
 static size_t huge_class_size;
+static const struct block_device_operations zram_devops;
+static const struct block_device_operations zram_wb_devops;
 static void zram_free_page(struct zram *zram, size_t index);
 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
 				u32 index, int offset, struct bio *bio);
 	zram->backing_dev = NULL;
 	zram->old_block_size = 0;
 	zram->bdev = NULL;
-	zram->disk->queue->backing_dev_info->capabilities |=
+	zram->disk->fops = &zram_devops;
-				BDI_CAP_SYNCHRONOUS_IO;
 	kvfree(zram->bitmap);
 	zram->bitmap = NULL;
 }
 		goto out;
 	}
-	bdev = bdgrab(I_BDEV(inode));
+	bdev = blkdev_get_by_dev(inode->i_rdev,
-	err = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
+			FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
-	if (err < 0) {
+	if (IS_ERR(bdev)) {
+		err = PTR_ERR(bdev);
 		bdev = NULL;
 		goto out;
 	}
 	 * freely but in fact, IO is going on so finally could cause
 	 * use-after-free when the IO is really done.
 	 */
-	zram->disk->queue->backing_dev_info->capabilities &=
+	zram->disk->fops = &zram_wb_devops;
-			~BDI_CAP_SYNCHRONOUS_IO;
 	up_write(&zram->init_lock);
 	pr_info("setup backing device %s\n", file_name);
 	zram->disksize = disksize;
 	set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
-	revalidate_disk(zram->disk);
+	revalidate_disk_size(zram->disk, true);
 	up_write(&zram->init_lock);
 	return len;
 	/* Make sure all the pending I/O are finished */
 	fsync_bdev(bdev);
 	zram_reset_device(zram);
-	revalidate_disk(zram->disk);
+	revalidate_disk_size(zram->disk, true);
 	bdput(bdev);
 	mutex_lock(&bdev->bd_mutex);
 	.owner = THIS_MODULE
 };
+static const struct block_device_operations zram_wb_devops = {
+	.open = zram_open,
+	.submit_bio = zram_submit_bio,
+	.swap_slot_free_notify = zram_slot_free_notify,
+	.owner = THIS_MODULE
+};
 static DEVICE_ATTR_WO(compact);
 static DEVICE_ATTR_RW(disksize);
 static DEVICE_ATTR_RO(initstate);
 	if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
 		blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX);
-	zram->disk->queue->backing_dev_info->capabilities |=
+	blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, zram->disk->queue);
-			(BDI_CAP_STABLE_WRITES | BDI_CAP_SYNCHRONOUS_IO);
 	device_add_disk(NULL, zram->disk, zram_disk_attr_groups);
 	strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));

drivers/cdrom/gdrom.c+1−1

@@ -479,7 +479,7 @@ static int gdrom_bdops_open(struct block_device *bdev, fmode_t mode) · 478 unmodified lines
479	{	479	{
480	int ret;	480	int ret;
481		481
482	check_disk_change(bdev);	482	bdev_check_media_change(bdev);
483		483
484	mutex_lock(&gdrom_mutex);	484	mutex_lock(&gdrom_mutex);
485	ret = cdrom_open(gd.cd_info, bdev, mode);	485	ret = cdrom_open(gd.cd_info, bdev, mode);

drivers/char/raw.c+24−32

@@ -28,7 +28,8 @@ · 27 unmodified lines
28	#include <linux/uaccess.h>	28	#include <linux/uaccess.h>
29		29
30	struct raw_device_data {	30	struct raw_device_data {
31	struct block_device *binding;	31	dev_t binding;
		32	struct block_device *bdev;
32	int inuse;	33	int inuse;
33	};	34	};
34		35
@@ -63,19 +64,25 @@ static int raw_open(struct inode inode, struct file filp) · 28 unmodified lines
63	return 0;	64	return 0;
64	}	65	}
65		66
		67	pr_warn_ratelimited(
		68	"process %s (pid %d) is using the deprecated raw device\n"
		69	"support will be removed in Linux 5.14.\n",
		70	current->comm, current->pid);
		71
66	mutex_lock(&raw_mutex);	72	mutex_lock(&raw_mutex);
67		73
68	/*	74	/*
69	* All we need to do on open is check that the device is bound.	75	* All we need to do on open is check that the device is bound.
70	*/	76	*/
71	bdev = raw_devices[minor].binding;
72	err = -ENODEV;	77	err = -ENODEV;
73	if (!bdev)	78	if (!raw_devices[minor].binding)
74	goto out;	79	goto out;
75	bdgrab(bdev);	80	bdev = blkdev_get_by_dev(raw_devices[minor].binding,
76	err = blkdev_get(bdev, filp->f_mode \| FMODE_EXCL, raw_open);	81	filp->f_mode \| FMODE_EXCL, raw_open);
77	if (err)	82	if (IS_ERR(bdev)) {
		83	err = PTR_ERR(bdev);
78	goto out;	84	goto out;
		85	}
79	err = set_blocksize(bdev, bdev_logical_block_size(bdev));	86	err = set_blocksize(bdev, bdev_logical_block_size(bdev));
80	if (err)	87	if (err)
81	goto out1;	88	goto out1;
@@ -85,6 +92,7 @@ static int raw_open(struct inode inode, struct file filp) · 3 unmodified lines
85	file_inode(filp)->i_mapping =	92	file_inode(filp)->i_mapping =
86	bdev->bd_inode->i_mapping;	93	bdev->bd_inode->i_mapping;
87	filp->private_data = bdev;	94	filp->private_data = bdev;
		95	raw_devices[minor].bdev = bdev;
88	mutex_unlock(&raw_mutex);	96	mutex_unlock(&raw_mutex);
89	return 0;	97	return 0;
90		98
@@ -105,7 +113,7 @@ static int raw_release(struct inode inode, struct file filp) · 14 unmodified lines
105	struct block_device *bdev;	113	struct block_device *bdev;
106		114
107	mutex_lock(&raw_mutex);	115	mutex_lock(&raw_mutex);
108	bdev = raw_devices[minor].binding;	116	bdev = raw_devices[minor].bdev;
109	if (--raw_devices[minor].inuse == 0)	117	if (--raw_devices[minor].inuse == 0)
110	/* Here inode->i_mapping == bdev->bd_inode->i_mapping */	118	/* Here inode->i_mapping == bdev->bd_inode->i_mapping */
111	inode->i_mapping = &inode->i_data;	119	inode->i_mapping = &inode->i_data;
@@ -128,6 +136,7 @@ raw_ioctl(struct file *filp, unsigned int command, unsigned long arg) · 16 unmodified lines
128	static int bind_set(int number, u64 major, u64 minor)	136	static int bind_set(int number, u64 major, u64 minor)
129	{	137	{
130	dev_t dev = MKDEV(major, minor);	138	dev_t dev = MKDEV(major, minor);
		139	dev_t raw = MKDEV(RAW_MAJOR, number);
131	struct raw_device_data *rawdev;	140	struct raw_device_data *rawdev;
132	int err = 0;	141	int err = 0;
133		142
@@ -161,25 +170,17 @@ static int bind_set(int number, u64 major, u64 minor) · 27 unmodified lines
161	mutex_unlock(&raw_mutex);	170	mutex_unlock(&raw_mutex);
162	return -EBUSY;	171	return -EBUSY;
163	}	172	}
164	if (rawdev->binding) {	173	if (rawdev->binding)
165	bdput(rawdev->binding);
166	module_put(THIS_MODULE);	174	module_put(THIS_MODULE);
167	}	175
		176	rawdev->binding = dev;
168	if (!dev) {	177	if (!dev) {
169	/* unbind */	178	/* unbind */
170	rawdev->binding = NULL;	179	device_destroy(raw_class, raw);
171	device_destroy(raw_class, MKDEV(RAW_MAJOR, number));
172	} else {	180	} else {
173	rawdev->binding = bdget(dev);	181	__module_get(THIS_MODULE);
174	if (rawdev->binding == NULL) {	182	device_destroy(raw_class, raw);
175	err = -ENOMEM;	183	device_create(raw_class, NULL, raw, NULL, "raw%d", number);
176	} else {
177	dev_t raw = MKDEV(RAW_MAJOR, number);
178	__module_get(THIS_MODULE);
179	device_destroy(raw_class, raw);
180	device_create(raw_class, NULL, raw, NULL,
181	"raw%d", number);
182	}
183	}	184	}
184	mutex_unlock(&raw_mutex);	185	mutex_unlock(&raw_mutex);
185	return err;	186	return err;
@@ -187,18 +188,9 @@ static int bind_set(int number, u64 major, u64 minor) · 1 unmodified line
187		188
188	static int bind_get(int number, dev_t *dev)	189	static int bind_get(int number, dev_t *dev)
189	{	190	{
190	struct raw_device_data *rawdev;
191	struct block_device *bdev;
192
193	if (number <= 0 \|\| number >= max_raw_minors)	191	if (number <= 0 \|\| number >= max_raw_minors)
194	return -EINVAL;	192	return -EINVAL;
195		193	*dev = raw_devices[number].binding;
196	rawdev = &raw_devices[number];
197
198	mutex_lock(&raw_mutex);
199	bdev = rawdev->binding;
200	*dev = bdev ? bdev->bd_dev : 0;
201	mutex_unlock(&raw_mutex);
202	return 0;	194	return 0;
203	}	195	}
204		196

drivers/ide/ide-cd.c+5−11

 	struct cdrom_info *info;
 	int rc = -ENXIO;
-	check_disk_change(bdev);
+	if (bdev_check_media_change(bdev)) {
+		info = ide_drv_g(bdev->bd_disk, cdrom_info);
+		ide_cd_read_toc(info->drive);
+	}
 	mutex_lock(&ide_cd_mutex);
 	info = ide_cd_get(bdev->bd_disk);
 	return cdrom_check_events(&info->devinfo, clearing);
 }
-static int idecd_revalidate_disk(struct gendisk *disk)
-{
-	struct cdrom_info *info = ide_drv_g(disk, cdrom_info);
-	ide_cd_read_toc(info->drive);
-	return  0;
-}
 static const struct block_device_operations idecd_ops = {
 	.owner			= THIS_MODULE,
 	.open			= idecd_open,
 	.compat_ioctl		= IS_ENABLED(CONFIG_COMPAT) ?
 				  idecd_compat_ioctl : NULL,
 	.check_events		= idecd_check_events,
-	.revalidate_disk	= idecd_revalidate_disk
 };
 /* module options */

drivers/ide/ide-disk.c+1−4

 	set_wcache(drive, 1);
 	if ((drive->dev_flags & IDE_DFLAG_LBA) == 0 &&
-	    (drive->head == 0 || drive->head > 16)) {
+	    (drive->head == 0 || drive->head > 16))
 		printk(KERN_ERR "%s: invalid geometry: %d physical heads?\n",
 			drive->name, drive->head);
-		drive->dev_flags &= ~IDE_DFLAG_ATTACH;
-	} else
-		drive->dev_flags |= IDE_DFLAG_ATTACH;
 }
 static void ide_disk_flush(ide_drive_t *drive)

drivers/ide/ide-floppy.c+0−2

@@ -516,8 +516,6 @@ static void ide_floppy_setup(ide_drive_t *drive) · 515 unmodified lines
516	(void) ide_floppy_get_capacity(drive);	516	(void) ide_floppy_get_capacity(drive);
517		517
518	ide_proc_register_driver(drive, floppy->driver);	518	ide_proc_register_driver(drive, floppy->driver);
519
520	drive->dev_flags \|= IDE_DFLAG_ATTACH;
521	}	519	}
522		520
523	static void ide_floppy_flush(ide_drive_t *drive)	521	static void ide_floppy_flush(ide_drive_t *drive)

drivers/ide/ide-gd.c+6−42

@@ -225,8 +225,12 @@ static int ide_gd_open(struct block_device *bdev, fmode_t mode) · 224 unmodified lines
225	* and the door_lock is irrelevant at this point.	225	* and the door_lock is irrelevant at this point.
226	*/	226	*/
227	drive->disk_ops->set_doorlock(drive, disk, 1);	227	drive->disk_ops->set_doorlock(drive, disk, 1);
228	drive->dev_flags \|= IDE_DFLAG_MEDIA_CHANGED;	228	if (__invalidate_device(bdev, true))
229	check_disk_change(bdev);	229	pr_warn("VFS: busy inodes on changed media %s\n",
		230	bdev->bd_disk->disk_name);
		231	drive->disk_ops->get_capacity(drive);
		232	set_capacity(disk, ide_gd_capacity(drive));
		233	set_bit(GD_NEED_PART_SCAN, &disk->state);
230	} else if (drive->dev_flags & IDE_DFLAG_FORMAT_IN_PROGRESS) {	234	} else if (drive->dev_flags & IDE_DFLAG_FORMAT_IN_PROGRESS) {
231	ret = -EBUSY;	235	ret = -EBUSY;
232	goto out_put_idkp;	236	goto out_put_idkp;
@@ -284,32 +288,6 @@ static int ide_gd_getgeo(struct block_device bdev, struct hd_geometry geo) · 51 unmodified lines
284	return 0;	288	return 0;
285	}	289	}
286		290
287	static unsigned int ide_gd_check_events(struct gendisk *disk,
288	unsigned int clearing)
289	{
290	struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj);
291	ide_drive_t *drive = idkp->drive;
292	bool ret;
293
294	/* do not scan partitions twice if this is a removable device */
295	if (drive->dev_flags & IDE_DFLAG_ATTACH) {
296	drive->dev_flags &= ~IDE_DFLAG_ATTACH;
297	return 0;
298	}
299
300	/*
301	* The following is used to force revalidation on the first open on
302	* removeable devices, and never gets reported to userland as
303	* DISK_EVENT_FLAG_UEVENT isn't set in genhd->event_flags.
304	* This is intended as removable ide disk can't really detect
305	* MEDIA_CHANGE events.
306	*/
307	ret = drive->dev_flags & IDE_DFLAG_MEDIA_CHANGED;
308	drive->dev_flags &= ~IDE_DFLAG_MEDIA_CHANGED;
309
310	return ret ? DISK_EVENT_MEDIA_CHANGE : 0;
311	}
312
313	static void ide_gd_unlock_native_capacity(struct gendisk *disk)	291	static void ide_gd_unlock_native_capacity(struct gendisk *disk)
314	{	292	{
315	struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj);	293	struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj);
@@ -320,18 +298,6 @@ static void ide_gd_unlock_native_capacity(struct gendisk *disk) · 4 unmodified lines
320	disk_ops->unlock_native_capacity(drive);	298	disk_ops->unlock_native_capacity(drive);
321	}	299	}
322		300
323	static int ide_gd_revalidate_disk(struct gendisk *disk)
324	{
325	struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj);
326	ide_drive_t *drive = idkp->drive;
327
328	if (ide_gd_check_events(disk, 0))
329	drive->disk_ops->get_capacity(drive);
330
331	set_capacity(disk, ide_gd_capacity(drive));
332	return 0;
333	}
334
335	static int ide_gd_ioctl(struct block_device *bdev, fmode_t mode,	301	static int ide_gd_ioctl(struct block_device *bdev, fmode_t mode,
336	unsigned int cmd, unsigned long arg)	302	unsigned int cmd, unsigned long arg)
337	{	303	{
@@ -364,9 +330,7 @@ static const struct block_device_operations ide_gd_ops = { · 26 unmodified lines
364	.compat_ioctl = ide_gd_compat_ioctl,	330	.compat_ioctl = ide_gd_compat_ioctl,
365	#endif	331	#endif
366	.getgeo = ide_gd_getgeo,	332	.getgeo = ide_gd_getgeo,
367	.check_events = ide_gd_check_events,
368	.unlock_native_capacity = ide_gd_unlock_native_capacity,	333	.unlock_native_capacity = ide_gd_unlock_native_capacity,
369	.revalidate_disk = ide_gd_revalidate_disk
370	};	334	};
371		335
372	static int ide_gd_probe(ide_drive_t *drive)	336	static int ide_gd_probe(ide_drive_t *drive)

drivers/md/bcache/request.c+6−4

 	unsigned int		read_dirty_data:1;
 	unsigned int		cache_missed:1;
+	struct hd_struct	*part;
 	unsigned long		start_time;
 	struct btree_op		op;
 {
 	if (s->orig_bio) {
 		/* Count on bcache device */
-		disk_end_io_acct(s->d->disk, bio_op(s->orig_bio), s->start_time);
+		part_end_io_acct(s->part, s->orig_bio, s->start_time);
 		trace_bcache_request_end(s->d, s->orig_bio);
 		s->orig_bio->bi_status = s->iop.status;
 	s->write		= op_is_write(bio_op(bio));
 	s->read_dirty_data	= 0;
 	/* Count on the bcache device */
-	s->start_time		= disk_start_io_acct(d->disk, bio_sectors(bio), bio_op(bio));
+	s->start_time		= part_start_io_acct(d->disk, &s->part, bio);
 	s->iop.c		= d->c;
 	s->iop.bio		= NULL;
 	s->iop.inode		= d->id;
 	unsigned long		start_time;
 	bio_end_io_t		*bi_end_io;
 	void			*bi_private;
+	struct hd_struct	*part;
 };
 static void detached_dev_end_io(struct bio *bio)
 	bio->bi_private = ddip->bi_private;
 	/* Count on the bcache device */
-	disk_end_io_acct(ddip->d->disk, bio_op(bio), ddip->start_time);
+	part_end_io_acct(ddip->part, bio, ddip->start_time);
 	if (bio->bi_status) {
 		struct cached_dev *dc = container_of(ddip->d,
 	ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO);
 	ddip->d = d;
 	/* Count on the bcache device */
-	ddip->start_time = disk_start_io_acct(d->disk, bio_sectors(bio), bio_op(bio));
+	ddip->start_time = part_start_io_acct(d->disk, &ddip->part, bio);
 	ddip->bi_end_io = bio->bi_end_io;
 	ddip->bi_private = bio->bi_private;
 	bio->bi_end_io = detached_dev_end_io;

drivers/md/bcache/super.c+2−3

@@ -1427,9 +1427,8 @@ static int cached_dev_init(struct cached_dev *dc, unsigned int block_size) · 1426 unmodified lines
1427	if (ret)	1427	if (ret)
1428	return ret;	1428	return ret;
1429		1429
1430	dc->disk.disk->queue->backing_dev_info->ra_pages =	1430	blk_queue_io_opt(dc->disk.disk->queue,
1431	max(dc->disk.disk->queue->backing_dev_info->ra_pages,	1431	max(queue_io_opt(dc->disk.disk->queue), queue_io_opt(q)));
1432	q->backing_dev_info->ra_pages);
1433		1432
1434	atomic_set(&dc->io_errors, 0);	1433	atomic_set(&dc->io_errors, 0);
1435	dc->io_disable = false;	1434	dc->io_disable = false;

drivers/md/dm-raid.c+1−1

@@ -701,7 +701,7 @@ static void rs_set_capacity(struct raid_set *rs) · 700 unmodified lines
701	struct gendisk *gendisk = dm_disk(dm_table_get_md(rs->ti->table));	701	struct gendisk *gendisk = dm_disk(dm_table_get_md(rs->ti->table));
702		702
703	set_capacity(gendisk, rs->md.array_sectors);	703	set_capacity(gendisk, rs->md.array_sectors);
704	revalidate_disk(gendisk);	704	revalidate_disk_size(gendisk, true);
705	}	705	}
706		706
707	/*	707	/*

drivers/md/dm-table.c+4−5

 {
 	struct request_queue *q = bdev_get_queue(dev->bdev);
-	return q && bdi_cap_stable_pages_required(q->backing_dev_info);
+	return q && blk_queue_stable_writes(q);
 }
 /*
 	 * because they do their own checksumming.
 	 */
 	if (dm_table_requires_stable_pages(t))
-		q->backing_dev_info->capabilities |= BDI_CAP_STABLE_WRITES;
+		blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
 	else
-		q->backing_dev_info->capabilities &= ~BDI_CAP_STABLE_WRITES;
+		blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
 	/*
 	 * Determine whether or not this queue's I/O timings contribute
 	}
 #endif
-	/* Allow reads to exceed readahead limits */
+	blk_queue_update_readahead(q);
-	q->backing_dev_info->io_pages = limits->max_sectors >> (PAGE_SHIFT - 9);
 }
 unsigned int dm_table_get_num_targets(struct dm_table *t)

drivers/md/dm.c+2−13

 	dm_issue_global_event();
 }
-/*
- * Protected by md->suspend_lock obtained by dm_swap_table().
- */
-static void __set_size(struct mapped_device *md, sector_t size)
-{
-	lockdep_assert_held(&md->suspend_lock);
-	set_capacity(md->disk, size);
-	i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
-}
 /*
  * Returns old map, which caller must destroy.
  */
 	if (size != dm_get_size(md))
 		memset(&md->geometry, 0, sizeof(md->geometry));
-	__set_size(md, size);
+	set_capacity(md->disk, size);
+	bd_set_nr_sectors(md->bdev, size);
 	dm_table_event_callback(t, event_callback, md);

drivers/md/md-cluster.c+3−3

@@ -582,7 +582,7 @@ static int process_recvd_msg(struct mddev mddev, struct cluster_msg msg) · 581 unmodified lines
582	break;	582	break;
583	case CHANGE_CAPACITY:	583	case CHANGE_CAPACITY:
584	set_capacity(mddev->gendisk, mddev->array_sectors);	584	set_capacity(mddev->gendisk, mddev->array_sectors);
585	revalidate_disk(mddev->gendisk);	585	revalidate_disk_size(mddev->gendisk, true);
586	break;	586	break;
587	case RESYNCING:	587	case RESYNCING:
588	set_bit(MD_RESYNCING_REMOTE, &mddev->recovery);	588	set_bit(MD_RESYNCING_REMOTE, &mddev->recovery);
@@ -1296,12 +1296,12 @@ static void update_size(struct mddev *mddev, sector_t old_dev_sectors) · 707 unmodified lines
1296	pr_err("%s:%d: failed to send CHANGE_CAPACITY msg\n",	1296	pr_err("%s:%d: failed to send CHANGE_CAPACITY msg\n",
1297	__func__, __LINE__);	1297	__func__, __LINE__);
1298	set_capacity(mddev->gendisk, mddev->array_sectors);	1298	set_capacity(mddev->gendisk, mddev->array_sectors);
1299	revalidate_disk(mddev->gendisk);	1299	revalidate_disk_size(mddev->gendisk, true);
1300	} else {	1300	} else {
1301	/* revert to previous sectors */	1301	/* revert to previous sectors */
1302	ret = mddev->pers->resize(mddev, old_dev_sectors);	1302	ret = mddev->pers->resize(mddev, old_dev_sectors);
1303	if (!ret)	1303	if (!ret)
1304	revalidate_disk(mddev->gendisk);	1304	revalidate_disk_size(mddev->gendisk, true);
1305	ret = __sendmsg(cinfo, &cmsg);	1305	ret = __sendmsg(cinfo, &cmsg);
1306	if (ret)	1306	if (ret)
1307	pr_err("%s:%d: failed to send METADATA_UPDATED msg\n",	1307	pr_err("%s:%d: failed to send METADATA_UPDATED msg\n",

drivers/md/md-linear.c+1−1

@@ -202,7 +202,7 @@ static int linear_add(struct mddev mddev, struct md_rdev rdev) · 201 unmodified lines
202	md_set_array_sectors(mddev, linear_size(mddev, 0, 0));	202	md_set_array_sectors(mddev, linear_size(mddev, 0, 0));
203	set_capacity(mddev->gendisk, mddev->array_sectors);	203	set_capacity(mddev->gendisk, mddev->array_sectors);
204	mddev_resume(mddev);	204	mddev_resume(mddev);
205	revalidate_disk(mddev->gendisk);	205	revalidate_disk_size(mddev->gendisk, true);
206	kfree_rcu(oldconf, rcu);	206	kfree_rcu(oldconf, rcu);
207	return 0;	207	return 0;
208	}	208	}

drivers/md/md.c+10−10

 	bio_end_io_t *orig_bi_end_io;
 	void *orig_bi_private;
 	unsigned long start_time;
+	struct hd_struct *part;
 };
 static void md_end_io(struct bio *bio)
 	struct md_io *md_io = bio->bi_private;
 	struct mddev *mddev = md_io->mddev;
-	disk_end_io_acct(mddev->gendisk, bio_op(bio), md_io->start_time);
+	part_end_io_acct(md_io->part, bio, md_io->start_time);
 	bio->bi_end_io = md_io->orig_bi_end_io;
 	bio->bi_private = md_io->orig_bi_private;
 		bio->bi_end_io = md_end_io;
 		bio->bi_private = md_io;
-		md_io->start_time = disk_start_io_acct(mddev->gendisk,
+		md_io->start_time = part_start_io_acct(mddev->gendisk,
-						       bio_sectors(bio),
+						       &md_io->part, bio);
-						       bio_op(bio));
 	}
 	/* bio could be mergeable after passing to underlayer */
 		mddev->array_sectors = sectors;
 		if (mddev->pers) {
 			set_capacity(mddev->gendisk, mddev->array_sectors);
-			revalidate_disk(mddev->gendisk);
+			revalidate_disk_size(mddev->gendisk, true);
 		}
 	}
 	mddev_unlock(mddev);
 	md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
 	set_capacity(mddev->gendisk, mddev->array_sectors);
-	revalidate_disk(mddev->gendisk);
+	revalidate_disk_size(mddev->gendisk, true);
 	clear_bit(MD_NOT_READY, &mddev->flags);
 	mddev->changed = 1;
 	kobject_uevent(&disk_to_dev(mddev->gendisk)->kobj, KOBJ_CHANGE);
 		set_capacity(disk, 0);
 		mutex_unlock(&mddev->open_mutex);
 		mddev->changed = 1;
-		revalidate_disk(disk);
+		revalidate_disk_size(disk, true);
 		if (mddev->ro)
 			mddev->ro = 0;
 			md_cluster_ops->update_size(mddev, old_dev_sectors);
 		else if (mddev->queue) {
 			set_capacity(mddev->gendisk, mddev->array_sectors);
-			revalidate_disk(mddev->gendisk);
+			revalidate_disk_size(mddev->gendisk, true);
 		}
 	}
 	return rv;
 	atomic_inc(&mddev->openers);
 	mutex_unlock(&mddev->open_mutex);
-	check_disk_change(bdev);
+	bdev_check_media_change(bdev);
  out:
 	if (err)
 		mddev_put(mddev);
 		mddev_unlock(mddev);
 		if (!mddev_is_clustered(mddev)) {
 			set_capacity(mddev->gendisk, mddev->array_sectors);
-			revalidate_disk(mddev->gendisk);
+			revalidate_disk_size(mddev->gendisk, true);
 		}
 	}

drivers/md/md.h+1−1

@@ -397,7 +397,7 @@ struct mddev { · 396 unmodified lines
397	* These locks are separate due to conflicting interactions	397	* These locks are separate due to conflicting interactions
398	* with bdev->bd_mutex.	398	* with bdev->bd_mutex.
399	* Lock ordering is:	399	* Lock ordering is:
400	* reconfig_mutex -> bd_mutex : e.g. do_md_run -> revalidate_disk	400	* reconfig_mutex -> bd_mutex
401	* bd_mutex -> open_mutex: e.g. __blkdev_get -> md_open	401	* bd_mutex -> open_mutex: e.g. __blkdev_get -> md_open
402	*/	402	*/
403	struct mutex open_mutex;	403	struct mutex open_mutex;

drivers/md/raid0.c+0−16

@@ -410,22 +410,6 @@ static int raid0_run(struct mddev *mddev) · 409 unmodified lines
410	mdname(mddev),	410	mdname(mddev),
411	(unsigned long long)mddev->array_sectors);	411	(unsigned long long)mddev->array_sectors);
412		412
413	if (mddev->queue) {
414	/* calculate the max read-ahead size.
415	* For read-ahead of large files to be effective, we need to
416	* readahead at least twice a whole stripe. i.e. number of devices
417	* multiplied by chunk size times 2.
418	* If an individual device has an ra_pages greater than the
419	* chunk size, then we will not drive that device as hard as it
420	* wants. We consider this a configuration error: a larger
421	* chunksize should be used in that case.
422	*/
423	int stripe = mddev->raid_disks *
424	(mddev->chunk_sectors << 9) / PAGE_SIZE;
425	if (mddev->queue->backing_dev_info->ra_pages < 2* stripe)
426	mddev->queue->backing_dev_info->ra_pages = 2* stripe;
427	}
428
429	dump_zones(mddev);	413	dump_zones(mddev);
430		414
431	ret = md_integrity_register(mddev);	415	ret = md_integrity_register(mddev);

drivers/md/raid10.c+15−31

 	return ERR_PTR(err);
 }
+static void raid10_set_io_opt(struct r10conf *conf)
+{
+	int raid_disks = conf->geo.raid_disks;
+	if (!(conf->geo.raid_disks % conf->geo.near_copies))
+		raid_disks /= conf->geo.near_copies;
+	blk_queue_io_opt(conf->mddev->queue, (conf->mddev->chunk_sectors << 9) *
+			 raid_disks);
+}
 static int raid10_run(struct mddev *mddev)
 {
 	struct r10conf *conf;
-	int i, disk_idx, chunk_size;
+	int i, disk_idx;
 	struct raid10_info *disk;
 	struct md_rdev *rdev;
 	sector_t size;
 	mddev->thread = conf->thread;
 	conf->thread = NULL;
-	chunk_size = mddev->chunk_sectors << 9;
 	if (mddev->queue) {
 		blk_queue_max_discard_sectors(mddev->queue,
 					      mddev->chunk_sectors);
 		blk_queue_max_write_same_sectors(mddev->queue, 0);
 		blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
-		blk_queue_io_min(mddev->queue, chunk_size);
+		blk_queue_io_min(mddev->queue, mddev->chunk_sectors << 9);
-		if (conf->geo.raid_disks % conf->geo.near_copies)
+		raid10_set_io_opt(conf);
-			blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
-		else
-			blk_queue_io_opt(mddev->queue, chunk_size *
-					 (conf->geo.raid_disks / conf->geo.near_copies));
 	}
 	rdev_for_each(rdev, mddev) {
 	mddev->resync_max_sectors = size;
 	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
-	if (mddev->queue) {
-		int stripe = conf->geo.raid_disks *
-			((mddev->chunk_sectors << 9) / PAGE_SIZE);
-		/* Calculate max read-ahead size.
-		 * We need to readahead at least twice a whole stripe....
-		 * maybe...
-		 */
-		stripe /= conf->geo.near_copies;
-		if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
-			mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
-	}
 	if (md_integrity_register(mddev))
 		goto out_free_conf;
 	conf->reshape_safe = MaxSector;
 	spin_unlock_irq(&conf->device_lock);
-	/* read-ahead size must cover two whole stripes, which is
+	if (conf->mddev->queue)
-	 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
+		raid10_set_io_opt(conf);
-	 */
-	if (conf->mddev->queue) {
-		int stripe = conf->geo.raid_disks *
-			((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
-		stripe /= conf->geo.near_copies;
-		if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
-			conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
-	}
 	conf->fullsync = 0;
 }

drivers/md/raid5.c+13−18

 	if (!conf)
 		err = -ENODEV;
 	else if (new != conf->skip_copy) {
+		struct request_queue *q = mddev->queue;
 		mddev_suspend(mddev);
 		conf->skip_copy = new;
 		if (new)
-			mddev->queue->backing_dev_info->capabilities |=
+			blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
-				BDI_CAP_STABLE_WRITES;
 		else
-			mddev->queue->backing_dev_info->capabilities &=
+			blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
-				~BDI_CAP_STABLE_WRITES;
 		mddev_resume(mddev);
 	}
 	mddev_unlock(mddev);
 	return 0;
 }
+static void raid5_set_io_opt(struct r5conf *conf)
+{
+	blk_queue_io_opt(conf->mddev->queue, (conf->chunk_sectors << 9) *
+			 (conf->raid_disks - conf->max_degraded));
+}
 static int raid5_run(struct mddev *mddev)
 {
 	struct r5conf *conf;
 		int data_disks = conf->previous_raid_disks - conf->max_degraded;
 		int stripe = data_disks *
 			((mddev->chunk_sectors << 9) / PAGE_SIZE);
-		if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
-			mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
 		chunk_size = mddev->chunk_sectors << 9;
 		blk_queue_io_min(mddev->queue, chunk_size);
-		blk_queue_io_opt(mddev->queue, chunk_size *
+		raid5_set_io_opt(conf);
-				 (conf->raid_disks - conf->max_degraded));
 		mddev->queue->limits.raid_partial_stripes_expensive = 1;
 		/*
 		 * We can only discard a whole stripe. It doesn't make sense to
 		spin_unlock_irq(&conf->device_lock);
 		wake_up(&conf->wait_for_overlap);
-		/* read-ahead size must cover two whole stripes, which is
+		if (conf->mddev->queue)
-		 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
+			raid5_set_io_opt(conf);
-		 */
-		if (conf->mddev->queue) {
-			int data_disks = conf->raid_disks - conf->max_degraded;
-			int stripe = data_disks * ((conf->chunk_sectors << 9)
-						   / PAGE_SIZE);
-			if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
-				conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
-		}
 	}
 }

drivers/mmc/core/queue.c+1−2

@@ -472,8 +472,7 @@ int mmc_init_queue(struct mmc_queue mq, struct mmc_card card) · 471 unmodified lines
472	}	472	}
473		473
474	if (mmc_host_is_spi(host) && host->use_spi_crc)	474	if (mmc_host_is_spi(host) && host->use_spi_crc)
475	mq->queue->backing_dev_info->capabilities \|=	475	blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, mq->queue);
476	BDI_CAP_STABLE_WRITES;
477		476
478	mq->queue->queuedata = mq;	477	mq->queue->queuedata = mq;
479	blk_queue_rq_timeout(mq->queue, 60 * HZ);	478	blk_queue_rq_timeout(mq->queue, 60 * HZ);

drivers/mtd/mtdcore.c+2−0

@@ -2196,6 +2196,8 @@ static struct backing_dev_info * __init mtd_bdi_init(char *name) · 2195 unmodified lines
2196	bdi = bdi_alloc(NUMA_NO_NODE);	2196	bdi = bdi_alloc(NUMA_NO_NODE);
2197	if (!bdi)	2197	if (!bdi)
2198	return ERR_PTR(-ENOMEM);	2198	return ERR_PTR(-ENOMEM);
		2199	bdi->ra_pages = 0;
		2200	bdi->io_pages = 0;
2199		2201
2200	/*	2202	/*
2201	* We put '-0' suffix to the name to get the same name format as we	2203	* We put '-0' suffix to the name to get the same name format as we

drivers/nvdimm/blk.c+1−2

 static const struct block_device_operations nd_blk_fops = {
 	.owner = THIS_MODULE,
 	.submit_bio =  nd_blk_submit_bio,
-	.revalidate_disk = nvdimm_revalidate_disk,
 };
 static void nd_blk_release_queue(void *q)
 	set_capacity(disk, available_disk_size >> SECTOR_SHIFT);
 	device_add_disk(dev, disk, NULL);
-	revalidate_disk(disk);
+	nvdimm_check_and_set_ro(disk);
 	return 0;
 }

drivers/nvdimm/btt.c+1−4

 	.submit_bio =		btt_submit_bio,
 	.rw_page =		btt_rw_page,
 	.getgeo =		btt_getgeo,
-	.revalidate_disk =	nvdimm_revalidate_disk,
 };
 static int btt_blk_init(struct btt *btt)
 	btt->btt_disk->private_data = btt;
 	btt->btt_disk->queue = btt->btt_queue;
 	btt->btt_disk->flags = GENHD_FL_EXT_DEVT;
-	btt->btt_disk->queue->backing_dev_info->capabilities |=
-			BDI_CAP_SYNCHRONOUS_IO;
 	blk_queue_logical_block_size(btt->btt_queue, btt->sector_size);
 	blk_queue_max_hw_sectors(btt->btt_queue, UINT_MAX);
 	set_capacity(btt->btt_disk, btt->nlba * btt->sector_size >> 9);
 	device_add_disk(&btt->nd_btt->dev, btt->btt_disk, NULL);
 	btt->nd_btt->size = btt->nlba * (u64)btt->sector_size;
-	revalidate_disk(btt->btt_disk);
+	nvdimm_check_and_set_ro(btt->btt_disk);
 	return 0;
 }

drivers/nvdimm/bus.c+3−6

 }
 EXPORT_SYMBOL(__nd_driver_register);
-int nvdimm_revalidate_disk(struct gendisk *disk)
+void nvdimm_check_and_set_ro(struct gendisk *disk)
 {
 	struct device *dev = disk_to_dev(disk)->parent;
 	struct nd_region *nd_region = to_nd_region(dev->parent);
 	 * read-only if the disk is already read-only.
 	 */
 	if (disk_ro || nd_region->ro == disk_ro)
-		return 0;
+		return;
 	dev_info(dev, "%s read-only, marking %s read-only\n",
 			dev_name(&nd_region->dev), disk->disk_name);
 	set_disk_ro(disk, 1);
-	return 0;
 }
-EXPORT_SYMBOL(nvdimm_revalidate_disk);
+EXPORT_SYMBOL(nvdimm_check_and_set_ro);
 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
 		char *buf)

drivers/nvdimm/nd.h+1−1

@@ -361,7 +361,7 @@ u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region); · 360 unmodified lines
361	void nvdimm_bus_lock(struct device *dev);	361	void nvdimm_bus_lock(struct device *dev);
362	void nvdimm_bus_unlock(struct device *dev);	362	void nvdimm_bus_unlock(struct device *dev);
363	bool is_nvdimm_bus_locked(struct device *dev);	363	bool is_nvdimm_bus_locked(struct device *dev);
364	int nvdimm_revalidate_disk(struct gendisk *disk);	364	void nvdimm_check_and_set_ro(struct gendisk *disk);
365	void nvdimm_drvdata_release(struct kref *kref);	365	void nvdimm_drvdata_release(struct kref *kref);
366	void put_ndd(struct nvdimm_drvdata *ndd);	366	void put_ndd(struct nvdimm_drvdata *ndd);
367	int nd_label_reserve_dpa(struct nvdimm_drvdata *ndd);	367	int nd_label_reserve_dpa(struct nvdimm_drvdata *ndd);

drivers/nvdimm/pmem.c+1−3

 	.owner =		THIS_MODULE,
 	.submit_bio =		pmem_submit_bio,
 	.rw_page =		pmem_rw_page,
-	.revalidate_disk =	nvdimm_revalidate_disk,
 };
 static int pmem_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
 	disk->queue		= q;
 	disk->flags		= GENHD_FL_EXT_DEVT;
 	disk->private_data	= pmem;
-	disk->queue->backing_dev_info->capabilities |= BDI_CAP_SYNCHRONOUS_IO;
 	nvdimm_namespace_disk_name(ndns, disk->disk_name);
 	set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)
 			/ 512);
 	if (devm_add_action_or_reset(dev, pmem_release_disk, pmem))
 		return -ENOMEM;
-	revalidate_disk(disk);
+	nvdimm_check_and_set_ro(disk);
 	pmem->bb_state = sysfs_get_dirent(disk_to_dev(disk)->kobj.sd,
 					  "badblocks");

drivers/nvme/host/core.c+35−18

@@ -94,21 +94,34 @@ static void nvme_put_subsystem(struct nvme_subsystem *subsys); · 93 unmodified lines
94	static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,	94	static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
95	unsigned nsid);	95	unsigned nsid);
96		96
		97	static void nvme_update_bdev_size(struct gendisk *disk)
		98	{
		99	struct block_device *bdev = bdget_disk(disk, 0);
		100
		101	if (bdev) {
		102	bd_set_nr_sectors(bdev, get_capacity(disk));
		103	bdput(bdev);
		104	}
		105	}
		106
		107	/*
		108	* Prepare a queue for teardown.
		109	*
		110	* This must forcibly unquiesce queues to avoid blocking dispatch, and only set
		111	* the capacity to 0 after that to avoid blocking dispatchers that may be
		112	* holding bd_butex. This will end buffered writers dirtying pages that can't
		113	* be synced.
		114	*/
97	static void nvme_set_queue_dying(struct nvme_ns *ns)	115	static void nvme_set_queue_dying(struct nvme_ns *ns)
98	{	116	{
99	/*
100	* Revalidating a dead namespace sets capacity to 0. This will end
101	* buffered writers dirtying pages that can't be synced.
102	*/
103	if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))	117	if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
104	return;	118	return;
		119
105	blk_set_queue_dying(ns->queue);	120	blk_set_queue_dying(ns->queue);
106	/* Forcibly unquiesce queues to avoid blocking dispatch */
107	blk_mq_unquiesce_queue(ns->queue);	121	blk_mq_unquiesce_queue(ns->queue);
108	/*	122
109	* Revalidate after unblocking dispatchers that may be holding bd_butex	123	set_capacity(ns->disk, 0);
110	*/	124	nvme_update_bdev_size(ns->disk);
111	revalidate_disk(ns->disk);
112	}	125	}
113		126
114	static void nvme_queue_scan(struct nvme_ctrl *ctrl)	127	static void nvme_queue_scan(struct nvme_ctrl *ctrl)
@@ -2134,7 +2147,8 @@ static int __nvme_revalidate_disk(struct gendisk disk, struct nvme_id_ns id) · 2019 unmodified lines
2134	nvme_update_disk_info(ns->head->disk, ns, id);	2147	nvme_update_disk_info(ns->head->disk, ns, id);
2135	blk_stack_limits(&ns->head->disk->queue->limits,	2148	blk_stack_limits(&ns->head->disk->queue->limits,
2136	&ns->queue->limits, 0);	2149	&ns->queue->limits, 0);
2137	nvme_mpath_update_disk_size(ns->head->disk);	2150	blk_queue_update_readahead(ns->head->disk->queue);
		2151	nvme_update_bdev_size(ns->head->disk);
2138	}	2152	}
2139	#endif	2153	#endif
2140	return 0;	2154	return 0;
@@ -2339,7 +2353,6 @@ static const struct block_device_operations nvme_fops = { · 198 unmodified lines
2339	.open = nvme_open,	2353	.open = nvme_open,
2340	.release = nvme_release,	2354	.release = nvme_release,
2341	.getgeo = nvme_getgeo,	2355	.getgeo = nvme_getgeo,
2342	.revalidate_disk= nvme_revalidate_disk,
2343	.report_zones = nvme_report_zones,	2356	.report_zones = nvme_report_zones,
2344	.pr_ops = &nvme_pr_ops,	2357	.pr_ops = &nvme_pr_ops,
2345	};	2358	};
@@ -3924,8 +3937,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) · 1578 unmodified lines
3924	goto out_free_ns;	3937	goto out_free_ns;
3925		3938
3926	if (ctrl->opts && ctrl->opts->data_digest)	3939	if (ctrl->opts && ctrl->opts->data_digest)
3927	ns->queue->backing_dev_info->capabilities	3940	blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
3928	\|= BDI_CAP_STABLE_WRITES;
3929		3941
3930	blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);	3942	blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3931	if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)	3943	if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
@@ -4051,14 +4063,19 @@ static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid) · 119 unmodified lines
4051	static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)	4063	static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4052	{	4064	{
4053	struct nvme_ns *ns;	4065	struct nvme_ns *ns;
		4066	int ret;
4054		4067
4055	ns = nvme_find_get_ns(ctrl, nsid);	4068	ns = nvme_find_get_ns(ctrl, nsid);
4056	if (ns) {	4069	if (!ns) {
4057	if (revalidate_disk(ns->disk))
4058	nvme_ns_remove(ns);
4059	nvme_put_ns(ns);
4060	} else
4061	nvme_alloc_ns(ctrl, nsid);	4070	nvme_alloc_ns(ctrl, nsid);
		4071	return;
		4072	}
		4073
		4074	ret = nvme_revalidate_disk(ns->disk);
		4075	revalidate_disk_size(ns->disk, ret == 0);
		4076	if (ret)
		4077	nvme_ns_remove(ns);
		4078	nvme_put_ns(ns);
4062	}	4079	}
4063		4080
4064	static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,	4081	static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,

drivers/nvme/host/multipath.c+3−7

@@ -673,13 +673,9 @@ void nvme_mpath_add_disk(struct nvme_ns ns, struct nvme_id_ns id) · 672 unmodified lines
673	nvme_mpath_set_live(ns);	673	nvme_mpath_set_live(ns);
674	}	674	}
675		675
676	if (bdi_cap_stable_pages_required(ns->queue->backing_dev_info)) {	676	if (blk_queue_stable_writes(ns->queue) && ns->head->disk)
677	struct gendisk *disk = ns->head->disk;	677	blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES,
678		678	ns->head->disk->queue);
679	if (disk)
680	disk->queue->backing_dev_info->capabilities \|=
681	BDI_CAP_STABLE_WRITES;
682	}
683	}	679	}
684		680
685	void nvme_mpath_remove_disk(struct nvme_ns_head *head)	681	void nvme_mpath_remove_disk(struct nvme_ns_head *head)

drivers/nvme/host/nvme.h+0−13

@@ -682,16 +682,6 @@ static inline void nvme_trace_bio_complete(struct request *req, · 681 unmodified lines
682	trace_block_bio_complete(ns->head->disk->queue, req->bio);	682	trace_block_bio_complete(ns->head->disk->queue, req->bio);
683	}	683	}
684		684
685	static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
686	{
687	struct block_device *bdev = bdget_disk(disk, 0);
688
689	if (bdev) {
690	bd_set_size(bdev, get_capacity(disk) << SECTOR_SHIFT);
691	bdput(bdev);
692	}
693	}
694
695	extern struct device_attribute dev_attr_ana_grpid;	685	extern struct device_attribute dev_attr_ana_grpid;
696	extern struct device_attribute dev_attr_ana_state;	686	extern struct device_attribute dev_attr_ana_state;
697	extern struct device_attribute subsys_attr_iopolicy;	687	extern struct device_attribute subsys_attr_iopolicy;
@@ -766,9 +756,6 @@ static inline void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) · 68 unmodified lines
766	static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)	756	static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
767	{	757	{
768	}	758	}
769	static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
770	{
771	}
772	#endif /* CONFIG_NVME_MULTIPATH */	759	#endif /* CONFIG_NVME_MULTIPATH */
773		760
774	#ifdef CONFIG_BLK_DEV_ZONED	761	#ifdef CONFIG_BLK_DEV_ZONED

drivers/s390/block/dasd_genhd.c+4−11

 	struct block_device *bdev;
 	int rc;
-	bdev = bdget_disk(block->gdp, 0);
+	bdev = blkdev_get_by_dev(disk_devt(block->gdp), FMODE_READ, NULL);
-	if (!bdev) {
+	if (IS_ERR(bdev)) {
-		DBF_DEV_EVENT(DBF_ERR, block->base, "%s",
-			      "scan partitions error, bdget returned NULL");
-		return -ENODEV;
-	}
-	rc = blkdev_get(bdev, FMODE_READ, NULL);
-	if (rc < 0) {
 		DBF_DEV_EVENT(DBF_ERR, block->base,
-			      "scan partitions error, blkdev_get returned %d",
+			      "scan partitions error, blkdev_get returned %ld",
-			      rc);
+			      PTR_ERR(bdev));
 		return -ENODEV;
 	}

drivers/s390/block/dasd_ioctl.c+2−7

 	dasd_enable_device(base);
 	/* Formatting the dasd device can change the capacity. */
-	mutex_lock(&bdev->bd_mutex);
+	bd_set_nr_sectors(bdev, get_capacity(base->block->gdp));
-	i_size_write(bdev->bd_inode,
-		     (loff_t)get_capacity(base->block->gdp) << 9);
-	mutex_unlock(&bdev->bd_mutex);
 	dasd_put_device(base);
 	return 0;
 }
 	 * Set i_size to zero, since read, write, etc. check against this
 	 * value.
 	 */
-	mutex_lock(&bdev->bd_mutex);
+	bd_set_nr_sectors(bdev, 0);
-	i_size_write(bdev->bd_inode, 0);
-	mutex_unlock(&bdev->bd_mutex);
 	dasd_put_device(base);
 	return 0;
 }

drivers/scsi/iscsi_tcp.c+2−2

@@ -962,8 +962,8 @@ static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev) · 961 unmodified lines
962	struct iscsi_conn *conn = session->leadconn;	962	struct iscsi_conn *conn = session->leadconn;
963		963
964	if (conn->datadgst_en)	964	if (conn->datadgst_en)
965	sdev->request_queue->backing_dev_info->capabilities	965	blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES,
966	\|= BDI_CAP_STABLE_WRITES;	966	sdev->request_queue);
967	blk_queue_dma_alignment(sdev->request_queue, 0);	967	blk_queue_dma_alignment(sdev->request_queue, 0);
968	return 0;	968	return 0;
969	}	969	}

drivers/scsi/sd.c+8−5

 			sd_print_sense_hdr(sdkp, &sshdr);
 		return -EINVAL;
 	}
-	revalidate_disk(sdkp->disk);
+	sd_revalidate_disk(sdkp->disk);
 	return count;
 }
 	if (!scsi_block_when_processing_errors(sdev))
 		goto error_out;
-	if (sdev->removable || sdkp->write_prot)
+	if (sdev->removable || sdkp->write_prot) {
-		check_disk_change(bdev);
+		if (bdev_check_media_change(bdev))
+			sd_revalidate_disk(bdev->bd_disk);
+	}
 	/*
 	 * If the drive is empty, just let the open fail.
 static void sd_rescan(struct device *dev)
 {
 	struct scsi_disk *sdkp = dev_get_drvdata(dev);
+	int ret;
-	revalidate_disk(sdkp->disk);
+	ret = sd_revalidate_disk(sdkp->disk);
+	revalidate_disk_size(sdkp->disk, ret == 0);
 }
 static int sd_ioctl(struct block_device *bdev, fmode_t mode,
 	.compat_ioctl		= sd_compat_ioctl,
 #endif
 	.check_events		= sd_check_events,
-	.revalidate_disk	= sd_revalidate_disk,
 	.unlock_native_capacity	= sd_unlock_native_capacity,
 	.report_zones		= sd_zbc_report_zones,
 	.pr_ops			= &sd_pr_ops,

drivers/scsi/sr.c+14−22

 	return ret;
 }
+static void sr_revalidate_disk(struct scsi_cd *cd)
+{
+	struct scsi_sense_hdr sshdr;
+	/* if the unit is not ready, nothing more to do */
+	if (scsi_test_unit_ready(cd->device, SR_TIMEOUT, MAX_RETRIES, &sshdr))
+		return;
+	sr_cd_check(&cd->cdi);
+	get_sectorsize(cd);
+}
 static int sr_block_open(struct block_device *bdev, fmode_t mode)
 {
 	struct scsi_cd *cd;
 	sdev = cd->device;
 	scsi_autopm_get_device(sdev);
-	check_disk_change(bdev);
+	if (bdev_check_media_change(bdev))
+		sr_revalidate_disk(cd);
 	mutex_lock(&cd->lock);
 	ret = cdrom_open(&cd->cdi, bdev, mode);
 	return ret;
 }
-static int sr_block_revalidate_disk(struct gendisk *disk)
-{
-	struct scsi_sense_hdr sshdr;
-	struct scsi_cd *cd;
-	cd = scsi_cd_get(disk);
-	if (!cd)
-		return -ENXIO;
-	/* if the unit is not ready, nothing more to do */
-	if (scsi_test_unit_ready(cd->device, SR_TIMEOUT, MAX_RETRIES, &sshdr))
-		goto out;
-	sr_cd_check(&cd->cdi);
-	get_sectorsize(cd);
-out:
-	scsi_cd_put(cd);
-	return 0;
-}
 static const struct block_device_operations sr_bdops =
 {
 	.owner		= THIS_MODULE,
 	.compat_ioctl	= sr_block_compat_ioctl,
 #endif
 	.check_events	= sr_block_check_events,
-	.revalidate_disk = sr_block_revalidate_disk,
 };
 static int sr_open(struct cdrom_device_info *cdi, int purpose)
 	dev_set_drvdata(dev, cd);
 	disk->flags |= GENHD_FL_REMOVABLE;
+	sr_revalidate_disk(cd);
 	device_add_disk(&sdev->sdev_gendev, disk, NULL);
 	sdev_printk(KERN_DEBUG, sdev,

fs/9p/vfs_file.c+1−1

@@ -625,7 +625,7 @@ static void v9fs_mmap_vm_close(struct vm_area_struct *vma) · 624 unmodified lines
625		625
626	inode = file_inode(vma->vm_file);	626	inode = file_inode(vma->vm_file);
627		627
628	if (!mapping_cap_writeback_dirty(inode->i_mapping))	628	if (!mapping_can_writeback(inode->i_mapping))
629	wbc.nr_to_write = 0;	629	wbc.nr_to_write = 0;
630		630
631	might_sleep();	631	might_sleep();

fs/9p/vfs_super.c+4−2

 	if (ret)
 		return ret;
-	if (v9ses->cache)
+	if (!v9ses->cache) {
-		sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
+		sb->s_bdi->ra_pages = 0;
+		sb->s_bdi->io_pages = 0;
+	}
 	sb->s_flags |= SB_ACTIVE | SB_DIRSYNC;
 	if (!v9ses->cache)

fs/afs/super.c+0−1

@@ -456,7 +456,6 @@ static int afs_fill_super(struct super_block sb, struct afs_fs_context ctx) · 455 unmodified lines
456	ret = super_setup_bdi(sb);	456	ret = super_setup_bdi(sb);
457	if (ret)	457	if (ret)
458	return ret;	458	return ret;
459	sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
460		459
461	/* allocate the root inode and dentry */	460	/* allocate the root inode and dentry */
462	if (as->dyn_root) {	461	if (as->dyn_root) {

fs/block_dev.c+85−90

 }
 EXPORT_SYMBOL(invalidate_bdev);
+/*
+ * Drop all buffers & page cache for given bdev range. This function bails
+ * with error if bdev has other exclusive owner (such as filesystem).
+ */
+int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
+			loff_t lstart, loff_t lend)
+{
+	struct block_device *claimed_bdev = NULL;
+	int err;
+	/*
+	 * If we don't hold exclusive handle for the device, upgrade to it
+	 * while we discard the buffer cache to avoid discarding buffers
+	 * under live filesystem.
+	 */
+	if (!(mode & FMODE_EXCL)) {
+		claimed_bdev = bdev->bd_contains;
+		err = bd_prepare_to_claim(bdev, claimed_bdev,
+					  truncate_bdev_range);
+		if (err)
+			return err;
+	}
+	truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend);
+	if (claimed_bdev)
+		bd_abort_claiming(bdev, claimed_bdev, truncate_bdev_range);
+	return 0;
+}
+EXPORT_SYMBOL(truncate_bdev_range);
 static void set_init_blocksize(struct block_device *bdev)
 {
 	bdev->bd_inode->i_blkbits = blksize_bits(bdev_logical_block_size(bdev));
 	bdev = &BDEV_I(inode)->bdev;
 	if (inode->i_state & I_NEW) {
+		spin_lock_init(&bdev->bd_size_lock);
 		bdev->bd_contains = NULL;
 		bdev->bd_super = NULL;
 		bdev->bd_inode = inode;
 		bdev->bd_part_count = 0;
-		bdev->bd_invalidated = 0;
 		inode->i_mode = S_IFBLK;
 		inode->i_rdev = dev;
 		inode->i_bdev = bdev;
 {
 	loff_t disk_size, bdev_size;
+	spin_lock(&bdev->bd_size_lock);
 	disk_size = (loff_t)get_capacity(disk) << 9;
 	bdev_size = i_size_read(bdev->bd_inode);
 	if (disk_size != bdev_size) {
 			       disk->disk_name, bdev_size, disk_size);
 		}
 		i_size_write(bdev->bd_inode, disk_size);
-		if (bdev_size > disk_size && __invalidate_device(bdev, false))
+	}
+	spin_unlock(&bdev->bd_size_lock);
+	if (bdev_size > disk_size) {
+		if (__invalidate_device(bdev, false))
 			pr_warn("VFS: busy inodes on resized disk %s\n",
 				disk->disk_name);
 	}
-	bdev->bd_invalidated = 0;
 }
 /**
- * revalidate_disk - wrapper for lower-level driver's revalidate_disk call-back
+ * revalidate_disk_size - checks for disk size change and adjusts bdev size.
- * @disk: struct gendisk to be revalidated
+ * @disk: struct gendisk to check
+ * @verbose: if %true log a message about a size change if there is any
  *
- * This routine is a wrapper for lower-level driver's revalidate_disk
+ * This routine checks to see if the bdev size does not match the disk size
- * call-backs.  It is used to do common pre and post operations needed
+ * and adjusts it if it differs. When shrinking the bdev size, its all caches
- * for all revalidate_disk operations.
+ * are freed.
  */
-int revalidate_disk(struct gendisk *disk)
+void revalidate_disk_size(struct gendisk *disk, bool verbose)
 {
-	int ret = 0;
+	struct block_device *bdev;
-	if (disk->fops->revalidate_disk)
-		ret = disk->fops->revalidate_disk(disk);
 	/*
 	 * Hidden disks don't have associated bdev so there's no point in
-	 * revalidating it.
+	 * revalidating them.
 	 */
-	if (!(disk->flags & GENHD_FL_HIDDEN)) {
+	if (disk->flags & GENHD_FL_HIDDEN)
-		struct block_device *bdev = bdget_disk(disk, 0);
+		return;
-		if (!bdev)
-			return ret;
-		mutex_lock(&bdev->bd_mutex);
+	bdev = bdget_disk(disk, 0);
-		check_disk_size_change(disk, bdev, ret == 0);
+	if (bdev) {
-		mutex_unlock(&bdev->bd_mutex);
+		check_disk_size_change(disk, bdev, verbose);
 		bdput(bdev);
 	}
-	return ret;
 }
-EXPORT_SYMBOL(revalidate_disk);
+EXPORT_SYMBOL(revalidate_disk_size);
-/*
+void bd_set_nr_sectors(struct block_device *bdev, sector_t sectors)
- * This routine checks whether a removable media has been changed,
- * and invalidates all buffer-cache-entries in that case. This
- * is a relatively slow routine, so we have to try to minimize using
- * it. Thus it is called only upon a 'mount' or 'open'. This
- * is the best way of combining speed and utility, I think.
- * People changing diskettes in the middle of an operation deserve
- * to lose :-)
- */
-int check_disk_change(struct block_device *bdev)
 {
-	struct gendisk *disk = bdev->bd_disk;
+	spin_lock(&bdev->bd_size_lock);
-	const struct block_device_operations *bdops = disk->fops;
+	i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT);
-	unsigned int events;
+	spin_unlock(&bdev->bd_size_lock);
-	events = disk_clear_events(disk, DISK_EVENT_MEDIA_CHANGE |
-				   DISK_EVENT_EJECT_REQUEST);
-	if (!(events & DISK_EVENT_MEDIA_CHANGE))
-		return 0;
-	if (__invalidate_device(bdev, true))
-		pr_warn("VFS: busy inodes on changed media %s\n",
-			disk->disk_name);
-	bdev->bd_invalidated = 1;
-	if (bdops->revalidate_disk)
-		bdops->revalidate_disk(bdev->bd_disk);
-	return 1;
-}
-EXPORT_SYMBOL(check_disk_change);
-void bd_set_size(struct block_device *bdev, loff_t size)
-{
-	inode_lock(bdev->bd_inode);
-	i_size_write(bdev->bd_inode, size);
-	inode_unlock(bdev->bd_inode);
 }
-EXPORT_SYMBOL(bd_set_size);
+EXPORT_SYMBOL(bd_set_nr_sectors);
 static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
 	lockdep_assert_held(&bdev->bd_mutex);
+	clear_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state);
 rescan:
 	ret = blk_drop_partitions(bdev);
 	if (ret)
 	struct gendisk *disk;
 	int ret;
 	int partno;
-	int perm = 0;
 	bool first_open = false, unblock_events = true, need_restart;
-	if (mode & FMODE_READ)
-		perm |= MAY_READ;
-	if (mode & FMODE_WRITE)
-		perm |= MAY_WRITE;
-	/*
-	 * hooks: /n/, see "layering violations".
-	 */
-	if (!for_part) {
-		ret = devcgroup_inode_permission(bdev->bd_inode, perm);
-		if (ret != 0)
-			return ret;
-	}
  restart:
 	need_restart = false;
 	ret = -ENXIO;
 			}
 			if (!ret) {
-				bd_set_size(bdev,(loff_t)get_capacity(disk)<<9);
+				bd_set_nr_sectors(bdev, get_capacity(disk));
 				set_init_blocksize(bdev);
 			}
 			 * The latter is necessary to prevent ghost
 			 * partitions on a removed medium.
 			 */
-			if (bdev->bd_invalidated &&
+			if (test_bit(GD_NEED_PART_SCAN, &disk->state) &&
 			    (!ret || ret == -ENOMEDIUM))
 				bdev_disk_changed(bdev, ret == -ENOMEDIUM);
 				ret = -ENXIO;
 				goto out_clear;
 			}
-			bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9);
+			bd_set_nr_sectors(bdev, bdev->bd_part->nr_sects);
 			set_init_blocksize(bdev);
 		}
 			if (bdev->bd_disk->fops->open)
 				ret = bdev->bd_disk->fops->open(bdev, mode);
 			/* the same as first opener case, read comment there */
-			if (bdev->bd_invalidated &&
+			if (test_bit(GD_NEED_PART_SCAN, &disk->state) &&
 			    (!ret || ret == -ENOMEDIUM))
 				bdev_disk_changed(bdev, ret == -ENOMEDIUM);
 			if (ret)
  * RETURNS:
  * 0 on success, -errno on failure.
  */
-int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
+static int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
 {
-	int res;
+	int ret, perm = 0;
-	res =__blkdev_get(bdev, mode, holder, 0);
+	if (mode & FMODE_READ)
-	if (res)
+		perm |= MAY_READ;
-		bdput(bdev);
+	if (mode & FMODE_WRITE)
-	return res;
+		perm |= MAY_WRITE;
+	ret = devcgroup_inode_permission(bdev->bd_inode, perm);
+	if (ret)
+		goto bdput;
+	ret =__blkdev_get(bdev, mode, holder, 0);
+	if (ret)
+		goto bdput;
+	return 0;
+bdput:
+	bdput(bdev);
+	return ret;
 }
-EXPORT_SYMBOL(blkdev_get);
 /**
  * blkdev_get_by_path - open a block device by name
 	if (bdev_read_only(I_BDEV(bd_inode)))
 		return -EPERM;
-	if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode))
+	if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode->i_rdev))
 		return -ETXTBSY;
 	if (!iov_iter_count(from))
 			     loff_t len)
 {
 	struct block_device *bdev = I_BDEV(bdev_file_inode(file));
-	struct address_space *mapping;
 	loff_t end = start + len - 1;
 	loff_t isize;
 	int error;
 		return -EINVAL;
 	/* Invalidate the page cache, including dirty pages. */
-	mapping = bdev->bd_inode->i_mapping;
+	error = truncate_bdev_range(bdev, file->f_mode, start, end);
-	truncate_inode_pages_range(mapping, start, end);
+	if (error)
+		return error;
 	switch (mode) {
 	case FALLOC_FL_ZERO_RANGE:
 	 * the caller will be given -EBUSY.  The third argument is
 	 * inclusive, so the rounding here is safe.
 	 */
-	return invalidate_inode_pages2_range(mapping,
+	return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
 					     start >> PAGE_SHIFT,
 					     end >> PAGE_SHIFT);
 }

fs/btrfs/disk-io.c+0−2

@@ -3091,8 +3091,6 @@ int __cold open_ctree(struct super_block sb, struct btrfs_fs_devices fs_device · 3090 unmodified lines
3091	goto fail_sb_buffer;	3091	goto fail_sb_buffer;
3092	}	3092	}
3093		3093
3094	sb->s_bdi->capabilities \|= BDI_CAP_CGROUP_WRITEBACK;
3095	sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
3096	sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);	3094	sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
3097	sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);	3095	sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
3098		3096

fs/buffer.c+0−16

@@ -2771,16 +2771,6 @@ int nobh_writepage(struct page page, get_block_t get_block, · 2770 unmodified lines
2771	/* Is the page fully outside i_size? (truncate in progress) */	2771	/* Is the page fully outside i_size? (truncate in progress) */
2772	offset = i_size & (PAGE_SIZE-1);	2772	offset = i_size & (PAGE_SIZE-1);
2773	if (page->index >= end_index+1 \|\| !offset) {	2773	if (page->index >= end_index+1 \|\| !offset) {
2774	/*
2775	* The page may have dirty, unmapped buffers. For example,
2776	* they may have been added in ext3_writepage(). Make them
2777	* freeable here, so the page does not leak.
2778	*/
2779	#if 0
2780	/* Not really sure about this - do we need this ? */
2781	if (page->mapping->a_ops->invalidatepage)
2782	page->mapping->a_ops->invalidatepage(page, offset);
2783	#endif
2784	unlock_page(page);	2774	unlock_page(page);
2785	return 0; /* don't care */	2775	return 0; /* don't care */
2786	}	2776	}
@@ -2975,12 +2965,6 @@ int block_write_full_page(struct page page, get_block_t get_block, · 188 unmodified lines
2975	/* Is the page fully outside i_size? (truncate in progress) */	2965	/* Is the page fully outside i_size? (truncate in progress) */
2976	offset = i_size & (PAGE_SIZE-1);	2966	offset = i_size & (PAGE_SIZE-1);
2977	if (page->index >= end_index+1 \|\| !offset) {	2967	if (page->index >= end_index+1 \|\| !offset) {
2978	/*
2979	* The page may have dirty, unmapped buffers. For example,
2980	* they may have been added in ext3_writepage(). Make them
2981	* freeable here, so the page does not leak.
2982	*/
2983	do_invalidatepage(page, 0, PAGE_SIZE);
2984	unlock_page(page);	2968	unlock_page(page);
2985	return 0; /* don't care */	2969	return 0; /* don't care */
2986	}	2970	}

fs/fs-writeback.c+4−3

 			wb = locked_inode_to_wb_and_lock_list(inode);
-			WARN(bdi_cap_writeback_dirty(wb->bdi) &&
+			WARN((wb->bdi->capabilities & BDI_CAP_WRITEBACK) &&
 			     !test_bit(WB_registered, &wb->state),
 			     "bdi-%s not registered\n", bdi_dev_name(wb->bdi));
 			 * to make sure background write-back happens
 			 * later.
 			 */
-			if (bdi_cap_writeback_dirty(wb->bdi) && wakeup_bdi)
+			if (wakeup_bdi &&
+			    (wb->bdi->capabilities & BDI_CAP_WRITEBACK))
 				wb_wakeup_delayed(wb);
 			return;
 		}
 		.range_end = LLONG_MAX,
 	};
-	if (!mapping_cap_writeback_dirty(inode->i_mapping))
+	if (!mapping_can_writeback(inode->i_mapping))
 		wbc.nr_to_write = 0;
 	might_sleep();

fs/fuse/inode.c+2−2

 	if (err)
 		return err;
-	sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
 	/* fuse does it's own writeback accounting */
-	sb->s_bdi->capabilities = BDI_CAP_NO_ACCT_WB | BDI_CAP_STRICTLIMIT;
+	sb->s_bdi->capabilities &= ~BDI_CAP_WRITEBACK_ACCT;
+	sb->s_bdi->capabilities |= BDI_CAP_STRICTLIMIT;
 	/*
 	 * For a single fuse filesystem use max 1% of dirty +

fs/namei.c+2−2

@@ -568,8 +568,8 @@ static bool path_connected(struct vfsmount mnt, struct dentry dentry) · 567 unmodified lines
568	{	568	{
569	struct super_block *sb = mnt->mnt_sb;	569	struct super_block *sb = mnt->mnt_sb;
570		570
571	/* Bind mounts and multi-root filesystems can have disconnected paths */	571	/* Bind mounts can have disconnected paths */
572	if (!(sb->s_iflags & SB_I_MULTIROOT) && (mnt->mnt_root == sb->s_root))	572	if (mnt->mnt_root == sb->s_root)
573	return true;	573	return true;
574		574
575	return is_subdir(dentry, mnt->mnt_root);	575	return is_subdir(dentry, mnt->mnt_root);

fs/nfs/super.c+1−8

 }
 #endif
-static void nfs_set_readahead(struct backing_dev_info *bdi,
-			      unsigned long iomax_pages)
-{
-	bdi->ra_pages = VM_READAHEAD_PAGES;
-	bdi->io_pages = iomax_pages;
-}
 int nfs_get_tree_common(struct fs_context *fc)
 {
 	struct nfs_fs_context *ctx = nfs_fc2context(fc);
 					     MINOR(server->s_dev));
 		if (error)
 			goto error_splat_super;
-		nfs_set_readahead(s->s_bdi, server->rpages);
+		s->s_bdi->io_pages = server->rpages;
 		server->super = s;
 	}

fs/ocfs2/cluster/heartbeat.c+10−18

@@ -1766,7 +1766,6 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, · 1765 unmodified lines
1766	int sectsize;	1766	int sectsize;
1767	char p = (char )page;	1767	char p = (char )page;
1768	struct fd f;	1768	struct fd f;
1769	struct inode *inode;
1770	ssize_t ret = -EINVAL;	1769	ssize_t ret = -EINVAL;
1771	int live_threshold;	1770	int live_threshold;
1772		1771
@@ -1793,20 +1792,16 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, · 20 unmodified lines
1793	reg->hr_block_bytes == 0)	1792	reg->hr_block_bytes == 0)
1794	goto out2;	1793	goto out2;
1795		1794
1796	inode = igrab(f.file->f_mapping->host);	1795	if (!S_ISBLK(f.file->f_mapping->host->i_mode))
1797	if (inode == NULL)
1798	goto out2;	1796	goto out2;
1799		1797
1800	if (!S_ISBLK(inode->i_mode))	1798	reg->hr_bdev = blkdev_get_by_dev(f.file->f_mapping->host->i_rdev,
1801	goto out3;	1799	FMODE_WRITE \| FMODE_READ, NULL);
1802		1800	if (IS_ERR(reg->hr_bdev)) {
1803	reg->hr_bdev = I_BDEV(f.file->f_mapping->host);	1801	ret = PTR_ERR(reg->hr_bdev);
1804	ret = blkdev_get(reg->hr_bdev, FMODE_WRITE \| FMODE_READ, NULL);
1805	if (ret) {
1806	reg->hr_bdev = NULL;	1802	reg->hr_bdev = NULL;
1807	goto out3;	1803	goto out2;
1808	}	1804	}
1809	inode = NULL;
1810		1805
1811	bdevname(reg->hr_bdev, reg->hr_dev_name);	1806	bdevname(reg->hr_bdev, reg->hr_dev_name);
1812		1807
@@ -1909,16 +1904,13 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, · 96 unmodified lines
1909	config_item_name(&reg->hr_item), reg->hr_dev_name);	1904	config_item_name(&reg->hr_item), reg->hr_dev_name);
1910		1905
1911	out3:	1906	out3:
1912	iput(inode);	1907	if (ret < 0) {
		1908	blkdev_put(reg->hr_bdev, FMODE_READ \| FMODE_WRITE);
		1909	reg->hr_bdev = NULL;
		1910	}
1913	out2:	1911	out2:
1914	fdput(f);	1912	fdput(f);
1915	out:	1913	out:
1916	if (ret < 0) {
1917	if (reg->hr_bdev) {
1918	blkdev_put(reg->hr_bdev, FMODE_READ\|FMODE_WRITE);
1919	reg->hr_bdev = NULL;
1920	}
1921	}
1922	return ret;	1914	return ret;
1923	}	1915	}
1924		1916

fs/super.c+2−0

@@ -1256,6 +1256,8 @@ static int set_bdev_super(struct super_block s, void data) · 1255 unmodified lines
1256	s->s_dev = s->s_bdev->bd_dev;	1256	s->s_dev = s->s_bdev->bd_dev;
1257	s->s_bdi = bdi_get(s->s_bdev->bd_bdi);	1257	s->s_bdi = bdi_get(s->s_bdev->bd_bdi);
1258		1258
		1259	if (blk_queue_stable_writes(s->s_bdev->bd_disk->queue))
		1260	s->s_iflags \|= SB_I_STABLE_WRITES;
1259	return 0;	1261	return 0;
1260	}	1262	}
1261		1263

fs/ubifs/super.c+2−0

@@ -2177,6 +2177,8 @@ static int ubifs_fill_super(struct super_block sb, void data, int silent) · 2176 unmodified lines
2177	c->vi.vol_id);	2177	c->vi.vol_id);
2178	if (err)	2178	if (err)
2179	goto out_close;	2179	goto out_close;
		2180	sb->s_bdi->ra_pages = 0;
		2181	sb->s_bdi->io_pages = 0;
2180		2182
2181	sb->s_fs_info = c;	2183	sb->s_fs_info = c;
2182	sb->s_magic = UBIFS_SUPER_MAGIC;	2184	sb->s_magic = UBIFS_SUPER_MAGIC;

fs/vboxsf/super.c+2−0

@@ -167,6 +167,8 @@ static int vboxsf_fill_super(struct super_block sb, struct fs_context fc) · 166 unmodified lines
167	err = super_setup_bdi_name(sb, "vboxsf-%d", sbi->bdi_id);	167	err = super_setup_bdi_name(sb, "vboxsf-%d", sbi->bdi_id);
168	if (err)	168	if (err)
169	goto fail_free;	169	goto fail_free;
		170	sb->s_bdi->ra_pages = 0;
		171	sb->s_bdi->io_pages = 0;
170		172
171	/* Turn source into a shfl_string and map the folder */	173	/* Turn source into a shfl_string and map the folder */
172	size = strlen(fc->source) + 1;	174	size = strlen(fc->source) + 1;

include/linux/backing-dev.h+13−65

@@ -110,33 +110,14 @@ int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio); · 109 unmodified lines
110	/*	110	/*
111	* Flags in backing_dev_info::capability	111	* Flags in backing_dev_info::capability
112	*	112	*
113	* The first three flags control whether dirty pages will contribute to the	113	* BDI_CAP_WRITEBACK: Supports dirty page writeback, and dirty pages
114	* VM's accounting and whether writepages() should be called for dirty pages	114	* should contribute to accounting
115	* (something that would not, for example, be appropriate for ramfs)	115	* BDI_CAP_WRITEBACK_ACCT: Automatically account writeback pages
116	*	116	* BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold
117	* WARNING: these flags are closely related and should not normally be
118	* used separately. The BDI_CAP_NO_ACCT_AND_WRITEBACK combines these
119	* three flags into a single convenience macro.
120	*
121	* BDI_CAP_NO_ACCT_DIRTY: Dirty pages shouldn't contribute to accounting
122	* BDI_CAP_NO_WRITEBACK: Don't write pages back
123	* BDI_CAP_NO_ACCT_WB: Don't automatically account writeback pages
124	* BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold.
125	*
126	* BDI_CAP_CGROUP_WRITEBACK: Supports cgroup-aware writeback.
127	* BDI_CAP_SYNCHRONOUS_IO: Device is so fast that asynchronous IO would be
128	* inefficient.
129	*/	117	*/
130	#define BDI_CAP_NO_ACCT_DIRTY 0x00000001	118	#define BDI_CAP_WRITEBACK (1 << 0)
131	#define BDI_CAP_NO_WRITEBACK 0x00000002	119	#define BDI_CAP_WRITEBACK_ACCT (1 << 1)
132	#define BDI_CAP_NO_ACCT_WB 0x00000004	120	#define BDI_CAP_STRICTLIMIT (1 << 2)
133	#define BDI_CAP_STABLE_WRITES 0x00000008
134	#define BDI_CAP_STRICTLIMIT 0x00000010
135	#define BDI_CAP_CGROUP_WRITEBACK 0x00000020
136	#define BDI_CAP_SYNCHRONOUS_IO 0x00000040
137
138	#define BDI_CAP_NO_ACCT_AND_WRITEBACK \
139	(BDI_CAP_NO_WRITEBACK \| BDI_CAP_NO_ACCT_DIRTY \| BDI_CAP_NO_ACCT_WB)
140		121
141	extern struct backing_dev_info noop_backing_dev_info;	122	extern struct backing_dev_info noop_backing_dev_info;
142		123
@@ -175,41 +156,9 @@ static inline int wb_congested(struct bdi_writeback *wb, int cong_bits) · 32 unmodified lines
175	long congestion_wait(int sync, long timeout);	156	long congestion_wait(int sync, long timeout);
176	long wait_iff_congested(int sync, long timeout);	157	long wait_iff_congested(int sync, long timeout);
177		158
178	static inline bool bdi_cap_synchronous_io(struct backing_dev_info *bdi)	159	static inline bool mapping_can_writeback(struct address_space *mapping)
179	{
180	return bdi->capabilities & BDI_CAP_SYNCHRONOUS_IO;
181	}
182
183	static inline bool bdi_cap_stable_pages_required(struct backing_dev_info *bdi)
184	{
185	return bdi->capabilities & BDI_CAP_STABLE_WRITES;
186	}
187
188	static inline bool bdi_cap_writeback_dirty(struct backing_dev_info *bdi)
189	{
190	return !(bdi->capabilities & BDI_CAP_NO_WRITEBACK);
191	}
192
193	static inline bool bdi_cap_account_dirty(struct backing_dev_info *bdi)
194	{
195	return !(bdi->capabilities & BDI_CAP_NO_ACCT_DIRTY);
196	}
197
198	static inline bool bdi_cap_account_writeback(struct backing_dev_info *bdi)
199	{
200	/* Paranoia: BDI_CAP_NO_WRITEBACK implies BDI_CAP_NO_ACCT_WB */
201	return !(bdi->capabilities & (BDI_CAP_NO_ACCT_WB \|
202	BDI_CAP_NO_WRITEBACK));
203	}
204
205	static inline bool mapping_cap_writeback_dirty(struct address_space *mapping)
206	{
207	return bdi_cap_writeback_dirty(inode_to_bdi(mapping->host));
208	}
209
210	static inline bool mapping_cap_account_dirty(struct address_space *mapping)
211	{	160	{
212	return bdi_cap_account_dirty(inode_to_bdi(mapping->host));	161	return inode_to_bdi(mapping->host)->capabilities & BDI_CAP_WRITEBACK;
213	}	162	}
214		163
215	static inline int bdi_sched_wait(void *word)	164	static inline int bdi_sched_wait(void *word)
@@ -233,9 +182,9 @@ int inode_congested(struct inode *inode, int cong_bits); · 17 unmodified lines
233	* inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode	182	* inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode
234	* @inode: inode of interest	183	* @inode: inode of interest
235	*	184	*
236	* cgroup writeback requires support from both the bdi and filesystem.	185	* Cgroup writeback requires support from the filesystem. Also, both memcg and
237	* Also, both memcg and iocg have to be on the default hierarchy. Test	186	* iocg have to be on the default hierarchy. Test whether all conditions are
238	* whether all conditions are met.	187	* met.
239	*	188	*
240	* Note that the test result may change dynamically on the same inode	189	* Note that the test result may change dynamically on the same inode
241	* depending on how memcg and iocg are configured.	190	* depending on how memcg and iocg are configured.
@@ -246,8 +195,7 @@ static inline bool inode_cgwb_enabled(struct inode *inode) · 4 unmodified lines
246		195
247	return cgroup_subsys_on_dfl(memory_cgrp_subsys) &&	196	return cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
248	cgroup_subsys_on_dfl(io_cgrp_subsys) &&	197	cgroup_subsys_on_dfl(io_cgrp_subsys) &&
249	bdi_cap_account_dirty(bdi) &&	198	(bdi->capabilities & BDI_CAP_WRITEBACK) &&
250	(bdi->capabilities & BDI_CAP_CGROUP_WRITEBACK) &&
251	(inode->i_sb->s_iflags & SB_I_CGROUPWB);	199	(inode->i_sb->s_iflags & SB_I_CGROUPWB);
252	}	200	}
253		201

include/linux/blk-mq.h+12−3

 	 * shared across request queues.
 	 */
 	atomic_t		nr_active;
+	/**
+	 * @elevator_queued: Number of queued requests on hctx.
+	 */
+	atomic_t                elevator_queued;
 	/** @cpuhp_online: List to store request if CPU is going to die */
 	struct hlist_node	cpuhp_online;
  * @flags:	   Zero or more BLK_MQ_F_* flags.
  * @driver_data:   Pointer to data owned by the block driver that created this
  *		   tag set.
+ * @__bitmap_tags: A shared tags sbitmap, used over all hctx's
+ * @__breserved_tags:
+ *		   A shared reserved tags sbitmap, used over all hctx's
  * @tags:	   Tag sets. One tag set per hardware queue. Has @nr_hw_queues
  *		   elements.
  * @tag_list_lock: Serializes tag_list accesses.
 	unsigned int		timeout;
 	unsigned int		flags;
 	void			*driver_data;
+	atomic_t		active_queues_shared_sbitmap;
+	struct sbitmap_queue	__bitmap_tags;
+	struct sbitmap_queue	__breserved_tags;
 	struct blk_mq_tags	**tags;
 	struct mutex		tag_list_lock;
 enum {
 	BLK_MQ_F_SHOULD_MERGE	= 1 << 0,
-	BLK_MQ_F_TAG_SHARED	= 1 << 1,
+	BLK_MQ_F_TAG_QUEUE_SHARED = 1 << 1,
 	/*
 	 * Set when this device requires underlying blk-mq device for
 	 * completing IO:
 	 */
 	BLK_MQ_F_STACKING	= 1 << 2,
+	BLK_MQ_F_TAG_HCTX_SHARED = 1 << 3,
 	BLK_MQ_F_BLOCKING	= 1 << 5,
 	BLK_MQ_F_NO_SCHED	= 1 << 6,
 	BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
 void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs);
 void blk_mq_complete_request(struct request *rq);
 bool blk_mq_complete_request_remote(struct request *rq);
-bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
-			   struct bio *bio, unsigned int nr_segs);
 bool blk_mq_queue_stopped(struct request_queue *q);
 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);

include/linux/blk_types.h+3−4

@@ -20,7 +20,7 @@ typedef void (bio_end_io_t) (struct bio *); · 19 unmodified lines
20	struct bio_crypt_ctx;	20	struct bio_crypt_ctx;
21		21
22	struct block_device {	22	struct block_device {
23	dev_t bd_dev; /* not a kdev_t - it's a search key */	23	dev_t bd_dev;
24	int bd_openers;	24	int bd_openers;
25	struct inode * bd_inode; /* will die */	25	struct inode * bd_inode; /* will die */
26	struct super_block * bd_super;	26	struct super_block * bd_super;
@@ -37,7 +37,8 @@ struct block_device { · 10 unmodified lines
37	struct hd_struct * bd_part;	37	struct hd_struct * bd_part;
38	/* number of times partitions within this device have been opened. */	38	/* number of times partitions within this device have been opened. */
39	unsigned bd_part_count;	39	unsigned bd_part_count;
40	int bd_invalidated;	40
		41	spinlock_t bd_size_lock; /* for bd_inode->i_size updates */
41	struct gendisk * bd_disk;	42	struct gendisk * bd_disk;
42	struct backing_dev_info *bd_bdi;	43	struct backing_dev_info *bd_bdi;
43		44
@@ -255,8 +256,6 @@ enum { · 211 unmodified lines
255	BIO_NO_PAGE_REF, /* don't put release vec pages */	256	BIO_NO_PAGE_REF, /* don't put release vec pages */
256	BIO_CLONED, /* doesn't own data */	257	BIO_CLONED, /* doesn't own data */
257	BIO_BOUNCED, /* bio is a bounce bio */	258	BIO_BOUNCED, /* bio is a bounce bio */
258	BIO_USER_MAPPED, /* contains user pages */
259	BIO_NULL_MAPPED, /* contains invalid user pages */
260	BIO_WORKINGSET, /* contains userspace workingset pages */	259	BIO_WORKINGSET, /* contains userspace workingset pages */
261	BIO_QUIET, /* Make BIO Quiet */	260	BIO_QUIET, /* Make BIO Quiet */
262	BIO_CHAIN, /* chained bio, ->bi_remaining in effect */	261	BIO_CHAIN, /* chained bio, ->bi_remaining in effect */

include/linux/blkdev.h+34−10

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	#include <linux/percpu-refcount.h>	24	#include <linux/percpu-refcount.h>
25	#include <linux/scatterlist.h>	25	#include <linux/scatterlist.h>
26	#include <linux/blkzoned.h>	26	#include <linux/blkzoned.h>
		27	#include <linux/pm.h>
27		28
28	struct module;	29	struct module;
29	struct scsi_ioctl_command;	30	struct scsi_ioctl_command;
@@ -458,7 +459,7 @@ struct request_queue { · 428 unmodified lines
458		459
459	#ifdef CONFIG_PM	460	#ifdef CONFIG_PM
460	struct device *dev;	461	struct device *dev;
461	int rpm_status;	462	enum rpm_status rpm_status;
462	unsigned int nr_pending;	463	unsigned int nr_pending;
463	#endif	464	#endif
464		465
@@ -484,6 +485,8 @@ struct request_queue { · 19 unmodified lines
484	struct timer_list timeout;	485	struct timer_list timeout;
485	struct work_struct timeout_work;	486	struct work_struct timeout_work;
486		487
		488	atomic_t nr_active_requests_shared_sbitmap;
		489
487	struct list_head icq_list;	490	struct list_head icq_list;
488	#ifdef CONFIG_BLK_CGROUP	491	#ifdef CONFIG_BLK_CGROUP
489	DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);	492	DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
@@ -603,6 +606,7 @@ struct request_queue { · 113 unmodified lines
603	#define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */	606	#define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */
604	#define QUEUE_FLAG_DEAD 13 /* queue tear-down finished */	607	#define QUEUE_FLAG_DEAD 13 /* queue tear-down finished */
605	#define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */	608	#define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */
		609	#define QUEUE_FLAG_STABLE_WRITES 15 /* don't modify blks until WB is done */
606	#define QUEUE_FLAG_POLL 16 /* IO polling enabled if set */	610	#define QUEUE_FLAG_POLL 16 /* IO polling enabled if set */
607	#define QUEUE_FLAG_WC 17 /* Write back caching */	611	#define QUEUE_FLAG_WC 17 /* Write back caching */
608	#define QUEUE_FLAG_FUA 18 /* device supports FUA writes */	612	#define QUEUE_FLAG_FUA 18 /* device supports FUA writes */
@@ -615,6 +619,7 @@ struct request_queue { · 6 unmodified lines
615	#define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */	619	#define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */
616	#define QUEUE_FLAG_ZONE_RESETALL 26 /* supports Zone Reset All */	620	#define QUEUE_FLAG_ZONE_RESETALL 26 /* supports Zone Reset All */
617	#define QUEUE_FLAG_RQ_ALLOC_TIME 27 /* record rq->alloc_time_ns */	621	#define QUEUE_FLAG_RQ_ALLOC_TIME 27 /* record rq->alloc_time_ns */
		622	#define QUEUE_FLAG_HCTX_ACTIVE 28 /* at least one blk-mq hctx is active */
618		623
619	#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) \| \	624	#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) \| \
620	(1 << QUEUE_FLAG_SAME_COMP))	625	(1 << QUEUE_FLAG_SAME_COMP))
@@ -631,6 +636,8 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q); · 10 unmodified lines
631	#define blk_queue_noxmerges(q) \	636	#define blk_queue_noxmerges(q) \
632	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)	637	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
633	#define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)	638	#define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
		639	#define blk_queue_stable_writes(q) \
		640	test_bit(QUEUE_FLAG_STABLE_WRITES, &(q)->queue_flags)
634	#define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)	641	#define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
635	#define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)	642	#define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
636	#define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)	643	#define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
@@ -1059,11 +1066,17 @@ static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q, · 422 unmodified lines
1059	static inline unsigned int blk_max_size_offset(struct request_queue *q,	1066	static inline unsigned int blk_max_size_offset(struct request_queue *q,
1060	sector_t offset)	1067	sector_t offset)
1061	{	1068	{
1062	if (!q->limits.chunk_sectors)	1069	unsigned int chunk_sectors = q->limits.chunk_sectors;
		1070
		1071	if (!chunk_sectors)
1063	return q->limits.max_sectors;	1072	return q->limits.max_sectors;
1064		1073
1065	return min(q->limits.max_sectors, (unsigned int)(q->limits.chunk_sectors -	1074	if (likely(is_power_of_2(chunk_sectors)))
1066	(offset & (q->limits.chunk_sectors - 1))));	1075	chunk_sectors -= offset & (chunk_sectors - 1);
		1076	else
		1077	chunk_sectors -= sector_div(offset, chunk_sectors);
		1078
		1079	return min(q->limits.max_sectors, chunk_sectors);
1067	}	1080	}
1068		1081
1069	static inline unsigned int blk_rq_get_max_sectors(struct request *rq,	1082	static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
@@ -1130,6 +1143,7 @@ extern void blk_queue_max_zone_append_sectors(struct request_queue *q, · 60 unmodified lines
1130	extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);	1143	extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1131	extern void blk_queue_alignment_offset(struct request_queue *q,	1144	extern void blk_queue_alignment_offset(struct request_queue *q,
1132	unsigned int alignment);	1145	unsigned int alignment);
		1146	void blk_queue_update_readahead(struct request_queue *q);
1133	extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);	1147	extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
1134	extern void blk_queue_io_min(struct request_queue *q, unsigned int min);	1148	extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
1135	extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);	1149	extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
@@ -1455,10 +1469,9 @@ static inline int bdev_alignment_offset(struct block_device *bdev) · 319 unmodified lines
1455		1469
1456	if (q->limits.misaligned)	1470	if (q->limits.misaligned)
1457	return -1;	1471	return -1;
1458
1459	if (bdev != bdev->bd_contains)	1472	if (bdev != bdev->bd_contains)
1460	return bdev->bd_part->alignment_offset;	1473	return queue_limit_alignment_offset(&q->limits,
1461		1474	bdev->bd_part->start_sect);
1462	return q->limits.alignment_offset;	1475	return q->limits.alignment_offset;
1463	}	1476	}
1464		1477
@@ -1498,8 +1511,8 @@ static inline int bdev_discard_alignment(struct block_device *bdev) · 33 unmodified lines
1498	struct request_queue *q = bdev_get_queue(bdev);	1511	struct request_queue *q = bdev_get_queue(bdev);
1499		1512
1500	if (bdev != bdev->bd_contains)	1513	if (bdev != bdev->bd_contains)
1501	return bdev->bd_part->discard_alignment;	1514	return queue_limit_discard_alignment(&q->limits,
1502		1515	bdev->bd_part->start_sect);
1503	return q->limits.discard_alignment;	1516	return q->limits.discard_alignment;
1504	}	1517	}
1505		1518
@@ -1930,6 +1943,11 @@ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, · 424 unmodified lines
1930	void disk_end_io_acct(struct gendisk *disk, unsigned int op,	1943	void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1931	unsigned long start_time);	1944	unsigned long start_time);
1932		1945
		1946	unsigned long part_start_io_acct(struct gendisk disk, struct hd_struct *part,
		1947	struct bio *bio);
		1948	void part_end_io_acct(struct hd_struct part, struct bio bio,
		1949	unsigned long start_time);
		1950
1933	/**	1951	/**
1934	* bio_start_io_acct - start I/O accounting for bio based drivers	1952	* bio_start_io_acct - start I/O accounting for bio based drivers
1935	* @bio: bio to start account for	1953	* @bio: bio to start account for
@@ -1967,7 +1985,6 @@ void blkdev_show(struct seq_file *seqf, off_t offset); · 31 unmodified lines
1967	#define BLKDEV_MAJOR_MAX 0	1985	#define BLKDEV_MAJOR_MAX 0
1968	#endif	1986	#endif
1969		1987
1970	int blkdev_get(struct block_device bdev, fmode_t mode, void holder);
1971	struct block_device blkdev_get_by_path(const char path, fmode_t mode,	1988	struct block_device blkdev_get_by_path(const char path, fmode_t mode,
1972	void *holder);	1989	void *holder);
1973	struct block_device blkdev_get_by_dev(dev_t dev, fmode_t mode, void holder);	1990	struct block_device blkdev_get_by_dev(dev_t dev, fmode_t mode, void holder);
@@ -1984,11 +2001,18 @@ void bdput(struct block_device *); · 10 unmodified lines
1984		2001
1985	#ifdef CONFIG_BLOCK	2002	#ifdef CONFIG_BLOCK
1986	void invalidate_bdev(struct block_device *bdev);	2003	void invalidate_bdev(struct block_device *bdev);
		2004	int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
		2005	loff_t lend);
1987	int sync_blockdev(struct block_device *bdev);	2006	int sync_blockdev(struct block_device *bdev);
1988	#else	2007	#else
1989	static inline void invalidate_bdev(struct block_device *bdev)	2008	static inline void invalidate_bdev(struct block_device *bdev)
1990	{	2009	{
1991	}	2010	}
		2011	static inline int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
		2012	loff_t lstart, loff_t lend)
		2013	{
		2014	return 0;
		2015	}
1992	static inline int sync_blockdev(struct block_device *bdev)	2016	static inline int sync_blockdev(struct block_device *bdev)
1993	{	2017	{
1994	return 0;	2018	return 0;

include/linux/fs.h+1−1

@@ -1385,7 +1385,7 @@ extern int send_sigurg(struct fown_struct *fown); · 1384 unmodified lines
1385	#define SB_I_CGROUPWB 0x00000001 /* cgroup-aware writeback enabled */	1385	#define SB_I_CGROUPWB 0x00000001 /* cgroup-aware writeback enabled */
1386	#define SB_I_NOEXEC 0x00000002 /* Ignore executables on this fs */	1386	#define SB_I_NOEXEC 0x00000002 /* Ignore executables on this fs */
1387	#define SB_I_NODEV 0x00000004 /* Ignore devices on this fs */	1387	#define SB_I_NODEV 0x00000004 /* Ignore devices on this fs */
1388	#define SB_I_MULTIROOT 0x00000008 /* Multiple roots to the dentry tree */	1388	#define SB_I_STABLE_WRITES 0x00000008 /* don't modify blks until WB is done */
1389		1389
1390	/* sb->s_iflags to limit user namespace mounts */	1390	/* sb->s_iflags to limit user namespace mounts */
1391	#define SB_I_USERNS_VISIBLE 0x00000010 /* fstype already mounted */	1391	#define SB_I_USERNS_VISIBLE 0x00000010 /* fstype already mounted */

include/linux/genhd.h+7−8

 	struct disk_stats __percpu *dkstats;
 	struct percpu_ref ref;
-	sector_t alignment_offset;
-	unsigned int discard_alignment;
 	struct device __dev;
 	struct kobject *holder_dir;
 	int policy, partno;
 	void *private_data;
 	int flags;
+	unsigned long state;
+#define GD_NEED_PART_SCAN		0
 	struct rw_semaphore lookup_sem;
 	struct kobject *slave_dir;
 extern void disk_block_events(struct gendisk *disk);
 extern void disk_unblock_events(struct gendisk *disk);
 extern void disk_flush_events(struct gendisk *disk, unsigned int mask);
-extern void set_capacity_revalidate_and_notify(struct gendisk *disk,
+void set_capacity_revalidate_and_notify(struct gendisk *disk, sector_t size,
-			sector_t size, bool revalidate);
+		bool update_bdev);
-extern unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask);
 /* drivers/char/random.c */
 extern void add_disk_randomness(struct gendisk *disk) __latent_entropy;
 int register_blkdev(unsigned int major, const char *name);
 void unregister_blkdev(unsigned int major, const char *name);
-int revalidate_disk(struct gendisk *disk);
+void revalidate_disk_size(struct gendisk *disk, bool verbose);
-int check_disk_change(struct block_device *bdev);
+bool bdev_check_media_change(struct block_device *bdev);
 int __invalidate_device(struct block_device *bdev, bool kill_dirty);
-void bd_set_size(struct block_device *bdev, loff_t size);
+void bd_set_nr_sectors(struct block_device *bdev, sector_t sectors);
 /* for drivers/char/raw.c: */
 int blkdev_ioctl(struct block_device *, fmode_t, unsigned, unsigned long);

include/linux/ide.h+0−2

 	IDE_DFLAG_NOPROBE		= BIT(9),
 	/* need to do check_media_change() */
 	IDE_DFLAG_REMOVABLE		= BIT(10),
-	/* needed for removable devices */
-	IDE_DFLAG_ATTACH		= BIT(11),
 	IDE_DFLAG_FORCED_GEOM		= BIT(12),
 	/* disallow setting unmask bit */
 	IDE_DFLAG_NO_UNMASK		= BIT(13),

include/linux/suspend.h+2−2

@@ -473,9 +473,9 @@ static inline int hibernate_quiet_exec(int (func)(void data), void *data) { · 472 unmodified lines
473	#endif /* CONFIG_HIBERNATION */	473	#endif /* CONFIG_HIBERNATION */
474		474
475	#ifdef CONFIG_HIBERNATION_SNAPSHOT_DEV	475	#ifdef CONFIG_HIBERNATION_SNAPSHOT_DEV
476	int is_hibernate_resume_dev(const struct inode *);	476	int is_hibernate_resume_dev(dev_t dev);
477	#else	477	#else
478	static inline int is_hibernate_resume_dev(const struct inode *i) { return 0; }	478	static inline int is_hibernate_resume_dev(dev_t dev) { return 0; }
479	#endif	479	#endif
480		480
481	/* Hibernation and suspend events */	481	/* Hibernation and suspend events */

include/linux/swap.h+2−1

 extern void swap_free(swp_entry_t);
 extern void swapcache_free_entries(swp_entry_t *entries, int n);
 extern int free_swap_and_cache(swp_entry_t);
-extern int swap_type_of(dev_t, sector_t, struct block_device **);
+int swap_type_of(dev_t device, sector_t offset);
+int find_first_swap(dev_t *device);
 extern unsigned int count_swap_pages(int, int);
 extern sector_t map_swap_page(struct page *, struct block_device **);
 extern sector_t swapdev_block(int, pgoff_t);

include/trace/events/iocost.h+9−17

 		__field(u64, vrate)
 		__field(u64, last_period)
 		__field(u64, cur_period)
-		__field(u64, last_vtime)
 		__field(u64, vtime)
 		__field(u32, weight)
 		__field(u32, inuse)
 		__entry->vrate = now->vrate;
 		__entry->last_period = last_period;
 		__entry->cur_period = cur_period;
-		__entry->last_vtime = iocg->last_vtime;
 		__entry->vtime = vtime;
 		__entry->weight = iocg->weight;
 		__entry->inuse = iocg->inuse;
 	),
 	TP_printk("[%s:%s] now=%llu:%llu vrate=%llu "
-		  "period=%llu->%llu vtime=%llu->%llu "
+		  "period=%llu->%llu vtime=%llu "
 		  "weight=%u/%u hweight=%llu/%llu",
 		__get_str(devname), __get_str(cgroup),
 		__entry->now, __entry->vnow, __entry->vrate,
 		__entry->last_period, __entry->cur_period,
-		__entry->last_vtime, __entry->vtime,
+		__entry->vtime, __entry->inuse, __entry->weight,
-		__entry->inuse, __entry->weight,
 		__entry->hweight_inuse, __entry->hweight_active
 	)
 );
 	)
 );
-DEFINE_EVENT(iocg_inuse_update, iocost_inuse_takeback,
+DEFINE_EVENT(iocg_inuse_update, iocost_inuse_shortage,
 	TP_PROTO(struct ioc_gq *iocg, const char *path, struct ioc_now *now,
 		u32 old_inuse, u32 new_inuse,
 		old_hw_inuse, new_hw_inuse)
 );
-DEFINE_EVENT(iocg_inuse_update, iocost_inuse_giveaway,
+DEFINE_EVENT(iocg_inuse_update, iocost_inuse_transfer,
 	TP_PROTO(struct ioc_gq *iocg, const char *path, struct ioc_now *now,
 		u32 old_inuse, u32 new_inuse,
 		old_hw_inuse, new_hw_inuse)
 );
-DEFINE_EVENT(iocg_inuse_update, iocost_inuse_reset,
+DEFINE_EVENT(iocg_inuse_update, iocost_inuse_adjust,
 	TP_PROTO(struct ioc_gq *iocg, const char *path, struct ioc_now *now,
 		u32 old_inuse, u32 new_inuse,
 TRACE_EVENT(iocost_ioc_vrate_adj,
 	TP_PROTO(struct ioc *ioc, u64 new_vrate, u32 *missed_ppm,
-		u32 rq_wait_pct, int nr_lagging, int nr_shortages,
+		u32 rq_wait_pct, int nr_lagging, int nr_shortages),
-		int nr_surpluses),
-	TP_ARGS(ioc, new_vrate, missed_ppm, rq_wait_pct, nr_lagging, nr_shortages,
+	TP_ARGS(ioc, new_vrate, missed_ppm, rq_wait_pct, nr_lagging, nr_shortages),
-		nr_surpluses),
 	TP_STRUCT__entry (
 		__string(devname, ioc_name(ioc))
 		__field(u32, rq_wait_pct)
 		__field(int, nr_lagging)
 		__field(int, nr_shortages)
-		__field(int, nr_surpluses)
 	),
 	TP_fast_assign(
 		__entry->rq_wait_pct = rq_wait_pct;
 		__entry->nr_lagging = nr_lagging;
 		__entry->nr_shortages = nr_shortages;
-		__entry->nr_surpluses = nr_surpluses;
 	),
-	TP_printk("[%s] vrate=%llu->%llu busy=%d missed_ppm=%u:%u rq_wait_pct=%u lagging=%d shortages=%d surpluses=%d",
+	TP_printk("[%s] vrate=%llu->%llu busy=%d missed_ppm=%u:%u rq_wait_pct=%u lagging=%d shortages=%d",
 		__get_str(devname), __entry->old_vrate, __entry->new_vrate,
 		__entry->busy_level,
 		__entry->read_missed_ppm, __entry->write_missed_ppm,
-		__entry->rq_wait_pct, __entry->nr_lagging, __entry->nr_shortages,
+		__entry->rq_wait_pct, __entry->nr_lagging, __entry->nr_shortages
-		__entry->nr_surpluses
 	)
 );

include/uapi/linux/capability.h+2−0

@@ -288,6 +288,8 @@ struct vfs_ns_cap_data { · 287 unmodified lines
288	processes and setting the scheduling algorithm used by another	288	processes and setting the scheduling algorithm used by another
289	process. */	289	process. */
290	/* Allow setting cpu affinity on other processes */	290	/* Allow setting cpu affinity on other processes */
		291	/* Allow setting realtime ioprio class */
		292	/* Allow setting ioprio class on other processes */
291		293
292	#define CAP_SYS_NICE 23	294	#define CAP_SYS_NICE 23
293		295

kernel/power/swap.c+9−12

@@ -335,26 +335,23 @@ static int swsusp_swap_check(void) · 334 unmodified lines
335	{	335	{
336	int res;	336	int res;
337		337
338	res = swap_type_of(swsusp_resume_device, swsusp_resume_block,	338	if (swsusp_resume_device)
339	&hib_resume_bdev);	339	res = swap_type_of(swsusp_resume_device, swsusp_resume_block);
		340	else
		341	res = find_first_swap(&swsusp_resume_device);
340	if (res < 0)	342	if (res < 0)
341	return res;	343	return res;
342
343	root_swap = res;	344	root_swap = res;
344	res = blkdev_get(hib_resume_bdev, FMODE_WRITE, NULL);	345
345	if (res)	346	hib_resume_bdev = blkdev_get_by_dev(swsusp_resume_device, FMODE_WRITE,
346	return res;	347	NULL);
		348	if (IS_ERR(hib_resume_bdev))
		349	return PTR_ERR(hib_resume_bdev);
347		350
348	res = set_blocksize(hib_resume_bdev, PAGE_SIZE);	351	res = set_blocksize(hib_resume_bdev, PAGE_SIZE);
349	if (res < 0)	352	if (res < 0)
350	blkdev_put(hib_resume_bdev, FMODE_WRITE);	353	blkdev_put(hib_resume_bdev, FMODE_WRITE);
351		354
352	/*
353	* Update the resume device to the one actually used,
354	* so the test_resume mode can use it in case it is
355	* invoked from hibernate() to test the snapshot.
356	*/
357	swsusp_resume_device = hib_resume_bdev->bd_dev;
358	return res;	355	return res;
359	}	356	}
360		357

kernel/power/user.c+9−17

@@ -35,12 +35,12 @@ static struct snapshot_data { · 34 unmodified lines
35	bool ready;	35	bool ready;
36	bool platform_support;	36	bool platform_support;
37	bool free_bitmaps;	37	bool free_bitmaps;
38	struct inode *bd_inode;	38	dev_t dev;
39	} snapshot_state;	39	} snapshot_state;
40		40
41	int is_hibernate_resume_dev(const struct inode *bd_inode)	41	int is_hibernate_resume_dev(dev_t dev)
42	{	42	{
43	return hibernation_available() && snapshot_state.bd_inode == bd_inode;	43	return hibernation_available() && snapshot_state.dev == dev;
44	}	44	}
45		45
46	static int snapshot_open(struct inode inode, struct file filp)	46	static int snapshot_open(struct inode inode, struct file filp)
@@ -69,8 +69,7 @@ static int snapshot_open(struct inode inode, struct file filp) · 22 unmodified lines
69	memset(&data->handle, 0, sizeof(struct snapshot_handle));	69	memset(&data->handle, 0, sizeof(struct snapshot_handle));
70	if ((filp->f_flags & O_ACCMODE) == O_RDONLY) {	70	if ((filp->f_flags & O_ACCMODE) == O_RDONLY) {
71	/* Hibernating. The image device should be accessible. */	71	/* Hibernating. The image device should be accessible. */
72	data->swap = swsusp_resume_device ?	72	data->swap = swap_type_of(swsusp_resume_device, 0);
73	swap_type_of(swsusp_resume_device, 0, NULL) : -1;
74	data->mode = O_RDONLY;	73	data->mode = O_RDONLY;
75	data->free_bitmaps = false;	74	data->free_bitmaps = false;
76	error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls);	75	error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls);
@@ -101,7 +100,7 @@ static int snapshot_open(struct inode inode, struct file filp) · 24 unmodified lines
101	data->frozen = false;	100	data->frozen = false;
102	data->ready = false;	101	data->ready = false;
103	data->platform_support = false;	102	data->platform_support = false;
104	data->bd_inode = NULL;	103	data->dev = 0;
105		104
106	Unlock:	105	Unlock:
107	unlock_system_sleep();	106	unlock_system_sleep();
@@ -117,7 +116,7 @@ static int snapshot_release(struct inode inode, struct file filp) · 9 unmodified lines
117		116
118	swsusp_free();	117	swsusp_free();
119	data = filp->private_data;	118	data = filp->private_data;
120	data->bd_inode = NULL;	119	data->dev = 0;
121	free_all_swap_pages(data->swap);	120	free_all_swap_pages(data->swap);
122	if (data->frozen) {	121	if (data->frozen) {
123	pm_restore_gfp_mask();	122	pm_restore_gfp_mask();
@@ -210,7 +209,6 @@ struct compat_resume_swap_area { · 86 unmodified lines
210	static int snapshot_set_swap_area(struct snapshot_data *data,	209	static int snapshot_set_swap_area(struct snapshot_data *data,
211	void __user *argp)	210	void __user *argp)
212	{	211	{
213	struct block_device *bdev;
214	sector_t offset;	212	sector_t offset;
215	dev_t swdev;	213	dev_t swdev;
216		214
@@ -237,16 +235,10 @@ static int snapshot_set_swap_area(struct snapshot_data *data, · 20 unmodified lines
237	* User space encodes device types as two-byte values,	235	* User space encodes device types as two-byte values,
238	* so we need to recode them	236	* so we need to recode them
239	*/	237	*/
240	if (!swdev) {	238	data->swap = swap_type_of(swdev, offset);
241	data->swap = -1;
242	return -EINVAL;
243	}
244	data->swap = swap_type_of(swdev, offset, &bdev);
245	if (data->swap < 0)	239	if (data->swap < 0)
246	return -ENODEV;	240	return swdev ? -ENODEV : -EINVAL;
247		241	data->dev = swdev;
248	data->bd_inode = bdev->bd_inode;
249	bdput(bdev);
250	return 0;	242	return 0;
251	}	243	}
252		244

kernel/trace/blktrace.c+1−1

@@ -793,7 +793,7 @@ static u64 blk_trace_bio_get_cgid(struct request_queue q, struct bio bio) · 792 unmodified lines
793	return cgroup_id(bio_blkcg(bio)->css.cgroup);	793	return cgroup_id(bio_blkcg(bio)->css.cgroup);
794	}	794	}
795	#else	795	#else
796	u64 blk_trace_bio_get_cgid(struct request_queue q, struct bio bio)	796	static u64 blk_trace_bio_get_cgid(struct request_queue q, struct bio bio)
797	{	797	{
798	return 0;	798	return 0;
799	}	799	}

mm/backing-dev.c+7−7

@@ -14,9 +14,7 @@ · 13 unmodified lines
14	#include <linux/device.h>	14	#include <linux/device.h>
15	#include <trace/events/writeback.h>	15	#include <trace/events/writeback.h>
16		16
17	struct backing_dev_info noop_backing_dev_info = {	17	struct backing_dev_info noop_backing_dev_info;
18	.capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK,
19	};
20	EXPORT_SYMBOL_GPL(noop_backing_dev_info);	18	EXPORT_SYMBOL_GPL(noop_backing_dev_info);
21		19
22	static struct class *bdi_class;	20	static struct class *bdi_class;
@@ -204,10 +202,9 @@ static ssize_t stable_pages_required_show(struct device *dev, · 181 unmodified lines
204	struct device_attribute *attr,	202	struct device_attribute *attr,
205	char *page)	203	char *page)
206	{	204	{
207	struct backing_dev_info *bdi = dev_get_drvdata(dev);	205	dev_warn_once(dev,
208		206	"the stable_pages_required attribute has been removed. Use the stable_writes queue attribute instead.\n");
209	return snprintf(page, PAGE_SIZE-1, "%d\n",	207	return snprintf(page, PAGE_SIZE-1, "%d\n", 0);
210	bdi_cap_stable_pages_required(bdi) ? 1 : 0);
211	}	208	}
212	static DEVICE_ATTR_RO(stable_pages_required);	209	static DEVICE_ATTR_RO(stable_pages_required);
213		210
@@ -746,6 +743,9 @@ struct backing_dev_info *bdi_alloc(int node_id) · 532 unmodified lines
746	kfree(bdi);	743	kfree(bdi);
747	return NULL;	744	return NULL;
748	}	745	}
		746	bdi->capabilities = BDI_CAP_WRITEBACK \| BDI_CAP_WRITEBACK_ACCT;
		747	bdi->ra_pages = VM_READAHEAD_PAGES;
		748	bdi->io_pages = VM_READAHEAD_PAGES;
749	return bdi;	749	return bdi;
750	}	750	}
751	EXPORT_SYMBOL(bdi_alloc);	751	EXPORT_SYMBOL(bdi_alloc);

mm/filemap.c+2−2

@@ -414,7 +414,7 @@ int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, · 413 unmodified lines
414	.range_end = end,	414	.range_end = end,
415	};	415	};
416		416
417	if (!mapping_cap_writeback_dirty(mapping) \|\|	417	if (!mapping_can_writeback(mapping) \|\|
418	!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))	418	!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
419	return 0;	419	return 0;
420		420
@@ -1800,7 +1800,7 @@ repeat: · 1379 unmodified lines
1800	no_page:	1800	no_page:
1801	if (!page && (fgp_flags & FGP_CREAT)) {	1801	if (!page && (fgp_flags & FGP_CREAT)) {
1802	int err;	1802	int err;
1803	if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))	1803	if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
1804	gfp_mask \|= __GFP_WRITE;	1804	gfp_mask \|= __GFP_WRITE;
1805	if (fgp_flags & FGP_NOFS)	1805	if (fgp_flags & FGP_NOFS)
1806	gfp_mask &= ~__GFP_FS;	1806	gfp_mask &= ~__GFP_FS;

mm/memcontrol.c+1−1

@@ -5643,7 +5643,7 @@ static int mem_cgroup_move_account(struct page *page, · 5642 unmodified lines
5643	if (PageDirty(page)) {	5643	if (PageDirty(page)) {
5644	struct address_space *mapping = page_mapping(page);	5644	struct address_space *mapping = page_mapping(page);
5645		5645
5646	if (mapping_cap_account_dirty(mapping)) {	5646	if (mapping_can_writeback(mapping)) {
5647	__mod_lruvec_state(from_vec, NR_FILE_DIRTY,	5647	__mod_lruvec_state(from_vec, NR_FILE_DIRTY,
5648	-nr_pages);	5648	-nr_pages);
5649	__mod_lruvec_state(to_vec, NR_FILE_DIRTY,	5649	__mod_lruvec_state(to_vec, NR_FILE_DIRTY,

mm/memory-failure.c+1−1

@@ -1006,7 +1006,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, · 1005 unmodified lines
1006	*/	1006	*/
1007	mapping = page_mapping(hpage);	1007	mapping = page_mapping(hpage);
1008	if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&	1008	if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
1009	mapping_cap_writeback_dirty(mapping)) {	1009	mapping_can_writeback(mapping)) {
1010	if (page_mkclean(hpage)) {	1010	if (page_mkclean(hpage)) {
1011	SetPageDirty(hpage);	1011	SetPageDirty(hpage);
1012	} else {	1012	} else {

mm/migrate.c+1−1

@@ -503,7 +503,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 502 unmodified lines
503	__dec_lruvec_state(old_lruvec, NR_SHMEM);	503	__dec_lruvec_state(old_lruvec, NR_SHMEM);
504	__inc_lruvec_state(new_lruvec, NR_SHMEM);	504	__inc_lruvec_state(new_lruvec, NR_SHMEM);
505	}	505	}
506	if (dirty && mapping_cap_account_dirty(mapping)) {	506	if (dirty && mapping_can_writeback(mapping)) {
507	__dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);	507	__dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);
508	__dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING);	508	__dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING);
509	__inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY);	509	__inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY);

mm/mmap.c+1−1

@@ -1666,7 +1666,7 @@ int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) · 1665 unmodified lines
1666		1666
1667	/* Can the mapping track the dirty pages? */	1667	/* Can the mapping track the dirty pages? */
1668	return vma->vm_file && vma->vm_file->f_mapping &&	1668	return vma->vm_file && vma->vm_file->f_mapping &&
1669	mapping_cap_account_dirty(vma->vm_file->f_mapping);	1669	mapping_can_writeback(vma->vm_file->f_mapping);
1670	}	1670	}
1671		1671
1672	/*	1672	/*

mm/page-writeback.c+9−9

@@ -1882,7 +1882,7 @@ void balance_dirty_pages_ratelimited(struct address_space *mapping) · 1881 unmodified lines
1882	int ratelimit;	1882	int ratelimit;
1883	int *p;	1883	int *p;
1884		1884
1885	if (!bdi_cap_account_dirty(bdi))	1885	if (!(bdi->capabilities & BDI_CAP_WRITEBACK))
1886	return;	1886	return;
1887		1887
1888	if (inode_cgwb_enabled(inode))	1888	if (inode_cgwb_enabled(inode))
@@ -2423,7 +2423,7 @@ void account_page_dirtied(struct page page, struct address_space mapping) · 534 unmodified lines
2423		2423
2424	trace_writeback_dirty_page(page, mapping);	2424	trace_writeback_dirty_page(page, mapping);
2425		2425
2426	if (mapping_cap_account_dirty(mapping)) {	2426	if (mapping_can_writeback(mapping)) {
2427	struct bdi_writeback *wb;	2427	struct bdi_writeback *wb;
2428		2428
2429	inode_attach_wb(inode, page);	2429	inode_attach_wb(inode, page);
@@ -2450,7 +2450,7 @@ void account_page_dirtied(struct page page, struct address_space mapping) · 20 unmodified lines
2450	void account_page_cleaned(struct page page, struct address_space mapping,	2450	void account_page_cleaned(struct page page, struct address_space mapping,
2451	struct bdi_writeback *wb)	2451	struct bdi_writeback *wb)
2452	{	2452	{
2453	if (mapping_cap_account_dirty(mapping)) {	2453	if (mapping_can_writeback(mapping)) {
2454	dec_lruvec_page_state(page, NR_FILE_DIRTY);	2454	dec_lruvec_page_state(page, NR_FILE_DIRTY);
2455	dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);	2455	dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
2456	dec_wb_stat(wb, WB_RECLAIMABLE);	2456	dec_wb_stat(wb, WB_RECLAIMABLE);
@@ -2513,7 +2513,7 @@ void account_page_redirty(struct page *page) · 56 unmodified lines
2513	{	2513	{
2514	struct address_space *mapping = page->mapping;	2514	struct address_space *mapping = page->mapping;
2515		2515
2516	if (mapping && mapping_cap_account_dirty(mapping)) {	2516	if (mapping && mapping_can_writeback(mapping)) {
2517	struct inode *inode = mapping->host;	2517	struct inode *inode = mapping->host;
2518	struct bdi_writeback *wb;	2518	struct bdi_writeback *wb;
2519	struct wb_lock_cookie cookie = {};	2519	struct wb_lock_cookie cookie = {};
@@ -2625,7 +2625,7 @@ void __cancel_dirty_page(struct page *page) · 105 unmodified lines
2625	{	2625	{
2626	struct address_space *mapping = page_mapping(page);	2626	struct address_space *mapping = page_mapping(page);
2627		2627
2628	if (mapping_cap_account_dirty(mapping)) {	2628	if (mapping_can_writeback(mapping)) {
2629	struct inode *inode = mapping->host;	2629	struct inode *inode = mapping->host;
2630	struct bdi_writeback *wb;	2630	struct bdi_writeback *wb;
2631	struct wb_lock_cookie cookie = {};	2631	struct wb_lock_cookie cookie = {};
@@ -2665,7 +2665,7 @@ int clear_page_dirty_for_io(struct page *page) · 33 unmodified lines
2665		2665
2666	VM_BUG_ON_PAGE(!PageLocked(page), page);	2666	VM_BUG_ON_PAGE(!PageLocked(page), page);
2667		2667
2668	if (mapping && mapping_cap_account_dirty(mapping)) {	2668	if (mapping && mapping_can_writeback(mapping)) {
2669	struct inode *inode = mapping->host;	2669	struct inode *inode = mapping->host;
2670	struct bdi_writeback *wb;	2670	struct bdi_writeback *wb;
2671	struct wb_lock_cookie cookie = {};	2671	struct wb_lock_cookie cookie = {};
@@ -2738,7 +2738,7 @@ int test_clear_page_writeback(struct page *page) · 66 unmodified lines
2738	if (ret) {	2738	if (ret) {
2739	__xa_clear_mark(&mapping->i_pages, page_index(page),	2739	__xa_clear_mark(&mapping->i_pages, page_index(page),
2740	PAGECACHE_TAG_WRITEBACK);	2740	PAGECACHE_TAG_WRITEBACK);
2741	if (bdi_cap_account_writeback(bdi)) {	2741	if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
2742	struct bdi_writeback *wb = inode_to_wb(inode);	2742	struct bdi_writeback *wb = inode_to_wb(inode);
2743		2743
2744	dec_wb_stat(wb, WB_WRITEBACK);	2744	dec_wb_stat(wb, WB_WRITEBACK);
@@ -2791,7 +2791,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write) · 46 unmodified lines
2791	PAGECACHE_TAG_WRITEBACK);	2791	PAGECACHE_TAG_WRITEBACK);
2792		2792
2793	xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);	2793	xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
2794	if (bdi_cap_account_writeback(bdi))	2794	if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT)
2795	inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK);	2795	inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK);
2796		2796
2797	/*	2797	/*
@@ -2849,7 +2849,7 @@ EXPORT_SYMBOL_GPL(wait_on_page_writeback); · 51 unmodified lines
2849	*/	2849	*/
2850	void wait_for_stable_page(struct page *page)	2850	void wait_for_stable_page(struct page *page)
2851	{	2851	{
2852	if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host)))	2852	if (page->mapping->host->i_sb->s_iflags & SB_I_STABLE_WRITES)
2853	wait_on_page_writeback(page);	2853	wait_on_page_writeback(page);
2854	}	2854	}
2855	EXPORT_SYMBOL_GPL(wait_for_stable_page);	2855	EXPORT_SYMBOL_GPL(wait_for_stable_page);

mm/page_io.c+10−8

@@ -403,15 +403,17 @@ int swap_readpage(struct page *page, bool synchronous) · 402 unmodified lines
403	goto out;	403	goto out;
404	}	404	}
405		405
406	ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);	406	if (sis->flags & SWP_SYNCHRONOUS_IO) {
407	if (!ret) {	407	ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
408	if (trylock_page(page)) {	408	if (!ret) {
409	swap_slot_free_notify(page);	409	if (trylock_page(page)) {
410	unlock_page(page);	410	swap_slot_free_notify(page);
411	}	411	unlock_page(page);
		412	}
412		413
413	count_vm_event(PSWPIN);	414	count_vm_event(PSWPIN);
414	goto out;	415	goto out;
		416	}
415	}	417	}
416		418
417	ret = 0;	419	ret = 0;

mm/swapfile.c+26−23

  *
  * This is needed for the suspend to disk (aka swsusp).
  */
-int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
+int swap_type_of(dev_t device, sector_t offset)
 {
-	struct block_device *bdev = NULL;
 	int type;
-	if (device)
+	if (!device)
-		bdev = bdget(device);
+		return -1;
 	spin_lock(&swap_lock);
 	for (type = 0; type < nr_swapfiles; type++) {
 		if (!(sis->flags & SWP_WRITEOK))
 			continue;
-		if (!bdev) {
+		if (device == sis->bdev->bd_dev) {
-			if (bdev_p)
-				*bdev_p = bdgrab(sis->bdev);
-			spin_unlock(&swap_lock);
-			return type;
-		}
-		if (bdev == sis->bdev) {
 			struct swap_extent *se = first_se(sis);
 			if (se->start_block == offset) {
-				if (bdev_p)
-					*bdev_p = bdgrab(sis->bdev);
 				spin_unlock(&swap_lock);
-				bdput(bdev);
 				return type;
 			}
 		}
 	}
 	spin_unlock(&swap_lock);
-	if (bdev)
+	return -ENODEV;
-		bdput(bdev);
+}
+int find_first_swap(dev_t *device)
+{
+	int type;
+	spin_lock(&swap_lock);
+	for (type = 0; type < nr_swapfiles; type++) {
+		struct swap_info_struct *sis = swap_info[type];
+		if (!(sis->flags & SWP_WRITEOK))
+			continue;
+		*device = sis->bdev->bd_dev;
+		spin_unlock(&swap_lock);
+		return type;
+	}
+	spin_unlock(&swap_lock);
 	return -ENODEV;
 }
 	int error;
 	if (S_ISBLK(inode->i_mode)) {
-		p->bdev = bdgrab(I_BDEV(inode));
+		p->bdev = blkdev_get_by_dev(inode->i_rdev,
-		error = blkdev_get(p->bdev,
 				   FMODE_READ | FMODE_WRITE | FMODE_EXCL, p);
-		if (error < 0) {
+		if (IS_ERR(p->bdev)) {
+			error = PTR_ERR(p->bdev);
 			p->bdev = NULL;
 			return error;
 		}
 		goto bad_swap_unlock_inode;
 	}
-	if (bdi_cap_stable_pages_required(inode_to_bdi(inode)))
+	if (p->bdev && blk_queue_stable_writes(p->bdev->bd_disk->queue))
 		p->flags |= SWP_STABLE_WRITES;
-	if (bdi_cap_synchronous_io(inode_to_bdi(inode)))
+	if (p->bdev && p->bdev->bd_disk->fops->rw_page)
 		p->flags |= SWP_SYNCHRONOUS_IO;
 	if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {

tools/cgroup/iocost_monitor.py+19−35

@@ -45,8 +45,7 @@ except: · 44 unmodified lines
45	err('The kernel does not have iocost enabled')	45	err('The kernel does not have iocost enabled')
46		46
47	IOC_RUNNING = prog['IOC_RUNNING'].value_()	47	IOC_RUNNING = prog['IOC_RUNNING'].value_()
48	NR_USAGE_SLOTS = prog['NR_USAGE_SLOTS'].value_()	48	WEIGHT_ONE = prog['WEIGHT_ONE'].value_()
49	HWEIGHT_WHOLE = prog['HWEIGHT_WHOLE'].value_()
50	VTIME_PER_SEC = prog['VTIME_PER_SEC'].value_()	49	VTIME_PER_SEC = prog['VTIME_PER_SEC'].value_()
51	VTIME_PER_USEC = prog['VTIME_PER_USEC'].value_()	50	VTIME_PER_USEC = prog['VTIME_PER_USEC'].value_()
52	AUTOP_SSD_FAST = prog['AUTOP_SSD_FAST'].value_()	51	AUTOP_SSD_FAST = prog['AUTOP_SSD_FAST'].value_()
@@ -100,7 +99,7 @@ class IocStat: · 47 unmodified lines
100	self.period_ms = ioc.period_us.value_() / 1_000	99	self.period_ms = ioc.period_us.value_() / 1_000
101	self.period_at = ioc.period_at.value_() / 1_000_000	100	self.period_at = ioc.period_at.value_() / 1_000_000
102	self.vperiod_at = ioc.period_at_vtime.value_() / VTIME_PER_SEC	101	self.vperiod_at = ioc.period_at_vtime.value_() / VTIME_PER_SEC
103	self.vrate_pct = ioc.vtime_rate.counter.value_() * 100 / VTIME_PER_USEC	102	self.vrate_pct = ioc.vtime_base_rate.value_() * 100 / VTIME_PER_USEC
104	self.busy_level = ioc.busy_level.value_()	103	self.busy_level = ioc.busy_level.value_()
105	self.autop_idx = ioc.autop_idx.value_()	104	self.autop_idx = ioc.autop_idx.value_()
106	self.user_cost_model = ioc.user_cost_model.value_()	105	self.user_cost_model = ioc.user_cost_model.value_()
@@ -136,7 +135,7 @@ class IocStat: · 29 unmodified lines
136		135
137	def table_header_str(self):	136	def table_header_str(self):
138	return f'{"":25} active {"weight":>9} {"hweight%":>13} {"inflt%":>6} ' \	137	return f'{"":25} active {"weight":>9} {"hweight%":>13} {"inflt%":>6} ' \
139	f'{"dbt":>3} {"delay":>6} {"usages%"}'	138	f'{"debt":>7} {"delay":>7} {"usage%"}'
140		139
141	class IocgStat:	140	class IocgStat:
142	def __init__(self, iocg):	141	def __init__(self, iocg):
@@ -144,11 +143,11 @@ class IocgStat: · 1 unmodified line
144	blkg = iocg.pd.blkg	143	blkg = iocg.pd.blkg
145		144
146	self.is_active = not list_empty(iocg.active_list.address_of_())	145	self.is_active = not list_empty(iocg.active_list.address_of_())
147	self.weight = iocg.weight.value_()	146	self.weight = iocg.weight.value_() / WEIGHT_ONE
148	self.active = iocg.active.value_()	147	self.active = iocg.active.value_() / WEIGHT_ONE
149	self.inuse = iocg.inuse.value_()	148	self.inuse = iocg.inuse.value_() / WEIGHT_ONE
150	self.hwa_pct = iocg.hweight_active.value_() * 100 / HWEIGHT_WHOLE	149	self.hwa_pct = iocg.hweight_active.value_() * 100 / WEIGHT_ONE
151	self.hwi_pct = iocg.hweight_inuse.value_() * 100 / HWEIGHT_WHOLE	150	self.hwi_pct = iocg.hweight_inuse.value_() * 100 / WEIGHT_ONE
152	self.address = iocg.value_()	151	self.address = iocg.value_()
153		152
154	vdone = iocg.done_vtime.counter.value_()	153	vdone = iocg.done_vtime.counter.value_()
@@ -160,23 +159,13 @@ class IocgStat: · 5 unmodified lines
160	else:	159	else:
161	self.inflight_pct = 0	160	self.inflight_pct = 0
162		161
163	# vdebt used to be an atomic64_t and is now u64, support both	162	self.usage = (100 * iocg.usage_delta_us.value_() /
164	try:	163	ioc.period_us.value_()) if self.active else 0
165	self.debt_ms = iocg.abs_vdebt.counter.value_() / VTIME_PER_USEC / 1000	164	self.debt_ms = iocg.abs_vdebt.value_() / VTIME_PER_USEC / 1000
166	except:	165	if blkg.use_delay.counter.value_() != 0:
167	self.debt_ms = iocg.abs_vdebt.value_() / VTIME_PER_USEC / 1000	166	self.delay_ms = blkg.delay_nsec.counter.value_() / 1_000_000
168		167	else:
169	self.use_delay = blkg.use_delay.counter.value_()	168	self.delay_ms = 0
170	self.delay_ms = blkg.delay_nsec.counter.value_() / 1_000_000
171
172	usage_idx = iocg.usage_idx.value_()
173	self.usages = []
174	self.usage = 0
175	for i in range(NR_USAGE_SLOTS):
176	usage = iocg.usages[(usage_idx + 1 + i) % NR_USAGE_SLOTS].value_()
177	upct = usage * 100 / HWEIGHT_WHOLE
178	self.usages.append(upct)
179	self.usage = max(self.usage, upct)
180		169
181	def dict(self, now, path):	170	def dict(self, now, path):
182	out = { 'cgroup' : path,	171	out = { 'cgroup' : path,
@@ -189,25 +178,20 @@ class IocgStat: · 6 unmodified lines
189	'hweight_inuse_pct' : self.hwi_pct,	178	'hweight_inuse_pct' : self.hwi_pct,
190	'inflight_pct' : self.inflight_pct,	179	'inflight_pct' : self.inflight_pct,
191	'debt_ms' : self.debt_ms,	180	'debt_ms' : self.debt_ms,
192	'use_delay' : self.use_delay,
193	'delay_ms' : self.delay_ms,	181	'delay_ms' : self.delay_ms,
194	'usage_pct' : self.usage,	182	'usage_pct' : self.usage,
195	'address' : self.address }	183	'address' : self.address }
196	for i in range(len(self.usages)):
197	out[f'usage_pct_{i}'] = str(self.usages[i])
198	return out	184	return out
199		185
200	def table_row_str(self, path):	186	def table_row_str(self, path):
201	out = f'{path[-28:]:28} ' \	187	out = f'{path[-28:]:28} ' \
202	f'{"*" if self.is_active else " "} ' \	188	f'{"*" if self.is_active else " "} ' \
203	f'{self.inuse:5}/{self.active:5} ' \	189	f'{round(self.inuse):5}/{round(self.active):5} ' \
204	f'{self.hwi_pct:6.2f}/{self.hwa_pct:6.2f} ' \	190	f'{self.hwi_pct:6.2f}/{self.hwa_pct:6.2f} ' \
205	f'{self.inflight_pct:6.2f} ' \	191	f'{self.inflight_pct:6.2f} ' \
206	f'{min(math.ceil(self.debt_ms), 999):3} ' \	192	f'{self.debt_ms:7.2f} ' \
207	f'{min(self.use_delay, 99):2}*'\	193	f'{self.delay_ms:7.2f} '\
208	f'{min(math.ceil(self.delay_ms), 999):03} '	194	f'{min(self.usage, 999):6.2f}'
209	for u in self.usages:
210	out += f'{min(round(u), 999):03d}:'
211	out = out.rstrip(':')	195	out = out.rstrip(':')
212	return out	196	return out
213		197