blk-mq-sched: add framework for MQ capable IO schedulers

This adds a set of hooks that intercepts the blk-mq path of allocating/inserting/issuing/completing requests, allowing us to develop a scheduler within that framework. We reuse the existing elevator scheduler API on the registration side, but augment that with the scheduler flagging support for the blk-mq interfce, and with a separate set of ops hooks for MQ devices. We split driver and scheduler tags, so we can run the scheduling independently of device queue depth.

Jens Axboe committed 9 years ago

Commit bd166ef1 parent 2af8cbe17 files · +984 −194Browse files

Unified · Split

17 files

block/Makefile +1 −1
block/blk-cgroup.c +20 −4
block/blk-core.c +3 −1
block/blk-exec.c +2 −1
block/blk-flush.c +7 −5
block/blk-ioc.c +6 −2
block/blk-merge.c +1 −1
block/blk-mq-sched.c +368 −0
block/blk-mq-sched.h +170 −0
block/blk-mq-sysfs.c +13 −0
block/blk-mq.c +187 −131
block/blk-mq.h +7 −1
block/blk-tag.c +1 −0
block/elevator.c +159 −45
include/linux/blk-mq.h +4 −1
include/linux/blkdev.h +3 −1
include/linux/elevator.h +32 −0

block/Makefile+1−1

@@ -6,7 +6,7 @@ obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \ · 5 unmodified lines
6	6	blk-flush.o blk-settings.o blk-ioc.o blk-map.o \
7	7	blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \
8	8	blk-lib.o blk-mq.o blk-mq-tag.o blk-stat.o \
9		− blk-mq-sysfs.o blk-mq-cpumap.o ioctl.o \
	9	+ blk-mq-sysfs.o blk-mq-cpumap.o blk-mq-sched.o ioctl.o \
10	10	genhd.o scsi_ioctl.o partition-generic.o ioprio.o \
11	11	badblocks.o partitions/
12	12

block/blk-cgroup.c+20−4

@@ -1223,7 +1223,11 @@ int blkcg_activate_policy(struct request_queue *q, · 1222 unmodified lines
1223	1223	if (blkcg_policy_enabled(q, pol))
1224	1224	return 0;
1225	1225
1226		− blk_queue_bypass_start(q);
	1226	+ if (q->mq_ops) {
	1227	+ blk_mq_freeze_queue(q);
	1228	+ blk_mq_quiesce_queue(q);
	1229	+ } else
	1230	+ blk_queue_bypass_start(q);
1227	1231	pd_prealloc:
1228	1232	if (!pd_prealloc) {
1229	1233	pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
@@ -1261,7 +1265,10 @@ pd_prealloc: · 31 unmodified lines
1261	1265
1262	1266	spin_unlock_irq(q->queue_lock);
1263	1267	out_bypass_end:
1264		− blk_queue_bypass_end(q);
	1268	+ if (q->mq_ops)
	1269	+ blk_mq_unfreeze_queue(q);
	1270	+ else
	1271	+ blk_queue_bypass_end(q);
1265	1272	if (pd_prealloc)
1266	1273	pol->pd_free_fn(pd_prealloc);
1267	1274	return ret;
@@ -1284,7 +1291,12 @@ void blkcg_deactivate_policy(struct request_queue *q, · 16 unmodified lines
1284	1291	if (!blkcg_policy_enabled(q, pol))
1285	1292	return;
1286	1293
1287		− blk_queue_bypass_start(q);
	1294	+ if (q->mq_ops) {
	1295	+ blk_mq_freeze_queue(q);
	1296	+ blk_mq_quiesce_queue(q);
	1297	+ } else
	1298	+ blk_queue_bypass_start(q);
	1299	+
1288	1300	spin_lock_irq(q->queue_lock);
1289	1301
1290	1302	__clear_bit(pol->plid, q->blkcg_pols);
@@ -1304,7 +1316,11 @@ void blkcg_deactivate_policy(struct request_queue *q, · 13 unmodified lines
1304	1316	}
1305	1317
1306	1318	spin_unlock_irq(q->queue_lock);
1307		− blk_queue_bypass_end(q);
	1319	+
	1320	+ if (q->mq_ops)
	1321	+ blk_mq_unfreeze_queue(q);
	1322	+ else
	1323	+ blk_queue_bypass_end(q);
1308	1324	}
1309	1325	EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1310	1326

block/blk-core.c+3−1

@@ -39,6 +39,7 @@ · 38 unmodified lines
39	39
40	40	#include "blk.h"
41	41	#include "blk-mq.h"
	42	+#include "blk-mq-sched.h"
42	43	#include "blk-wbt.h"
43	44
44	45	EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
@@ -134,6 +135,7 @@ void blk_rq_init(struct request_queue q, struct request rq) · 89 unmodified lines
134	135	rq->cmd = rq->__cmd;
135	136	rq->cmd_len = BLK_MAX_CDB;
136	137	rq->tag = -1;
	138	+ rq->internal_tag = -1;
137	139	rq->start_time = jiffies;
138	140	set_start_time_ns(rq);
139	141	rq->part = NULL;
@@ -2127,7 +2129,7 @@ int blk_insert_cloned_request(struct request_queue q, struct request rq) · 1987 unmodified lines
2127	2129	if (q->mq_ops) {
2128	2130	if (blk_queue_io_stat(q))
2129	2131	blk_account_io_start(rq, true);
2130		− blk_mq_insert_request(rq, false, true, false);
	2132	+ blk_mq_sched_insert_request(rq, false, true, false);
2131	2133	return 0;
2132	2134	}
2133	2135

block/blk-exec.c+2−1

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#include <linux/sched/sysctl.h>
10	10
11	11	#include "blk.h"
	12	+#include "blk-mq-sched.h"
12	13
13	14	/*
14	15	* for max sense size
@@ -65,7 +66,7 @@ void blk_execute_rq_nowait(struct request_queue q, struct gendisk bd_disk, · 50 unmodified lines
65	66	* be reused after dying flag is set
66	67	*/
67	68	if (q->mq_ops) {
68		− blk_mq_insert_request(rq, at_head, true, false);
	69	+ blk_mq_sched_insert_request(rq, at_head, true, false);
69	70	return;
70	71	}
71	72

block/blk-flush.c+7−5

@@ -74,6 +74,7 @@ · 73 unmodified lines
74	74	#include "blk.h"
75	75	#include "blk-mq.h"
76	76	#include "blk-mq-tag.h"
	77	+#include "blk-mq-sched.h"
77	78
78	79	/* FLUSH/FUA sequences */
79	80	enum {
@@ -391,9 +392,10 @@ static void mq_flush_data_end_io(struct request *rq, int error) · 311 unmodified lines
391	392	* the comment in flush_end_io().
392	393	*/
393	394	spin_lock_irqsave(&fq->mq_flush_lock, flags);
394		− if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))
395		− blk_mq_run_hw_queue(hctx, true);
	395	+ blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);
396	396	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
	397	+
	398	+ blk_mq_run_hw_queue(hctx, true);
397	399	}
398	400
399	401	/**
@@ -453,9 +455,9 @@ void blk_insert_flush(struct request *rq) · 53 unmodified lines
453	455	*/
454	456	if ((policy & REQ_FSEQ_DATA) &&
455	457	!(policy & (REQ_FSEQ_PREFLUSH \| REQ_FSEQ_POSTFLUSH))) {
456		− if (q->mq_ops) {
457		− blk_mq_insert_request(rq, false, true, false);
458		− } else
	458	+ if (q->mq_ops)
	459	+ blk_mq_sched_insert_request(rq, false, true, false);
	460	+ else
459	461	list_add_tail(&rq->queuelist, &q->queue_head);
460	462	return;
461	463	}

block/blk-ioc.c+6−2

@@ -43,7 +43,9 @@ static void ioc_exit_icq(struct io_cq *icq) · 42 unmodified lines
43	43	if (icq->flags & ICQ_EXITED)
44	44	return;
45	45
46		− if (et->ops.sq.elevator_exit_icq_fn)
	46	+ if (et->uses_mq && et->ops.mq.exit_icq)
	47	+ et->ops.mq.exit_icq(icq);
	48	+ else if (!et->uses_mq && et->ops.sq.elevator_exit_icq_fn)
47	49	et->ops.sq.elevator_exit_icq_fn(icq);
48	50
49	51	icq->flags \|= ICQ_EXITED;
@@ -383,7 +385,9 @@ struct io_cq ioc_create_icq(struct io_context ioc, struct request_queue *q, · 333 unmodified lines
383	385	if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
384	386	hlist_add_head(&icq->ioc_node, &ioc->icq_list);
385	387	list_add(&icq->q_node, &q->icq_list);
386		− if (et->ops.sq.elevator_init_icq_fn)
	388	+ if (et->uses_mq && et->ops.mq.init_icq)
	389	+ et->ops.mq.init_icq(icq);
	390	+ else if (!et->uses_mq && et->ops.sq.elevator_init_icq_fn)
387	391	et->ops.sq.elevator_init_icq_fn(icq);
388	392	} else {
389	393	kmem_cache_free(et->icq_cache, icq);

block/blk-merge.c+1−1

@@ -763,7 +763,7 @@ int blk_attempt_req_merge(struct request_queue q, struct request rq, · 762 unmodified lines
763	763	{
764	764	struct elevator_queue *e = q->elevator;
765	765
766		− if (e->type->ops.sq.elevator_allow_rq_merge_fn)
	766	+ if (!e->uses_mq && e->type->ops.sq.elevator_allow_rq_merge_fn)
767	767	if (!e->type->ops.sq.elevator_allow_rq_merge_fn(q, rq, next))
768	768	return 0;
769	769

block/blk-mq-sched.c+368−0

@@ -0,0 +1,368 @@
	1	+/*
	2	+ * blk-mq scheduling framework
	3	+ *
	4	+ * Copyright (C) 2016 Jens Axboe
	5	+ */
	6	+#include <linux/kernel.h>
	7	+#include <linux/module.h>
	8	+#include <linux/blk-mq.h>
	9	+
	10	+#include <trace/events/block.h>
	11	+
	12	+#include "blk.h"
	13	+#include "blk-mq.h"
	14	+#include "blk-mq-sched.h"
	15	+#include "blk-mq-tag.h"
	16	+#include "blk-wbt.h"
	17	+
	18	+void blk_mq_sched_free_hctx_data(struct request_queue *q,
	19	+ void (exit)(struct blk_mq_hw_ctx ))
	20	+{
	21	+ struct blk_mq_hw_ctx *hctx;
	22	+ int i;
	23	+
	24	+ queue_for_each_hw_ctx(q, hctx, i) {
	25	+ if (exit && hctx->sched_data)
	26	+ exit(hctx);
	27	+ kfree(hctx->sched_data);
	28	+ hctx->sched_data = NULL;
	29	+ }
	30	+}
	31	+EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);
	32	+
	33	+int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
	34	+ int (init)(struct blk_mq_hw_ctx ),
	35	+ void (exit)(struct blk_mq_hw_ctx ))
	36	+{
	37	+ struct blk_mq_hw_ctx *hctx;
	38	+ int ret;
	39	+ int i;
	40	+
	41	+ queue_for_each_hw_ctx(q, hctx, i) {
	42	+ hctx->sched_data = kmalloc_node(size, GFP_KERNEL, hctx->numa_node);
	43	+ if (!hctx->sched_data) {
	44	+ ret = -ENOMEM;
	45	+ goto error;
	46	+ }
	47	+
	48	+ if (init) {
	49	+ ret = init(hctx);
	50	+ if (ret) {
	51	+ /*
	52	+ * We don't want to give exit() a partially
	53	+ * initialized sched_data. init() must clean up
	54	+ * if it fails.
	55	+ */
	56	+ kfree(hctx->sched_data);
	57	+ hctx->sched_data = NULL;
	58	+ goto error;
	59	+ }
	60	+ }
	61	+ }
	62	+
	63	+ return 0;
	64	+error:
	65	+ blk_mq_sched_free_hctx_data(q, exit);
	66	+ return ret;
	67	+}
	68	+EXPORT_SYMBOL_GPL(blk_mq_sched_init_hctx_data);
	69	+
	70	+static void __blk_mq_sched_assign_ioc(struct request_queue *q,
	71	+ struct request rq, struct io_context ioc)
	72	+{
	73	+ struct io_cq *icq;
	74	+
	75	+ spin_lock_irq(q->queue_lock);
	76	+ icq = ioc_lookup_icq(ioc, q);
	77	+ spin_unlock_irq(q->queue_lock);
	78	+
	79	+ if (!icq) {
	80	+ icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
	81	+ if (!icq)
	82	+ return;
	83	+ }
	84	+
	85	+ rq->elv.icq = icq;
	86	+ if (!blk_mq_sched_get_rq_priv(q, rq)) {
	87	+ rq->rq_flags \|= RQF_ELVPRIV;
	88	+ get_io_context(icq->ioc);
	89	+ return;
	90	+ }
	91	+
	92	+ rq->elv.icq = NULL;
	93	+}
	94	+
	95	+static void blk_mq_sched_assign_ioc(struct request_queue *q,
	96	+ struct request rq, struct bio bio)
	97	+{
	98	+ struct io_context *ioc;
	99	+
	100	+ ioc = rq_ioc(bio);
	101	+ if (ioc)
	102	+ __blk_mq_sched_assign_ioc(q, rq, ioc);
	103	+}
	104	+
	105	+struct request blk_mq_sched_get_request(struct request_queue q,
	106	+ struct bio *bio,
	107	+ unsigned int op,
	108	+ struct blk_mq_alloc_data *data)
	109	+{
	110	+ struct elevator_queue *e = q->elevator;
	111	+ struct blk_mq_hw_ctx *hctx;
	112	+ struct blk_mq_ctx *ctx;
	113	+ struct request *rq;
	114	+ const bool is_flush = op & (REQ_PREFLUSH \| REQ_FUA);
	115	+
	116	+ blk_queue_enter_live(q);
	117	+ ctx = blk_mq_get_ctx(q);
	118	+ hctx = blk_mq_map_queue(q, ctx->cpu);
	119	+
	120	+ blk_mq_set_alloc_data(data, q, 0, ctx, hctx);
	121	+
	122	+ if (e) {
	123	+ data->flags \|= BLK_MQ_REQ_INTERNAL;
	124	+
	125	+ /*
	126	+ * Flush requests are special and go directly to the
	127	+ * dispatch list.
	128	+ */
	129	+ if (!is_flush && e->type->ops.mq.get_request) {
	130	+ rq = e->type->ops.mq.get_request(q, op, data);
	131	+ if (rq)
	132	+ rq->rq_flags \|= RQF_QUEUED;
	133	+ } else
	134	+ rq = __blk_mq_alloc_request(data, op);
	135	+ } else {
	136	+ rq = __blk_mq_alloc_request(data, op);
	137	+ data->hctx->tags->rqs[rq->tag] = rq;
	138	+ }
	139	+
	140	+ if (rq) {
	141	+ if (!is_flush) {
	142	+ rq->elv.icq = NULL;
	143	+ if (e && e->type->icq_cache)
	144	+ blk_mq_sched_assign_ioc(q, rq, bio);
	145	+ }
	146	+ data->hctx->queued++;
	147	+ return rq;
	148	+ }
	149	+
	150	+ blk_queue_exit(q);
	151	+ return NULL;
	152	+}
	153	+
	154	+void blk_mq_sched_put_request(struct request *rq)
	155	+{
	156	+ struct request_queue *q = rq->q;
	157	+ struct elevator_queue *e = q->elevator;
	158	+
	159	+ if (rq->rq_flags & RQF_ELVPRIV) {
	160	+ blk_mq_sched_put_rq_priv(rq->q, rq);
	161	+ if (rq->elv.icq) {
	162	+ put_io_context(rq->elv.icq->ioc);
	163	+ rq->elv.icq = NULL;
	164	+ }
	165	+ }
	166	+
	167	+ if ((rq->rq_flags & RQF_QUEUED) && e && e->type->ops.mq.put_request)
	168	+ e->type->ops.mq.put_request(rq);
	169	+ else
	170	+ blk_mq_finish_request(rq);
	171	+}
	172	+
	173	+void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
	174	+{
	175	+ struct elevator_queue *e = hctx->queue->elevator;
	176	+ LIST_HEAD(rq_list);
	177	+
	178	+ if (unlikely(blk_mq_hctx_stopped(hctx)))
	179	+ return;
	180	+
	181	+ hctx->run++;
	182	+
	183	+ /*
	184	+ * If we have previous entries on our dispatch list, grab them first for
	185	+ * more fair dispatch.
	186	+ */
	187	+ if (!list_empty_careful(&hctx->dispatch)) {
	188	+ spin_lock(&hctx->lock);
	189	+ if (!list_empty(&hctx->dispatch))
	190	+ list_splice_init(&hctx->dispatch, &rq_list);
	191	+ spin_unlock(&hctx->lock);
	192	+ }
	193	+
	194	+ /*
	195	+ * Only ask the scheduler for requests, if we didn't have residual
	196	+ * requests from the dispatch list. This is to avoid the case where
	197	+ * we only ever dispatch a fraction of the requests available because
	198	+ * of low device queue depth. Once we pull requests out of the IO
	199	+ * scheduler, we can no longer merge or sort them. So it's best to
	200	+ * leave them there for as long as we can. Mark the hw queue as
	201	+ * needing a restart in that case.
	202	+ */
	203	+ if (list_empty(&rq_list)) {
	204	+ if (e && e->type->ops.mq.dispatch_requests)
	205	+ e->type->ops.mq.dispatch_requests(hctx, &rq_list);
	206	+ else
	207	+ blk_mq_flush_busy_ctxs(hctx, &rq_list);
	208	+ } else
	209	+ blk_mq_sched_mark_restart(hctx);
	210	+
	211	+ blk_mq_dispatch_rq_list(hctx, &rq_list);
	212	+}
	213	+
	214	+void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
	215	+ struct list_head *rq_list,
	216	+ struct request (get_rq)(struct blk_mq_hw_ctx *))
	217	+{
	218	+ do {
	219	+ struct request *rq;
	220	+
	221	+ rq = get_rq(hctx);
	222	+ if (!rq)
	223	+ break;
	224	+
	225	+ list_add_tail(&rq->queuelist, rq_list);
	226	+ } while (1);
	227	+}
	228	+EXPORT_SYMBOL_GPL(blk_mq_sched_move_to_dispatch);
	229	+
	230	+bool blk_mq_sched_try_merge(struct request_queue q, struct bio bio)
	231	+{
	232	+ struct request *rq;
	233	+ int ret;
	234	+
	235	+ ret = elv_merge(q, &rq, bio);
	236	+ if (ret == ELEVATOR_BACK_MERGE) {
	237	+ if (!blk_mq_sched_allow_merge(q, rq, bio))
	238	+ return false;
	239	+ if (bio_attempt_back_merge(q, rq, bio)) {
	240	+ if (!attempt_back_merge(q, rq))
	241	+ elv_merged_request(q, rq, ret);
	242	+ return true;
	243	+ }
	244	+ } else if (ret == ELEVATOR_FRONT_MERGE) {
	245	+ if (!blk_mq_sched_allow_merge(q, rq, bio))
	246	+ return false;
	247	+ if (bio_attempt_front_merge(q, rq, bio)) {
	248	+ if (!attempt_front_merge(q, rq))
	249	+ elv_merged_request(q, rq, ret);
	250	+ return true;
	251	+ }
	252	+ }
	253	+
	254	+ return false;
	255	+}
	256	+EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
	257	+
	258	+bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio)
	259	+{
	260	+ struct elevator_queue *e = q->elevator;
	261	+
	262	+ if (e->type->ops.mq.bio_merge) {
	263	+ struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
	264	+ struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	265	+
	266	+ blk_mq_put_ctx(ctx);
	267	+ return e->type->ops.mq.bio_merge(hctx, bio);
	268	+ }
	269	+
	270	+ return false;
	271	+}
	272	+
	273	+bool blk_mq_sched_try_insert_merge(struct request_queue q, struct request rq)
	274	+{
	275	+ return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
	276	+}
	277	+EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);
	278	+
	279	+void blk_mq_sched_request_inserted(struct request *rq)
	280	+{
	281	+ trace_block_rq_insert(rq->q, rq);
	282	+}
	283	+EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);
	284	+
	285	+bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx hctx, struct request rq)
	286	+{
	287	+ if (rq->tag == -1) {
	288	+ rq->rq_flags \|= RQF_SORTED;
	289	+ return false;
	290	+ }
	291	+
	292	+ /*
	293	+ * If we already have a real request tag, send directly to
	294	+ * the dispatch list.
	295	+ */
	296	+ spin_lock(&hctx->lock);
	297	+ list_add(&rq->queuelist, &hctx->dispatch);
	298	+ spin_unlock(&hctx->lock);
	299	+ return true;
	300	+}
	301	+EXPORT_SYMBOL_GPL(blk_mq_sched_bypass_insert);
	302	+
	303	+static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
	304	+ struct blk_mq_hw_ctx *hctx,
	305	+ unsigned int hctx_idx)
	306	+{
	307	+ if (hctx->sched_tags) {
	308	+ blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
	309	+ blk_mq_free_rq_map(hctx->sched_tags);
	310	+ hctx->sched_tags = NULL;
	311	+ }
	312	+}
	313	+
	314	+int blk_mq_sched_setup(struct request_queue *q)
	315	+{
	316	+ struct blk_mq_tag_set *set = q->tag_set;
	317	+ struct blk_mq_hw_ctx *hctx;
	318	+ int ret, i;
	319	+
	320	+ /*
	321	+ * Default to 256, since we don't split into sync/async like the
	322	+ * old code did. Additionally, this is a per-hw queue depth.
	323	+ */
	324	+ q->nr_requests = 2 * BLKDEV_MAX_RQ;
	325	+
	326	+ /*
	327	+ * We're switching to using an IO scheduler, so setup the hctx
	328	+ * scheduler tags and switch the request map from the regular
	329	+ * tags to scheduler tags. First allocate what we need, so we
	330	+ * can safely fail and fallback, if needed.
	331	+ */
	332	+ ret = 0;
	333	+ queue_for_each_hw_ctx(q, hctx, i) {
	334	+ hctx->sched_tags = blk_mq_alloc_rq_map(set, i, q->nr_requests, 0);
	335	+ if (!hctx->sched_tags) {
	336	+ ret = -ENOMEM;
	337	+ break;
	338	+ }
	339	+ ret = blk_mq_alloc_rqs(set, hctx->sched_tags, i, q->nr_requests);
	340	+ if (ret)
	341	+ break;
	342	+ }
	343	+
	344	+ /*
	345	+ * If we failed, free what we did allocate
	346	+ */
	347	+ if (ret) {
	348	+ queue_for_each_hw_ctx(q, hctx, i) {
	349	+ if (!hctx->sched_tags)
	350	+ continue;
	351	+ blk_mq_sched_free_tags(set, hctx, i);
	352	+ }
	353	+
	354	+ return ret;
	355	+ }
	356	+
	357	+ return 0;
	358	+}
	359	+
	360	+void blk_mq_sched_teardown(struct request_queue *q)
	361	+{
	362	+ struct blk_mq_tag_set *set = q->tag_set;
	363	+ struct blk_mq_hw_ctx *hctx;
	364	+ int i;
	365	+
	366	+ queue_for_each_hw_ctx(q, hctx, i)
	367	+ blk_mq_sched_free_tags(set, hctx, i);
	368	+}

block/blk-mq-sched.h+170−0

@@ -0,0 +1,170 @@
	1	+#ifndef BLK_MQ_SCHED_H
	2	+#define BLK_MQ_SCHED_H
	3	+
	4	+#include "blk-mq.h"
	5	+#include "blk-mq-tag.h"
	6	+
	7	+int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
	8	+ int (init)(struct blk_mq_hw_ctx ),
	9	+ void (exit)(struct blk_mq_hw_ctx ));
	10	+
	11	+void blk_mq_sched_free_hctx_data(struct request_queue *q,
	12	+ void (exit)(struct blk_mq_hw_ctx ));
	13	+
	14	+struct request blk_mq_sched_get_request(struct request_queue q, struct bio bio, unsigned int op, struct blk_mq_alloc_data data);
	15	+void blk_mq_sched_put_request(struct request *rq);
	16	+
	17	+void blk_mq_sched_request_inserted(struct request *rq);
	18	+bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx hctx, struct request rq);
	19	+bool blk_mq_sched_try_merge(struct request_queue q, struct bio bio);
	20	+bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio);
	21	+bool blk_mq_sched_try_insert_merge(struct request_queue q, struct request rq);
	22	+
	23	+void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx);
	24	+void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
	25	+ struct list_head *rq_list,
	26	+ struct request (get_rq)(struct blk_mq_hw_ctx *));
	27	+
	28	+int blk_mq_sched_setup(struct request_queue *q);
	29	+void blk_mq_sched_teardown(struct request_queue *q);
	30	+
	31	+static inline bool
	32	+blk_mq_sched_bio_merge(struct request_queue q, struct bio bio)
	33	+{
	34	+ struct elevator_queue *e = q->elevator;
	35	+
	36	+ if (!e \|\| blk_queue_nomerges(q) \|\| !bio_mergeable(bio))
	37	+ return false;
	38	+
	39	+ return __blk_mq_sched_bio_merge(q, bio);
	40	+}
	41	+
	42	+static inline int blk_mq_sched_get_rq_priv(struct request_queue *q,
	43	+ struct request *rq)
	44	+{
	45	+ struct elevator_queue *e = q->elevator;
	46	+
	47	+ if (e && e->type->ops.mq.get_rq_priv)
	48	+ return e->type->ops.mq.get_rq_priv(q, rq);
	49	+
	50	+ return 0;
	51	+}
	52	+
	53	+static inline void blk_mq_sched_put_rq_priv(struct request_queue *q,
	54	+ struct request *rq)
	55	+{
	56	+ struct elevator_queue *e = q->elevator;
	57	+
	58	+ if (e && e->type->ops.mq.put_rq_priv)
	59	+ e->type->ops.mq.put_rq_priv(q, rq);
	60	+}
	61	+
	62	+static inline void
	63	+blk_mq_sched_insert_request(struct request *rq, bool at_head, bool run_queue,
	64	+ bool async)
	65	+{
	66	+ struct request_queue *q = rq->q;
	67	+ struct elevator_queue *e = q->elevator;
	68	+ struct blk_mq_ctx *ctx = rq->mq_ctx;
	69	+ struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	70	+
	71	+ if (e && e->type->ops.mq.insert_requests) {
	72	+ LIST_HEAD(list);
	73	+
	74	+ list_add(&rq->queuelist, &list);
	75	+ e->type->ops.mq.insert_requests(hctx, &list, at_head);
	76	+ } else {
	77	+ spin_lock(&ctx->lock);
	78	+ __blk_mq_insert_request(hctx, rq, at_head);
	79	+ spin_unlock(&ctx->lock);
	80	+ }
	81	+
	82	+ if (run_queue)
	83	+ blk_mq_run_hw_queue(hctx, async);
	84	+}
	85	+
	86	+static inline void
	87	+blk_mq_sched_insert_requests(struct request_queue q, struct blk_mq_ctx ctx,
	88	+ struct list_head *list, bool run_queue_async)
	89	+{
	90	+ struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	91	+ struct elevator_queue *e = hctx->queue->elevator;
	92	+
	93	+ if (e && e->type->ops.mq.insert_requests)
	94	+ e->type->ops.mq.insert_requests(hctx, list, false);
	95	+ else
	96	+ blk_mq_insert_requests(hctx, ctx, list);
	97	+
	98	+ blk_mq_run_hw_queue(hctx, run_queue_async);
	99	+}
	100	+
	101	+static inline bool
	102	+blk_mq_sched_allow_merge(struct request_queue q, struct request rq,
	103	+ struct bio *bio)
	104	+{
	105	+ struct elevator_queue *e = q->elevator;
	106	+
	107	+ if (e && e->type->ops.mq.allow_merge)
	108	+ return e->type->ops.mq.allow_merge(q, rq, bio);
	109	+
	110	+ return true;
	111	+}
	112	+
	113	+static inline void
	114	+blk_mq_sched_completed_request(struct blk_mq_hw_ctx hctx, struct request rq)
	115	+{
	116	+ struct elevator_queue *e = hctx->queue->elevator;
	117	+
	118	+ if (e && e->type->ops.mq.completed_request)
	119	+ e->type->ops.mq.completed_request(hctx, rq);
	120	+
	121	+ BUG_ON(rq->internal_tag == -1);
	122	+
	123	+ blk_mq_put_tag(hctx, hctx->sched_tags, rq->mq_ctx, rq->internal_tag);
	124	+
	125	+ if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) {
	126	+ clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
	127	+ blk_mq_run_hw_queue(hctx, true);
	128	+ }
	129	+}
	130	+
	131	+static inline void blk_mq_sched_started_request(struct request *rq)
	132	+{
	133	+ struct request_queue *q = rq->q;
	134	+ struct elevator_queue *e = q->elevator;
	135	+
	136	+ if (e && e->type->ops.mq.started_request)
	137	+ e->type->ops.mq.started_request(rq);
	138	+}
	139	+
	140	+static inline void blk_mq_sched_requeue_request(struct request *rq)
	141	+{
	142	+ struct request_queue *q = rq->q;
	143	+ struct elevator_queue *e = q->elevator;
	144	+
	145	+ if (e && e->type->ops.mq.requeue_request)
	146	+ e->type->ops.mq.requeue_request(rq);
	147	+}
	148	+
	149	+static inline bool blk_mq_sched_has_work(struct blk_mq_hw_ctx *hctx)
	150	+{
	151	+ struct elevator_queue *e = hctx->queue->elevator;
	152	+
	153	+ if (e && e->type->ops.mq.has_work)
	154	+ return e->type->ops.mq.has_work(hctx);
	155	+
	156	+ return false;
	157	+}
	158	+
	159	+static inline void blk_mq_sched_mark_restart(struct blk_mq_hw_ctx *hctx)
	160	+{
	161	+ if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
	162	+ set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
	163	+}
	164	+
	165	+static inline bool blk_mq_sched_needs_restart(struct blk_mq_hw_ctx *hctx)
	166	+{
	167	+ return test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
	168	+}
	169	+
	170	+#endif

block/blk-mq-sysfs.c+13−0

@@ -231,6 +231,14 @@ static ssize_t blk_mq_hw_sysfs_rq_list_show(struct blk_mq_hw_ctx *hctx, · 230 unmodified lines
231	231	return ret;
232	232	}
233	233
	234	+static ssize_t blk_mq_hw_sysfs_sched_tags_show(struct blk_mq_hw_ctx hctx, char page)
	235	+{
	236	+ if (hctx->sched_tags)
	237	+ return blk_mq_tag_sysfs_show(hctx->sched_tags, page);
	238	+
	239	+ return 0;
	240	+}
	241	+
234	242	static ssize_t blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx hctx, char page)
235	243	{
236	244	return blk_mq_tag_sysfs_show(hctx->tags, page);
@@ -345,6 +353,10 @@ static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_pending = { · 108 unmodified lines
345	353	.attr = {.name = "pending", .mode = S_IRUGO },
346	354	.show = blk_mq_hw_sysfs_rq_list_show,
347	355	};
	356	+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_sched_tags = {
	357	+ .attr = {.name = "sched_tags", .mode = S_IRUGO },
	358	+ .show = blk_mq_hw_sysfs_sched_tags_show,
	359	+};
348	360	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags = {
349	361	.attr = {.name = "tags", .mode = S_IRUGO },
350	362	.show = blk_mq_hw_sysfs_tags_show,
@@ -370,6 +382,7 @@ static struct attribute *default_hw_ctx_attrs[] = { · 19 unmodified lines
370	382	&blk_mq_hw_sysfs_dispatched.attr,
371	383	&blk_mq_hw_sysfs_pending.attr,
372	384	&blk_mq_hw_sysfs_tags.attr,
	385	+ &blk_mq_hw_sysfs_sched_tags.attr,
373	386	&blk_mq_hw_sysfs_cpus.attr,
374	387	&blk_mq_hw_sysfs_active.attr,
375	388	&blk_mq_hw_sysfs_poll.attr,

block/blk-mq.c+187−131

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	32	#include "blk-mq-tag.h"
33	33	#include "blk-stat.h"
34	34	#include "blk-wbt.h"
	35	+#include "blk-mq-sched.h"
35	36
36	37	static DEFINE_MUTEX(all_q_mutex);
37	38	static LIST_HEAD(all_q_list);
@@ -41,7 +42,9 @@ static LIST_HEAD(all_q_list); · 3 unmodified lines
41	42	*/
42	43	static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
43	44	{
44		− return sbitmap_any_bit_set(&hctx->ctx_map);
	45	+ return sbitmap_any_bit_set(&hctx->ctx_map) \|\|
	46	+ !list_empty_careful(&hctx->dispatch) \|\|
	47	+ blk_mq_sched_has_work(hctx);
45	48	}
46	49
47	50	/*
@@ -223,15 +226,23 @@ struct request __blk_mq_alloc_request(struct blk_mq_alloc_data data, · 175 unmodified lines
223	226
224	227	tag = blk_mq_get_tag(data);
225	228	if (tag != BLK_MQ_TAG_FAIL) {
226		− rq = data->hctx->tags->static_rqs[tag];
	229	+ struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
	230	+
	231	+ rq = tags->static_rqs[tag];
227	232
228	233	if (blk_mq_tag_busy(data->hctx)) {
229	234	rq->rq_flags = RQF_MQ_INFLIGHT;
230	235	atomic_inc(&data->hctx->nr_active);
231	236	}
232	237
233		− rq->tag = tag;
234		− data->hctx->tags->rqs[tag] = rq;
	238	+ if (data->flags & BLK_MQ_REQ_INTERNAL) {
	239	+ rq->tag = -1;
	240	+ rq->internal_tag = tag;
	241	+ } else {
	242	+ rq->tag = tag;
	243	+ rq->internal_tag = -1;
	244	+ }
	245	+
235	246	blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
236	247	return rq;
237	248	}
@@ -243,26 +254,21 @@ EXPORT_SYMBOL_GPL(__blk_mq_alloc_request); · 5 unmodified lines
243	254	struct request blk_mq_alloc_request(struct request_queue q, int rw,
244	255	unsigned int flags)
245	256	{
246		− struct blk_mq_ctx *ctx;
247		− struct blk_mq_hw_ctx *hctx;
248		− struct request *rq;
249	257	struct blk_mq_alloc_data alloc_data;
	258	+ struct request *rq;
250	259	int ret;
251	260
252	261	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
253	262	if (ret)
254	263	return ERR_PTR(ret);
255	264
256		− ctx = blk_mq_get_ctx(q);
257		− hctx = blk_mq_map_queue(q, ctx->cpu);
258		− blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
259		− rq = __blk_mq_alloc_request(&alloc_data, rw);
260		− blk_mq_put_ctx(ctx);
	265	+ rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
261	266
262		− if (!rq) {
263		− blk_queue_exit(q);
	267	+ blk_mq_put_ctx(alloc_data.ctx);
	268	+ blk_queue_exit(q);
	269	+
	270	+ if (!rq)
264	271	return ERR_PTR(-EWOULDBLOCK);
265		− }
266	272
267	273	rq->__data_len = 0;
268	274	rq->__sector = (sector_t) -1;
@@ -322,10 +328,10 @@ out_queue_exit: · 53 unmodified lines
322	328	}
323	329	EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
324	330
325		−void __blk_mq_free_request(struct blk_mq_hw_ctx hctx, struct blk_mq_ctx ctx,
326		− struct request *rq)
	331	+void __blk_mq_finish_request(struct blk_mq_hw_ctx hctx, struct blk_mq_ctx ctx,
	332	+ struct request *rq)
327	333	{
328		− const int tag = rq->tag;
	334	+ const int sched_tag = rq->internal_tag;
329	335	struct request_queue *q = rq->q;
330	336
331	337	if (rq->rq_flags & RQF_MQ_INFLIGHT)
@@ -336,22 +342,30 @@ void __blk_mq_free_request(struct blk_mq_hw_ctx hctx, struct blk_mq_ctx ctx, · 4 unmodified lines
336	342
337	343	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
338	344	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
339		− blk_mq_put_tag(hctx, hctx->tags, ctx, tag);
	345	+ if (rq->tag != -1)
	346	+ blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
	347	+ if (sched_tag != -1)
	348	+ blk_mq_sched_completed_request(hctx, rq);
340	349	blk_queue_exit(q);
341	350	}
342	351
343		−static void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *hctx,
	352	+static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
344	353	struct request *rq)
345	354	{
346	355	struct blk_mq_ctx *ctx = rq->mq_ctx;
347	356
348	357	ctx->rq_completed[rq_is_sync(rq)]++;
349		− __blk_mq_free_request(hctx, ctx, rq);
	358	+ __blk_mq_finish_request(hctx, ctx, rq);
	359	+}
	360	+
	361	+void blk_mq_finish_request(struct request *rq)
	362	+{
	363	+ blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
350	364	}
351	365
352	366	void blk_mq_free_request(struct request *rq)
353	367	{
354		− blk_mq_free_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
	368	+ blk_mq_sched_put_request(rq);
355	369	}
356	370	EXPORT_SYMBOL_GPL(blk_mq_free_request);
357	371
@@ -469,6 +483,8 @@ void blk_mq_start_request(struct request *rq) · 111 unmodified lines
469	483	{
470	484	struct request_queue *q = rq->q;
471	485
	486	+ blk_mq_sched_started_request(rq);
	487	+
472	488	trace_block_rq_issue(q, rq);
473	489
474	490	rq->resid_len = blk_rq_bytes(rq);
@@ -517,6 +533,7 @@ static void __blk_mq_requeue_request(struct request *rq) · 42 unmodified lines
517	533
518	534	trace_block_rq_requeue(q, rq);
519	535	wbt_requeue(q->rq_wb, &rq->issue_stat);
	536	+ blk_mq_sched_requeue_request(rq);
520	537
521	538	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
522	539	if (q->dma_drain_size && blk_rq_bytes(rq))
@@ -551,13 +568,13 @@ static void blk_mq_requeue_work(struct work_struct *work) · 28 unmodified lines
551	568
552	569	rq->rq_flags &= ~RQF_SOFTBARRIER;
553	570	list_del_init(&rq->queuelist);
554		− blk_mq_insert_request(rq, true, false, false);
	571	+ blk_mq_sched_insert_request(rq, true, false, false);
555	572	}
556	573
557	574	while (!list_empty(&rq_list)) {
558	575	rq = list_entry(rq_list.next, struct request, queuelist);
559	576	list_del_init(&rq->queuelist);
560		− blk_mq_insert_request(rq, false, false, false);
	577	+ blk_mq_sched_insert_request(rq, false, false, false);
561	578	}
562	579
563	580	blk_mq_run_hw_queues(q, false);
@@ -765,6 +782,12 @@ static bool blk_mq_attempt_merge(struct request_queue *q, · 201 unmodified lines
765	782	continue;
766	783
767	784	el_ret = blk_try_merge(rq, bio);
	785	+ if (el_ret == ELEVATOR_NO_MERGE)
	786	+ continue;
	787	+
	788	+ if (!blk_mq_sched_allow_merge(q, rq, bio))
	789	+ break;
	790	+
768	791	if (el_ret == ELEVATOR_BACK_MERGE) {
769	792	if (bio_attempt_back_merge(q, rq, bio)) {
770	793	ctx->rq_merged++;
@@ -824,6 +847,59 @@ static inline unsigned int queued_to_index(unsigned int queued) · 53 unmodified lines
824	847	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
825	848	}
826	849
	850	+static bool blk_mq_get_driver_tag(struct request *rq,
	851	+ struct blk_mq_hw_ctx **hctx, bool wait)
	852	+{
	853	+ struct blk_mq_alloc_data data = {
	854	+ .q = rq->q,
	855	+ .ctx = rq->mq_ctx,
	856	+ .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
	857	+ .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
	858	+ };
	859	+
	860	+ if (blk_mq_hctx_stopped(data.hctx))
	861	+ return false;
	862	+
	863	+ if (rq->tag != -1) {
	864	+done:
	865	+ if (hctx)
	866	+ *hctx = data.hctx;
	867	+ return true;
	868	+ }
	869	+
	870	+ rq->tag = blk_mq_get_tag(&data);
	871	+ if (rq->tag >= 0) {
	872	+ data.hctx->tags->rqs[rq->tag] = rq;
	873	+ goto done;
	874	+ }
	875	+
	876	+ return false;
	877	+}
	878	+
	879	+/*
	880	+ * If we fail getting a driver tag because all the driver tags are already
	881	+ * assigned and on the dispatch list, BUT the first entry does not have a
	882	+ * tag, then we could deadlock. For that case, move entries with assigned
	883	+ * driver tags to the front, leaving the set of tagged requests in the
	884	+ * same order, and the untagged set in the same order.
	885	+ */
	886	+static bool reorder_tags_to_front(struct list_head *list)
	887	+{
	888	+ struct request rq, tmp, *first = NULL;
	889	+
	890	+ list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
	891	+ if (rq == first)
	892	+ break;
	893	+ if (rq->tag != -1) {
	894	+ list_move(&rq->queuelist, list);
	895	+ if (!first)
	896	+ first = rq;
	897	+ }
	898	+ }
	899	+
	900	+ return first != NULL;
	901	+}
	902	+
827	903	bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx hctx, struct list_head list)
828	904	{
829	905	struct request_queue *q = hctx->queue;
@@ -846,6 +922,12 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx hctx, struct list_head list) · 16 unmodified lines
846	922	struct blk_mq_queue_data bd;
847	923
848	924	rq = list_first_entry(list, struct request, queuelist);
	925	+ if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
	926	+ if (!queued && reorder_tags_to_front(list))
	927	+ continue;
	928	+ blk_mq_sched_mark_restart(hctx);
	929	+ break;
	930	+ }
849	931	list_del_init(&rq->queuelist);
850	932
851	933	bd.rq = rq;
@@ -899,48 +981,17 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx hctx, struct list_head list) · 47 unmodified lines
899	981	* the requests in rq_list might get lost.
900	982	*
901	983	* blk_mq_run_hw_queue() already checks the STOPPED bit
902		− **/
903		− blk_mq_run_hw_queue(hctx, true);
	984	+ *
	985	+ * If RESTART is set, then let completion restart the queue
	986	+ * instead of potentially looping here.
	987	+ */
	988	+ if (!blk_mq_sched_needs_restart(hctx))
	989	+ blk_mq_run_hw_queue(hctx, true);
904	990	}
905	991
906	992	return ret != BLK_MQ_RQ_QUEUE_BUSY;
907	993	}
908	994
909		−/*
910		− * Run this hardware queue, pulling any software queues mapped to it in.
911		− * Note that this function currently has various problems around ordering
912		− * of IO. In particular, we'd like FIFO behaviour on handling existing
913		− * items on the hctx->dispatch list. Ignore that for now.
914		− */
915		−static void blk_mq_process_rq_list(struct blk_mq_hw_ctx *hctx)
916		−{
917		− LIST_HEAD(rq_list);
918		− LIST_HEAD(driver_list);
919		−
920		− if (unlikely(blk_mq_hctx_stopped(hctx)))
921		− return;
922		−
923		− hctx->run++;
924		−
925		− /*
926		− * Touch any software queue that has pending entries.
927		− */
928		− blk_mq_flush_busy_ctxs(hctx, &rq_list);
929		−
930		− /*
931		− * If we have previous entries on our dispatch list, grab them
932		− * and stuff them at the front for more fair dispatch.
933		− */
934		− if (!list_empty_careful(&hctx->dispatch)) {
935		− spin_lock(&hctx->lock);
936		− if (!list_empty(&hctx->dispatch))
937		− list_splice_init(&hctx->dispatch, &rq_list);
938		− spin_unlock(&hctx->lock);
939		− }
940		−
941		− blk_mq_dispatch_rq_list(hctx, &rq_list);
942		−}
943		−
944	995	static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
945	996	{
946	997	int srcu_idx;
@@ -950,11 +1001,11 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) · 3 unmodified lines
950	1001
951	1002	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
952	1003	rcu_read_lock();
953		− blk_mq_process_rq_list(hctx);
	1004	+ blk_mq_sched_dispatch_requests(hctx);
954	1005	rcu_read_unlock();
955	1006	} else {
956	1007	srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
957		− blk_mq_process_rq_list(hctx);
	1008	+ blk_mq_sched_dispatch_requests(hctx);
958	1009	srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
959	1010	}
960	1011	}
@@ -1010,8 +1061,7 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async) · 49 unmodified lines
1010	1061	int i;
1011	1062
1012	1063	queue_for_each_hw_ctx(q, hctx, i) {
1013		− if ((!blk_mq_hctx_has_pending(hctx) &&
1014		− list_empty_careful(&hctx->dispatch)) \|\|
	1064	+ if (!blk_mq_hctx_has_pending(hctx) \|\|
1015	1065	blk_mq_hctx_stopped(hctx))
1016	1066	continue;
1017	1067
@@ -1148,32 +1198,10 @@ void __blk_mq_insert_request(struct blk_mq_hw_ctx hctx, struct request rq, · 130 unmodified lines
1148	1198	blk_mq_hctx_mark_pending(hctx, ctx);
1149	1199	}
1150	1200
1151		−void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
1152		− bool async)
1153		−{
1154		− struct blk_mq_ctx *ctx = rq->mq_ctx;
1155		− struct request_queue *q = rq->q;
1156		− struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
1157		−
1158		− spin_lock(&ctx->lock);
1159		− __blk_mq_insert_request(hctx, rq, at_head);
1160		− spin_unlock(&ctx->lock);
1161		−
1162		− if (run_queue)
1163		− blk_mq_run_hw_queue(hctx, async);
1164		−}
1165		−
1166		−static void blk_mq_insert_requests(struct request_queue *q,
1167		− struct blk_mq_ctx *ctx,
1168		− struct list_head *list,
1169		− int depth,
1170		− bool from_schedule)
	1201	+void blk_mq_insert_requests(struct blk_mq_hw_ctx hctx, struct blk_mq_ctx ctx,
	1202	+ struct list_head *list)
1171	1203
1172	1204	{
1173		− struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
1174		−
1175		− trace_block_unplug(q, depth, !from_schedule);
1176		−
1177	1205	/*
1178	1206	* preemption doesn't flush plug list, so it's possible ctx->cpu is
1179	1207	* offline now
@@ -1189,8 +1217,6 @@ static void blk_mq_insert_requests(struct request_queue *q, · 9 unmodified lines
1189	1217	}
1190	1218	blk_mq_hctx_mark_pending(hctx, ctx);
1191	1219	spin_unlock(&ctx->lock);
1192		−
1193		− blk_mq_run_hw_queue(hctx, from_schedule);
1194	1220	}
1195	1221
1196	1222	static int plug_ctx_cmp(void priv, struct list_head a, struct list_head *b)
@@ -1226,9 +1252,10 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) · 29 unmodified lines
1226	1252	BUG_ON(!rq->q);
1227	1253	if (rq->mq_ctx != this_ctx) {
1228	1254	if (this_ctx) {
1229		− blk_mq_insert_requests(this_q, this_ctx,
1230		− &ctx_list, depth,
1231		− from_schedule);
	1255	+ trace_block_unplug(this_q, depth, from_schedule);
	1256	+ blk_mq_sched_insert_requests(this_q, this_ctx,
	1257	+ &ctx_list,
	1258	+ from_schedule);
1232	1259	}
1233	1260
1234	1261	this_ctx = rq->mq_ctx;
@@ -1245,8 +1272,9 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) · 10 unmodified lines
1245	1272	* on 'ctx_list'. Do those.
1246	1273	*/
1247	1274	if (this_ctx) {
1248		− blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1249		− from_schedule);
	1275	+ trace_block_unplug(this_q, depth, from_schedule);
	1276	+ blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
	1277	+ from_schedule);
1250	1278	}
1251	1279	}
1252	1280
@@ -1284,51 +1312,39 @@ insert_rq: · 31 unmodified lines
1284	1312	}
1285	1313
1286	1314	spin_unlock(&ctx->lock);
1287		− __blk_mq_free_request(hctx, ctx, rq);
	1315	+ __blk_mq_finish_request(hctx, ctx, rq);
1288	1316	return true;
1289	1317	}
1290	1318	}
1291	1319
1292		−static struct request blk_mq_map_request(struct request_queue q,
1293		− struct bio *bio,
1294		− struct blk_mq_alloc_data *data)
1295		−{
1296		− struct blk_mq_hw_ctx *hctx;
1297		− struct blk_mq_ctx *ctx;
1298		− struct request *rq;
1299		−
1300		− blk_queue_enter_live(q);
1301		− ctx = blk_mq_get_ctx(q);
1302		− hctx = blk_mq_map_queue(q, ctx->cpu);
1303		−
1304		− trace_block_getrq(q, bio, bio->bi_opf);
1305		− blk_mq_set_alloc_data(data, q, 0, ctx, hctx);
1306		− rq = __blk_mq_alloc_request(data, bio->bi_opf);
1307		−
1308		− data->hctx->queued++;
1309		− return rq;
1310		−}
1311		−
1312	1320	static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx hctx, struct request rq)
1313	1321	{
1314		− return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);
	1322	+ if (rq->tag != -1)
	1323	+ return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);
	1324	+
	1325	+ return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
1315	1326	}
1316	1327
1317	1328	static void blk_mq_try_issue_directly(struct request rq, blk_qc_t cookie)
1318	1329	{
1319		− int ret;
1320	1330	struct request_queue *q = rq->q;
1321		− struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);
1322	1331	struct blk_mq_queue_data bd = {
1323	1332	.rq = rq,
1324	1333	.list = NULL,
1325	1334	.last = 1
1326	1335	};
1327		− blk_qc_t new_cookie = request_to_qc_t(hctx, rq);
	1336	+ struct blk_mq_hw_ctx *hctx;
	1337	+ blk_qc_t new_cookie;
	1338	+ int ret;
1328	1339
1329		− if (blk_mq_hctx_stopped(hctx))
	1340	+ if (q->elevator)
1330	1341	goto insert;
1331	1342
	1343	+ if (!blk_mq_get_driver_tag(rq, &hctx, false))
	1344	+ goto insert;
	1345	+
	1346	+ new_cookie = request_to_qc_t(hctx, rq);
	1347	+
1332	1348	/*
1333	1349	* For OK queue, we are done. For error, kill it. Any other
1334	1350	* error (busy), just add it to our list as we previously
@@ -1350,7 +1366,7 @@ static void blk_mq_try_issue_directly(struct request rq, blk_qc_t cookie) · 15 unmodified lines
1350	1366	}
1351	1367
1352	1368	insert:
1353		− blk_mq_insert_request(rq, false, true, true);
	1369	+ blk_mq_sched_insert_request(rq, false, true, true);
1354	1370	}
1355	1371
1356	1372	/*
@@ -1383,9 +1399,14 @@ static blk_qc_t blk_mq_make_request(struct request_queue q, struct bio bio) · 26 unmodified lines
1383	1399	blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
1384	1400	return BLK_QC_T_NONE;
1385	1401
	1402	+ if (blk_mq_sched_bio_merge(q, bio))
	1403	+ return BLK_QC_T_NONE;
	1404	+
1386	1405	wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1387	1406
1388		− rq = blk_mq_map_request(q, bio, &data);
	1407	+ trace_block_getrq(q, bio, bio->bi_opf);
	1408	+
	1409	+ rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
1389	1410	if (unlikely(!rq)) {
1390	1411	__wbt_done(q->rq_wb, wb_acct);
1391	1412	return BLK_QC_T_NONE;
@@ -1397,6 +1418,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue q, struct bio bio) · 5 unmodified lines
1397	1418
1398	1419	if (unlikely(is_flush_fua)) {
1399	1420	blk_mq_bio_to_request(rq, bio);
	1421	+ blk_mq_get_driver_tag(rq, NULL, true);
1400	1422	blk_insert_flush(rq);
1401	1423	goto run_queue;
1402	1424	}
@@ -1447,6 +1469,12 @@ static blk_qc_t blk_mq_make_request(struct request_queue q, struct bio bio) · 44 unmodified lines
1447	1469	goto done;
1448	1470	}
1449	1471
	1472	+ if (q->elevator) {
	1473	+ blk_mq_put_ctx(data.ctx);
	1474	+ blk_mq_bio_to_request(rq, bio);
	1475	+ blk_mq_sched_insert_request(rq, false, true, true);
	1476	+ goto done;
	1477	+ }
1450	1478	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1451	1479	/*
1452	1480	* For a SYNC request, send it to the hardware immediately. For
@@ -1492,9 +1520,14 @@ static blk_qc_t blk_sq_make_request(struct request_queue q, struct bio bio) · 39 unmodified lines
1492	1520	} else
1493	1521	request_count = blk_plug_queued_count(q);
1494	1522
	1523	+ if (blk_mq_sched_bio_merge(q, bio))
	1524	+ return BLK_QC_T_NONE;
	1525	+
1495	1526	wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1496	1527
1497		− rq = blk_mq_map_request(q, bio, &data);
	1528	+ trace_block_getrq(q, bio, bio->bi_opf);
	1529	+
	1530	+ rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
1498	1531	if (unlikely(!rq)) {
1499	1532	__wbt_done(q->rq_wb, wb_acct);
1500	1533	return BLK_QC_T_NONE;
@@ -1506,6 +1539,7 @@ static blk_qc_t blk_sq_make_request(struct request_queue q, struct bio bio) · 5 unmodified lines
1506	1539
1507	1540	if (unlikely(is_flush_fua)) {
1508	1541	blk_mq_bio_to_request(rq, bio);
	1542	+ blk_mq_get_driver_tag(rq, NULL, true);
1509	1543	blk_insert_flush(rq);
1510	1544	goto run_queue;
1511	1545	}
@@ -1544,6 +1578,12 @@ static blk_qc_t blk_sq_make_request(struct request_queue q, struct bio bio) · 32 unmodified lines
1544	1578	return cookie;
1545	1579	}
1546	1580
	1581	+ if (q->elevator) {
	1582	+ blk_mq_put_ctx(data.ctx);
	1583	+ blk_mq_bio_to_request(rq, bio);
	1584	+ blk_mq_sched_insert_request(rq, false, true, true);
	1585	+ goto done;
	1586	+ }
1547	1587	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1548	1588	/*
1549	1589	* For a SYNC request, send it to the hardware immediately. For
@@ -1556,6 +1596,7 @@ run_queue: · 6 unmodified lines
1556	1596	}
1557	1597
1558	1598	blk_mq_put_ctx(data.ctx);
	1599	+done:
1559	1600	return cookie;
1560	1601	}
1561	1602
@@ -1925,9 +1966,11 @@ static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx) · 363 unmodified lines
1925	1966	static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
1926	1967	unsigned int hctx_idx)
1927	1968	{
1928		− blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
1929		− blk_mq_free_rq_map(set->tags[hctx_idx]);
1930		− set->tags[hctx_idx] = NULL;
	1969	+ if (set->tags[hctx_idx]) {
	1970	+ blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
	1971	+ blk_mq_free_rq_map(set->tags[hctx_idx]);
	1972	+ set->tags[hctx_idx] = NULL;
	1973	+ }
1931	1974	}
1932	1975
1933	1976	static void blk_mq_map_swqueue(struct request_queue *q,
@@ -2084,6 +2127,8 @@ void blk_mq_release(struct request_queue *q) · 150 unmodified lines
2084	2127	struct blk_mq_hw_ctx *hctx;
2085	2128	unsigned int i;
2086	2129
	2130	+ blk_mq_sched_teardown(q);
	2131	+
2087	2132	/* hctx kobj stays in hctx */
2088	2133	queue_for_each_hw_ctx(q, hctx, i) {
2089	2134	if (!hctx)
@@ -2504,14 +2549,22 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) · 414 unmodified lines
2504	2549	struct blk_mq_hw_ctx *hctx;
2505	2550	int i, ret;
2506	2551
2507		− if (!set \|\| nr > set->queue_depth)
	2552	+ if (!set)
2508	2553	return -EINVAL;
2509	2554
2510	2555	ret = 0;
2511	2556	queue_for_each_hw_ctx(q, hctx, i) {
2512	2557	if (!hctx->tags)
2513	2558	continue;
2514		− ret = blk_mq_tag_update_depth(hctx->tags, nr);
	2559	+ /*
	2560	+ * If we're using an MQ scheduler, just update the scheduler
	2561	+ * queue depth. This is similar to what the old code would do.
	2562	+ */
	2563	+ if (!hctx->sched_tags)
	2564	+ ret = blk_mq_tag_update_depth(hctx->tags,
	2565	+ min(nr, set->queue_depth));
	2566	+ else
	2567	+ ret = blk_mq_tag_update_depth(hctx->sched_tags, nr);
2515	2568	if (ret)
2516	2569	break;
2517	2570	}
@@ -2704,7 +2757,10 @@ bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) · 186 unmodified lines
2704	2757	blk_flush_plug_list(plug, false);
2705	2758
2706	2759	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
2707		− rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
	2760	+ if (!blk_qc_t_is_internal(cookie))
	2761	+ rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
	2762	+ else
	2763	+ rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
2708	2764
2709	2765	return __blk_mq_poll(hctx, rq);
2710	2766	}

block/blk-mq.h+7−1

@@ -52,6 +52,8 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set set, struct blk_mq_tags tags, · 51 unmodified lines
52	52	*/
53	53	void __blk_mq_insert_request(struct blk_mq_hw_ctx hctx, struct request rq,
54	54	bool at_head);
	55	+void blk_mq_insert_requests(struct blk_mq_hw_ctx hctx, struct blk_mq_ctx ctx,
	56	+ struct list_head *list);
55	57	/*
56	58	* CPU hotplug helpers
57	59	*/
@@ -124,6 +126,9 @@ static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data, · 66 unmodified lines
124	126
125	127	static inline struct blk_mq_tags blk_mq_tags_from_data(struct blk_mq_alloc_data data)
126	128	{
	129	+ if (data->flags & BLK_MQ_REQ_INTERNAL)
	130	+ return data->hctx->sched_tags;
	131	+
127	132	return data->hctx->tags;
128	133	}
129	134
@@ -132,8 +137,9 @@ static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data · 2 unmodified lines
132	137	*/
133	138	void blk_mq_rq_ctx_init(struct request_queue q, struct blk_mq_ctx ctx,
134	139	struct request *rq, unsigned int op);
135		−void __blk_mq_free_request(struct blk_mq_hw_ctx hctx, struct blk_mq_ctx ctx,
	140	+void __blk_mq_finish_request(struct blk_mq_hw_ctx hctx, struct blk_mq_ctx ctx,
136	141	struct request *rq);
	142	+void blk_mq_finish_request(struct request *rq);
137	143	struct request __blk_mq_alloc_request(struct blk_mq_alloc_data data,
138	144	unsigned int op);
139	145

block/blk-tag.c+1−0

@@ -272,6 +272,7 @@ void blk_queue_end_tag(struct request_queue q, struct request rq) · 271 unmodified lines
272	272	list_del_init(&rq->queuelist);
273	273	rq->rq_flags &= ~RQF_QUEUED;
274	274	rq->tag = -1;
	275	+ rq->internal_tag = -1;
275	276
276	277	if (unlikely(bqt->tag_index[tag] == NULL))
277	278	printk(KERN_ERR "%s: tag %d is missing\n",

block/elevator.c+159−45

@@ -40,6 +40,7 @@ · 39 unmodified lines
40	40	#include <trace/events/block.h>
41	41
42	42	#include "blk.h"
	43	+#include "blk-mq-sched.h"
43	44
44	45	static DEFINE_SPINLOCK(elv_list_lock);
45	46	static LIST_HEAD(elv_list);
@@ -58,7 +59,9 @@ static int elv_iosched_allow_bio_merge(struct request rq, struct bio bio) · 12 unmodified lines
58	59	struct request_queue *q = rq->q;
59	60	struct elevator_queue *e = q->elevator;
60	61
61		− if (e->type->ops.sq.elevator_allow_bio_merge_fn)
	62	+ if (e->uses_mq && e->type->ops.mq.allow_merge)
	63	+ return e->type->ops.mq.allow_merge(q, rq, bio);
	64	+ else if (!e->uses_mq && e->type->ops.sq.elevator_allow_bio_merge_fn)
62	65	return e->type->ops.sq.elevator_allow_bio_merge_fn(q, rq, bio);
63	66
64	67	return 1;
@@ -163,6 +166,7 @@ struct elevator_queue elevator_alloc(struct request_queue q, · 98 unmodified lines
163	166	kobject_init(&eq->kobj, &elv_ktype);
164	167	mutex_init(&eq->sysfs_lock);
165	168	hash_init(eq->hash);
	169	+ eq->uses_mq = e->uses_mq;
166	170
167	171	return eq;
168	172	}
@@ -219,14 +223,26 @@ int elevator_init(struct request_queue q, char name) · 50 unmodified lines
219	223	if (!e) {
220	224	printk(KERN_ERR
221	225	"Default I/O scheduler not found. " \
222		− "Using noop.\n");
	226	+ "Using noop/none.\n");
	227	+ if (q->mq_ops) {
	228	+ elevator_put(e);
	229	+ return 0;
	230	+ }
223	231	e = elevator_get("noop", false);
224	232	}
225	233	}
226	234
227		− err = e->ops.sq.elevator_init_fn(q, e);
228		− if (err)
	235	+ if (e->uses_mq) {
	236	+ err = blk_mq_sched_setup(q);
	237	+ if (!err)
	238	+ err = e->ops.mq.init_sched(q, e);
	239	+ } else
	240	+ err = e->ops.sq.elevator_init_fn(q, e);
	241	+ if (err) {
	242	+ if (e->uses_mq)
	243	+ blk_mq_sched_teardown(q);
229	244	elevator_put(e);
	245	+ }
230	246	return err;
231	247	}
232	248	EXPORT_SYMBOL(elevator_init);
@@ -234,7 +250,9 @@ EXPORT_SYMBOL(elevator_init); · 1 unmodified line
234	250	void elevator_exit(struct elevator_queue *e)
235	251	{
236	252	mutex_lock(&e->sysfs_lock);
237		− if (e->type->ops.sq.elevator_exit_fn)
	253	+ if (e->uses_mq && e->type->ops.mq.exit_sched)
	254	+ e->type->ops.mq.exit_sched(e);
	255	+ else if (!e->uses_mq && e->type->ops.sq.elevator_exit_fn)
238	256	e->type->ops.sq.elevator_exit_fn(e);
239	257	mutex_unlock(&e->sysfs_lock);
240	258
@@ -253,6 +271,7 @@ void elv_rqhash_del(struct request_queue q, struct request rq) · 12 unmodified lines
253	271	if (ELV_ON_HASH(rq))
254	272	__elv_rqhash_del(rq);
255	273	}
	274	+EXPORT_SYMBOL_GPL(elv_rqhash_del);
256	275
257	276	void elv_rqhash_add(struct request_queue q, struct request rq)
258	277	{
@@ -262,6 +281,7 @@ void elv_rqhash_add(struct request_queue q, struct request rq) · 3 unmodified lines
262	281	hash_add(e->hash, &rq->hash, rq_hash_key(rq));
263	282	rq->rq_flags \|= RQF_HASHED;
264	283	}
	284	+EXPORT_SYMBOL_GPL(elv_rqhash_add);
265	285
266	286	void elv_rqhash_reposition(struct request_queue q, struct request rq)
267	287	{
@@ -443,7 +463,9 @@ int elv_merge(struct request_queue q, struct request req, struct bio bio) · 175 unmodified lines
443	463	return ELEVATOR_BACK_MERGE;
444	464	}
445	465
446		− if (e->type->ops.sq.elevator_merge_fn)
	466	+ if (e->uses_mq && e->type->ops.mq.request_merge)
	467	+ return e->type->ops.mq.request_merge(q, req, bio);
	468	+ else if (!e->uses_mq && e->type->ops.sq.elevator_merge_fn)
447	469	return e->type->ops.sq.elevator_merge_fn(q, req, bio);
448	470
449	471	return ELEVATOR_NO_MERGE;
@@ -456,8 +478,7 @@ int elv_merge(struct request_queue q, struct request req, struct bio bio) · 6 unmodified lines
456	478	*
457	479	* Returns true if we merged, false otherwise
458	480	*/
459		−static bool elv_attempt_insert_merge(struct request_queue *q,
460		− struct request *rq)
	481	+bool elv_attempt_insert_merge(struct request_queue q, struct request rq)
461	482	{
462	483	struct request *__rq;
463	484	bool ret;
@@ -495,7 +516,9 @@ void elv_merged_request(struct request_queue q, struct request rq, int type) · 31 unmodified lines
495	516	{
496	517	struct elevator_queue *e = q->elevator;
497	518
498		− if (e->type->ops.sq.elevator_merged_fn)
	519	+ if (e->uses_mq && e->type->ops.mq.request_merged)
	520	+ e->type->ops.mq.request_merged(q, rq, type);
	521	+ else if (!e->uses_mq && e->type->ops.sq.elevator_merged_fn)
499	522	e->type->ops.sq.elevator_merged_fn(q, rq, type);
500	523
501	524	if (type == ELEVATOR_BACK_MERGE)
@@ -508,10 +531,15 @@ void elv_merge_requests(struct request_queue q, struct request rq, · 6 unmodified lines
508	531	struct request *next)
509	532	{
510	533	struct elevator_queue *e = q->elevator;
511		− const int next_sorted = next->rq_flags & RQF_SORTED;
512		−
513		− if (next_sorted && e->type->ops.sq.elevator_merge_req_fn)
514		− e->type->ops.sq.elevator_merge_req_fn(q, rq, next);
	534	+ bool next_sorted = false;
	535	+
	536	+ if (e->uses_mq && e->type->ops.mq.requests_merged)
	537	+ e->type->ops.mq.requests_merged(q, rq, next);
	538	+ else if (e->type->ops.sq.elevator_merge_req_fn) {
	539	+ next_sorted = next->rq_flags & RQF_SORTED;
	540	+ if (next_sorted)
	541	+ e->type->ops.sq.elevator_merge_req_fn(q, rq, next);
	542	+ }
515	543
516	544	elv_rqhash_reposition(q, rq);
517	545
@@ -528,6 +556,9 @@ void elv_bio_merged(struct request_queue q, struct request rq, · 10 unmodified lines
528	556	{
529	557	struct elevator_queue *e = q->elevator;
530	558
	559	+ if (WARN_ON_ONCE(e->uses_mq))
	560	+ return;
	561	+
531	562	if (e->type->ops.sq.elevator_bio_merged_fn)
532	563	e->type->ops.sq.elevator_bio_merged_fn(q, rq, bio);
533	564	}
@@ -574,11 +605,15 @@ void elv_requeue_request(struct request_queue q, struct request rq) · 40 unmodified lines
574	605
575	606	void elv_drain_elevator(struct request_queue *q)
576	607	{
	608	+ struct elevator_queue *e = q->elevator;
577	609	static int printed;
578	610
	611	+ if (WARN_ON_ONCE(e->uses_mq))
	612	+ return;
	613	+
579	614	lockdep_assert_held(q->queue_lock);
580	615
581		− while (q->elevator->type->ops.sq.elevator_dispatch_fn(q, 1))
	616	+ while (e->type->ops.sq.elevator_dispatch_fn(q, 1))
582	617	;
583	618	if (q->nr_sorted && printed++ < 10) {
584	619	printk(KERN_ERR "%s: forced dispatching is broken "
@@ -682,8 +717,11 @@ struct request elv_latter_request(struct request_queue q, struct request *rq) · 97 unmodified lines
682	717	{
683	718	struct elevator_queue *e = q->elevator;
684	719
685		− if (e->type->ops.sq.elevator_latter_req_fn)
	720	+ if (e->uses_mq && e->type->ops.mq.next_request)
	721	+ return e->type->ops.mq.next_request(q, rq);
	722	+ else if (!e->uses_mq && e->type->ops.sq.elevator_latter_req_fn)
686	723	return e->type->ops.sq.elevator_latter_req_fn(q, rq);
	724	+
687	725	return NULL;
688	726	}
689	727
@@ -691,7 +729,9 @@ struct request elv_former_request(struct request_queue q, struct request *rq) · 1 unmodified line
691	729	{
692	730	struct elevator_queue *e = q->elevator;
693	731
694		− if (e->type->ops.sq.elevator_former_req_fn)
	732	+ if (e->uses_mq && e->type->ops.mq.former_request)
	733	+ return e->type->ops.mq.former_request(q, rq);
	734	+ if (!e->uses_mq && e->type->ops.sq.elevator_former_req_fn)
695	735	return e->type->ops.sq.elevator_former_req_fn(q, rq);
696	736	return NULL;
697	737	}
@@ -701,6 +741,9 @@ int elv_set_request(struct request_queue q, struct request rq, · 3 unmodified lines
701	741	{
702	742	struct elevator_queue *e = q->elevator;
703	743
	744	+ if (WARN_ON_ONCE(e->uses_mq))
	745	+ return 0;
	746	+
704	747	if (e->type->ops.sq.elevator_set_req_fn)
705	748	return e->type->ops.sq.elevator_set_req_fn(q, rq, bio, gfp_mask);
706	749	return 0;
@@ -710,6 +753,9 @@ void elv_put_request(struct request_queue q, struct request rq) · 3 unmodified lines
710	753	{
711	754	struct elevator_queue *e = q->elevator;
712	755
	756	+ if (WARN_ON_ONCE(e->uses_mq))
	757	+ return;
	758	+
713	759	if (e->type->ops.sq.elevator_put_req_fn)
714	760	e->type->ops.sq.elevator_put_req_fn(rq);
715	761	}
@@ -718,6 +764,9 @@ int elv_may_queue(struct request_queue *q, unsigned int op) · 2 unmodified lines
718	764	{
719	765	struct elevator_queue *e = q->elevator;
720	766
	767	+ if (WARN_ON_ONCE(e->uses_mq))
	768	+ return 0;
	769	+
721	770	if (e->type->ops.sq.elevator_may_queue_fn)
722	771	return e->type->ops.sq.elevator_may_queue_fn(q, op);
723	772
@@ -728,6 +777,9 @@ void elv_completed_request(struct request_queue q, struct request rq) · 4 unmodified lines
728	777	{
729	778	struct elevator_queue *e = q->elevator;
730	779
	780	+ if (WARN_ON_ONCE(e->uses_mq))
	781	+ return;
	782	+
731	783	/*
732	784	* request is released from the driver, io must be done
733	785	*/
@@ -803,7 +855,7 @@ int elv_register_queue(struct request_queue *q) · 69 unmodified lines
803	855	}
804	856	kobject_uevent(&e->kobj, KOBJ_ADD);
805	857	e->registered = 1;
806		− if (e->type->ops.sq.elevator_registered_fn)
	858	+ if (!e->uses_mq && e->type->ops.sq.elevator_registered_fn)
807	859	e->type->ops.sq.elevator_registered_fn(q);
808	860	}
809	861	return error;
@@ -891,9 +943,14 @@ EXPORT_SYMBOL_GPL(elv_unregister); · 81 unmodified lines
891	943	static int elevator_switch(struct request_queue q, struct elevator_type new_e)
892	944	{
893	945	struct elevator_queue *old = q->elevator;
894		− bool registered = old->registered;
	946	+ bool old_registered = false;
895	947	int err;
896	948
	949	+ if (q->mq_ops) {
	950	+ blk_mq_freeze_queue(q);
	951	+ blk_mq_quiesce_queue(q);
	952	+ }
	953	+
897	954	/*
898	955	* Turn on BYPASS and drain all requests w/ elevator private data.
899	956	* Block layer doesn't call into a quiesced elevator - all requests
@@ -901,42 +958,76 @@ static int elevator_switch(struct request_queue q, struct elevator_type new_e) · 1 unmodified line
901	958	* using INSERT_BACK. All requests have SOFTBARRIER set and no
902	959	* merge happens either.
903	960	*/
904		− blk_queue_bypass_start(q);
	961	+ if (old) {
	962	+ old_registered = old->registered;
	963	+
	964	+ if (old->uses_mq)
	965	+ blk_mq_sched_teardown(q);
905	966
906		− /* unregister and clear all auxiliary data of the old elevator */
907		− if (registered)
908		− elv_unregister_queue(q);
	967	+ if (!q->mq_ops)
	968	+ blk_queue_bypass_start(q);
909	969
910		− spin_lock_irq(q->queue_lock);
911		− ioc_clear_queue(q);
912		− spin_unlock_irq(q->queue_lock);
	970	+ /* unregister and clear all auxiliary data of the old elevator */
	971	+ if (old_registered)
	972	+ elv_unregister_queue(q);
	973	+
	974	+ spin_lock_irq(q->queue_lock);
	975	+ ioc_clear_queue(q);
	976	+ spin_unlock_irq(q->queue_lock);
	977	+ }
913	978
914	979	/* allocate, init and register new elevator */
915		− err = new_e->ops.sq.elevator_init_fn(q, new_e);
916		− if (err)
917		− goto fail_init;
	980	+ if (new_e) {
	981	+ if (new_e->uses_mq) {
	982	+ err = blk_mq_sched_setup(q);
	983	+ if (!err)
	984	+ err = new_e->ops.mq.init_sched(q, new_e);
	985	+ } else
	986	+ err = new_e->ops.sq.elevator_init_fn(q, new_e);
	987	+ if (err)
	988	+ goto fail_init;
918	989
919		− if (registered) {
920	990	err = elv_register_queue(q);
921	991	if (err)
922	992	goto fail_register;
923		− }
	993	+ } else
	994	+ q->elevator = NULL;
924	995
925	996	/* done, kill the old one and finish */
926		− elevator_exit(old);
927		− blk_queue_bypass_end(q);
	997	+ if (old) {
	998	+ elevator_exit(old);
	999	+ if (!q->mq_ops)
	1000	+ blk_queue_bypass_end(q);
	1001	+ }
	1002	+
	1003	+ if (q->mq_ops) {
	1004	+ blk_mq_unfreeze_queue(q);
	1005	+ blk_mq_start_stopped_hw_queues(q, true);
	1006	+ }
928	1007
929		− blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
	1008	+ if (new_e)
	1009	+ blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
	1010	+ else
	1011	+ blk_add_trace_msg(q, "elv switch: none");
930	1012
931	1013	return 0;
932	1014
933	1015	fail_register:
	1016	+ if (q->mq_ops)
	1017	+ blk_mq_sched_teardown(q);
934	1018	elevator_exit(q->elevator);
935	1019	fail_init:
936	1020	/* switch failed, restore and re-register old elevator */
937		− q->elevator = old;
938		− elv_register_queue(q);
939		− blk_queue_bypass_end(q);
	1021	+ if (old) {
	1022	+ q->elevator = old;
	1023	+ elv_register_queue(q);
	1024	+ if (!q->mq_ops)
	1025	+ blk_queue_bypass_end(q);
	1026	+ }
	1027	+ if (q->mq_ops) {
	1028	+ blk_mq_unfreeze_queue(q);
	1029	+ blk_mq_start_stopped_hw_queues(q, true);
	1030	+ }
940	1031
941	1032	return err;
942	1033	}
@@ -949,8 +1040,11 @@ static int __elevator_change(struct request_queue q, const char name) · 6 unmodified lines
949	1040	char elevator_name[ELV_NAME_MAX];
950	1041	struct elevator_type *e;
951	1042
952		− if (!q->elevator)
953		− return -ENXIO;
	1043	+ /*
	1044	+ * Special case for mq, turn off scheduling
	1045	+ */
	1046	+ if (q->mq_ops && !strncmp(name, "none", 4))
	1047	+ return elevator_switch(q, NULL);
954	1048
955	1049	strlcpy(elevator_name, name, sizeof(elevator_name));
956	1050	e = elevator_get(strstrip(elevator_name), true);
@@ -959,11 +1053,21 @@ static int __elevator_change(struct request_queue q, const char name) · 2 unmodified lines
959	1053	return -EINVAL;
960	1054	}
961	1055
962		− if (!strcmp(elevator_name, q->elevator->type->elevator_name)) {
	1056	+ if (q->elevator &&
	1057	+ !strcmp(elevator_name, q->elevator->type->elevator_name)) {
963	1058	elevator_put(e);
964	1059	return 0;
965	1060	}
966	1061
	1062	+ if (!e->uses_mq && q->mq_ops) {
	1063	+ elevator_put(e);
	1064	+ return -EINVAL;
	1065	+ }
	1066	+ if (e->uses_mq && !q->mq_ops) {
	1067	+ elevator_put(e);
	1068	+ return -EINVAL;
	1069	+ }
	1070	+
967	1071	return elevator_switch(q, e);
968	1072	}
969	1073
@@ -985,7 +1089,7 @@ ssize_t elv_iosched_store(struct request_queue q, const char name, · 15 unmodified lines
985	1089	{
986	1090	int ret;
987	1091
988		− if (!q->elevator)
	1092	+ if (!(q->mq_ops \|\| q->request_fn))
989	1093	return count;
990	1094
991	1095	ret = __elevator_change(q, name);
@@ -999,24 +1103,34 @@ ssize_t elv_iosched_store(struct request_queue q, const char name, · 7 unmodified lines
999	1103	ssize_t elv_iosched_show(struct request_queue q, char name)
1000	1104	{
1001	1105	struct elevator_queue *e = q->elevator;
1002		− struct elevator_type *elv;
	1106	+ struct elevator_type *elv = NULL;
1003	1107	struct elevator_type *__e;
1004	1108	int len = 0;
1005	1109
1006		− if (!q->elevator \|\| !blk_queue_stackable(q))
	1110	+ if (!blk_queue_stackable(q))
1007	1111	return sprintf(name, "none\n");
1008	1112
1009		− elv = e->type;
	1113	+ if (!q->elevator)
	1114	+ len += sprintf(name+len, "[none] ");
	1115	+ else
	1116	+ elv = e->type;
1010	1117
1011	1118	spin_lock(&elv_list_lock);
1012	1119	list_for_each_entry(__e, &elv_list, list) {
1013		− if (!strcmp(elv->elevator_name, __e->elevator_name))
	1120	+ if (elv && !strcmp(elv->elevator_name, __e->elevator_name)) {
1014	1121	len += sprintf(name+len, "[%s] ", elv->elevator_name);
1015		− else
	1122	+ continue;
	1123	+ }
	1124	+ if (__e->uses_mq && q->mq_ops)
	1125	+ len += sprintf(name+len, "%s ", __e->elevator_name);
	1126	+ else if (!__e->uses_mq && !q->mq_ops)
1016	1127	len += sprintf(name+len, "%s ", __e->elevator_name);
1017	1128	}
1018	1129	spin_unlock(&elv_list_lock);
1019	1130
	1131	+ if (q->mq_ops && q->elevator)
	1132	+ len += sprintf(name+len, "none");
	1133	+
1020	1134	len += sprintf(len+name, "\n");
1021	1135	return len;
1022	1136	}

include/linux/blk-mq.h+4−1

@@ -22,6 +22,7 @@ struct blk_mq_hw_ctx { · 21 unmodified lines
22	22
23	23	unsigned long flags; /* BLK_MQ_F_* flags */
24	24
	25	+ void *sched_data;
25	26	struct request_queue *queue;
26	27	struct blk_flush_queue *fq;
27	28
@@ -35,6 +36,7 @@ struct blk_mq_hw_ctx { · 7 unmodified lines
35	36	atomic_t wait_index;
36	37
37	38	struct blk_mq_tags *tags;
	39	+ struct blk_mq_tags *sched_tags;
38	40
39	41	struct srcu_struct queue_rq_srcu;
40	42
@@ -156,6 +158,7 @@ enum { · 115 unmodified lines
156	158
157	159	BLK_MQ_S_STOPPED = 0,
158	160	BLK_MQ_S_TAG_ACTIVE = 1,
	161	+ BLK_MQ_S_SCHED_RESTART = 2,
159	162
160	163	BLK_MQ_MAX_DEPTH = 10240,
161	164
@@ -179,13 +182,13 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set); · 17 unmodified lines
179	182
180	183	void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
181	184
182		−void blk_mq_insert_request(struct request *, bool, bool, bool);
183	185	void blk_mq_free_request(struct request *rq);
184	186	bool blk_mq_can_queue(struct blk_mq_hw_ctx *);
185	187
186	188	enum {
187	189	BLK_MQ_REQ_NOWAIT = (1 << 0), /* return when out of requests */
188	190	BLK_MQ_REQ_RESERVED = (1 << 1), /* allocate from reserved pool */
	191	+ BLK_MQ_REQ_INTERNAL = (1 << 2), /* allocate internal/sched tag */
189	192	};
190	193
191	194	struct request blk_mq_alloc_request(struct request_queue q, int rw,

include/linux/blkdev.h+3−1

@@ -154,6 +154,7 @@ struct request { · 153 unmodified lines
154	154
155	155	/* the following two fields are internal, NEVER access directly */
156	156	unsigned int __data_len; /* total data len */
	157	+ int tag;
157	158	sector_t __sector; /* sector cursor */
158	159
159	160	struct bio *bio;
@@ -220,9 +221,10 @@ struct request { · 60 unmodified lines
220	221
221	222	unsigned short ioprio;
222	223
	224	+ int internal_tag;
	225	+
223	226	void special; / opaque pointer available for LLD use */
224	227
225		− int tag;
226	228	int errors;
227	229
228	230	/*

include/linux/elevator.h+32−0

@@ -77,6 +77,34 @@ struct elevator_ops · 76 unmodified lines
77	77	elevator_registered_fn *elevator_registered_fn;
78	78	};
79	79
	80	+struct blk_mq_alloc_data;
	81	+struct blk_mq_hw_ctx;
	82	+
	83	+struct elevator_mq_ops {
	84	+ int (init_sched)(struct request_queue , struct elevator_type *);
	85	+ void (exit_sched)(struct elevator_queue );
	86	+
	87	+ bool (allow_merge)(struct request_queue , struct request , struct bio );
	88	+ bool (bio_merge)(struct blk_mq_hw_ctx , struct bio *);
	89	+ int (request_merge)(struct request_queue q, struct request *, struct bio );
	90	+ void (request_merged)(struct request_queue , struct request *, int);
	91	+ void (requests_merged)(struct request_queue , struct request , struct request );
	92	+ struct request (get_request)(struct request_queue , unsigned int, struct blk_mq_alloc_data );
	93	+ void (put_request)(struct request );
	94	+ void (insert_requests)(struct blk_mq_hw_ctx , struct list_head *, bool);
	95	+ void (dispatch_requests)(struct blk_mq_hw_ctx , struct list_head *);
	96	+ bool (has_work)(struct blk_mq_hw_ctx );
	97	+ void (completed_request)(struct blk_mq_hw_ctx , struct request *);
	98	+ void (started_request)(struct request );
	99	+ void (requeue_request)(struct request );
	100	+ struct request (former_request)(struct request_queue , struct request );
	101	+ struct request (next_request)(struct request_queue , struct request );
	102	+ int (get_rq_priv)(struct request_queue , struct request *);
	103	+ void (put_rq_priv)(struct request_queue , struct request *);
	104	+ void (init_icq)(struct io_cq );
	105	+ void (exit_icq)(struct io_cq );
	106	+};
	107	+
80	108	#define ELV_NAME_MAX (16)
81	109
82	110	struct elv_fs_entry {
@@ -96,12 +124,14 @@ struct elevator_type · 13 unmodified lines
96	124	/* fields provided by elevator implementation */
97	125	union {
98	126	struct elevator_ops sq;
	127	+ struct elevator_mq_ops mq;
99	128	} ops;
100	129	size_t icq_size; /* see iocontext.h */
101	130	size_t icq_align; /* ditto */
102	131	struct elv_fs_entry *elevator_attrs;
103	132	char elevator_name[ELV_NAME_MAX];
104	133	struct module *elevator_owner;
	134	+ bool uses_mq;
105	135
106	136	/* managed by elevator core */
107	137	char icq_cache_name[ELV_NAME_MAX + 5]; /* elvname + "_io_cq" */
@@ -125,6 +155,7 @@ struct elevator_queue · 17 unmodified lines
125	155	struct kobject kobj;
126	156	struct mutex sysfs_lock;
127	157	unsigned int registered:1;
	158	+ unsigned int uses_mq:1;
128	159	DECLARE_HASHTABLE(hash, ELV_HASH_BITS);
129	160	};
130	161
@@ -141,6 +172,7 @@ extern void elv_merge_requests(struct request_queue , struct request , · 10 unmodified lines
141	172	extern void elv_merged_request(struct request_queue , struct request , int);
142	173	extern void elv_bio_merged(struct request_queue q, struct request ,
143	174	struct bio *);
	175	+extern bool elv_attempt_insert_merge(struct request_queue , struct request );
144	176	extern void elv_requeue_request(struct request_queue , struct request );
145	177	extern struct request elv_former_request(struct request_queue , struct request *);
146	178	extern struct request elv_latter_request(struct request_queue , struct request *);