History

blk-mq: new multi-queue block IO queueing mechanism

Linux currently has two models for block devices: - The classic request_fn based approach, where drivers use struct request units for IO. The block layer provides various helper functionalities to let drivers share code, things like tag management, timeout handling, queueing, etc. - The "stacked" approach, where a driver squeezes in between the block layer and IO submitter. Since this bypasses the IO stack, driver generally have to manage everything themselves. With drivers being written for new high IOPS devices, the classic request_fn based driver doesn't work well enough. The design dates back to when both SMP and high IOPS was rare. It has problems with scaling to bigger machines, and runs into scaling issues even on smaller machines when you have IOPS in the hundreds of thousands per device. The stacked approach is then most often selected as the model for the driver. But this means that everybody has to re-invent everything, and along with that we get all the problems again that the shared approach solved. This commit introduces blk-mq, block multi queue support. The design is centered around per-cpu queues for queueing IO, which then funnel down into x number of hardware submission queues. We might have a 1:1 mapping between the two, or it might be an N:M mapping. That all depends on what the hardware supports. blk-mq provides various helper functions, which include: - Scalable support for request tagging. Most devices need to be able to uniquely identify a request both in the driver and to the hardware. The tagging uses per-cpu caches for freed tags, to enable cache hot reuse. - Timeout handling without tracking request on a per-device basis. Basically the driver should be able to get a notification, if a request happens to fail. - Optional support for non 1:1 mappings between issue and submission queues. blk-mq can redirect IO completions to the desired location. - Support for per-request payloads. Drivers almost always need to associate a request structure with some driver private command structure. Drivers can tell blk-mq this at init time, and then any request handed to the driver will have the required size of memory associated with it. - Support for merging of IO, and plugging. The stacked model gets neither of these. Even for high IOPS devices, merging sequential IO reduces per-command overhead and thus increases bandwidth. For now, this is provided as a potential 3rd queueing model, with the hope being that, as it matures, it can replace both the classic and stacked model. That would get us back to having just 1 real model for block devices, leaving the stacked approach to dm/md devices (as it was originally intended). Contributions in this patch from the following people: Shaohua Li <shli@fusionio.com> Alexander Gordeev <agordeev@redhat.com> Christoph Hellwig <hch@infradead.org> Mike Christie <michaelc@cs.wisc.edu> Matias Bjorling <m@bjorling.me> Jeff Moyer <jmoyer@redhat.com>

Jens Axboe committed 13 years ago

Commit 320ae511 parent 1dddc0118 files · +2890 −109Browse files

Unified · Split

18 files
  1. block/Makefile +3 −2
  2. block/blk-core.c +84 −58
  3. block/blk-exec.c +7 −0
  4. block/blk-flush.c +139 −15
  5. block/blk-mq-cpu.c +93 −0
  6. block/blk-mq-cpumap.c +108 −0
  7. block/blk-mq-sysfs.c +384 −0
  8. block/blk-mq-tag.c +204 −0
  9. block/blk-mq-tag.h +27 −0
  10. block/blk-mq.c +1480 −0
  11. block/blk-mq.h +52 −0
  12. block/blk-sysfs.c +13 −0
  13. block/blk-timeout.c +46 −27
  14. block/blk.h +17 −0
  15. include/linux/bio.h +2 −0
  16. include/linux/blk-mq.h +182 −0
  17. include/linux/blk_types.h +2 −0
  18. include/linux/blkdev.h +47 −7
block/Makefile+3−2
@@ -5,8 +5,9 @@ · 4 unmodified lines
55 obj-$(CONFIG_BLOCK) := elevator.o blk-core.o blk-tag.o blk-sysfs.o \
66 blk-flush.o blk-settings.o blk-ioc.o blk-map.o \
77 blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \
8− blk-iopoll.o blk-lib.o ioctl.o genhd.o scsi_ioctl.o \
9− partition-generic.o partitions/
8+ blk-iopoll.o blk-lib.o blk-mq.o blk-mq-tag.o \
9+ blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \
10+ genhd.o scsi_ioctl.o partition-generic.o partitions/
1011
1112 obj-$(CONFIG_BLK_DEV_BSG) += bsg.o
1213 obj-$(CONFIG_BLK_DEV_BSGLIB) += bsg-lib.o
block/blk-core.c+84−58
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #include <linux/backing-dev.h>
1717 #include <linux/bio.h>
1818 #include <linux/blkdev.h>
19+#include <linux/blk-mq.h>
1920 #include <linux/highmem.h>
2021 #include <linux/mm.h>
2122 #include <linux/kernel_stat.h>
@@ -48,7 +49,7 @@ DEFINE_IDA(blk_queue_ida); · 26 unmodified lines
4849 /*
4950 * For the allocated request tables
5051 */
51−static struct kmem_cache *request_cachep;
52+struct kmem_cache *request_cachep = NULL;
5253
5354 /*
5455 * For queue allocation
@@ -60,42 +61,6 @@ struct kmem_cache *blk_requestq_cachep; · 5 unmodified lines
6061 */
6162 static struct workqueue_struct *kblockd_workqueue;
6263
63−static void drive_stat_acct(struct request *rq, int new_io)
64−{
65− struct hd_struct *part;
66− int rw = rq_data_dir(rq);
67− int cpu;
68−
69− if (!blk_do_io_stat(rq))
70− return;
71−
72− cpu = part_stat_lock();
73−
74− if (!new_io) {
75− part = rq->part;
76− part_stat_inc(cpu, part, merges[rw]);
77− } else {
78− part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
79− if (!hd_struct_try_get(part)) {
80− /*
81− * The partition is already being removed,
82− * the request will be accounted on the disk only
83− *
84− * We take a reference on disk->part0 although that
85− * partition will never be deleted, so we can treat
86− * it as any other partition.
87− */
88− part = &rq->rq_disk->part0;
89− hd_struct_get(part);
90− }
91− part_round_stats(cpu, part);
92− part_inc_in_flight(part, rw);
93− rq->part = part;
94− }
95−
96− part_stat_unlock();
97−}
98−
9964 void blk_queue_congestion_threshold(struct request_queue *q)
10065 {
10166 int nr;
@@ -594,9 +559,12 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) · 492 unmodified lines
594559 if (!q)
595560 return NULL;
596561
562+ if (percpu_counter_init(&q->mq_usage_counter, 0))
563+ goto fail_q;
564+
597565 q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
598566 if (q->id < 0)
599− goto fail_q;
567+ goto fail_c;
600568
601569 q->backing_dev_info.ra_pages =
602570 (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
@@ -643,6 +611,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) · 40 unmodified lines
643611 q->bypass_depth = 1;
644612 __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
645613
614+ init_waitqueue_head(&q->mq_freeze_wq);
615+
646616 if (blkcg_init_queue(q))
647617 goto fail_id;
648618
@@ -650,6 +620,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) · 1 unmodified line
650620
651621 fail_id:
652622 ida_simple_remove(&blk_queue_ida, q->id);
623+fail_c:
624+ percpu_counter_destroy(&q->mq_usage_counter);
653625 fail_q:
654626 kmem_cache_free(blk_requestq_cachep, q);
655627 return NULL;
@@ -1108,7 +1080,8 @@ retry: · 452 unmodified lines
11081080 goto retry;
11091081 }
11101082
1111−struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
1083+static struct request *blk_old_get_request(struct request_queue *q, int rw,
1084+ gfp_t gfp_mask)
11121085 {
11131086 struct request *rq;
11141087
@@ -1125,6 +1098,14 @@ struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask) · 10 unmodified lines
11251098
11261099 return rq;
11271100 }
1101+
1102+struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
1103+{
1104+ if (q->mq_ops)
1105+ return blk_mq_alloc_request(q, rw, gfp_mask);
1106+ else
1107+ return blk_old_get_request(q, rw, gfp_mask);
1108+}
11281109 EXPORT_SYMBOL(blk_get_request);
11291110
11301111 /**
@@ -1210,7 +1191,7 @@ EXPORT_SYMBOL(blk_requeue_request); · 79 unmodified lines
12101191 static void add_acct_request(struct request_queue *q, struct request *rq,
12111192 int where)
12121193 {
1213− drive_stat_acct(rq, 1);
1194+ blk_account_io_start(rq, true);
12141195 __elv_add_request(q, rq, where);
12151196 }
12161197
@@ -1299,12 +1280,17 @@ EXPORT_SYMBOL_GPL(__blk_put_request); · 82 unmodified lines
12991280
13001281 void blk_put_request(struct request *req)
13011282 {
1302− unsigned long flags;
13031283 struct request_queue *q = req->q;
13041284
1305− spin_lock_irqsave(q->queue_lock, flags);
1306− __blk_put_request(q, req);
1307− spin_unlock_irqrestore(q->queue_lock, flags);
1285+ if (q->mq_ops)
1286+ blk_mq_free_request(req);
1287+ else {
1288+ unsigned long flags;
1289+
1290+ spin_lock_irqsave(q->queue_lock, flags);
1291+ __blk_put_request(q, req);
1292+ spin_unlock_irqrestore(q->queue_lock, flags);
1293+ }
13081294 }
13091295 EXPORT_SYMBOL(blk_put_request);
13101296
@@ -1340,8 +1326,8 @@ void blk_add_request_payload(struct request *rq, struct page *page, · 29 unmodified lines
13401326 }
13411327 EXPORT_SYMBOL_GPL(blk_add_request_payload);
13421328
1343−static bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1344− struct bio *bio)
1329+bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1330+ struct bio *bio)
13451331 {
13461332 const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
13471333
@@ -1358,12 +1344,12 @@ static bool bio_attempt_back_merge(struct request_queue *q, struct request *req, · 10 unmodified lines
13581344 req->__data_len += bio->bi_size;
13591345 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
13601346
1361− drive_stat_acct(req, 0);
1347+ blk_account_io_start(req, false);
13621348 return true;
13631349 }
13641350
1365−static bool bio_attempt_front_merge(struct request_queue *q,
1366− struct request *req, struct bio *bio)
1351+bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
1352+ struct bio *bio)
13671353 {
13681354 const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
13691355
@@ -1388,12 +1374,12 @@ static bool bio_attempt_front_merge(struct request_queue *q, · 18 unmodified lines
13881374 req->__data_len += bio->bi_size;
13891375 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
13901376
1391− drive_stat_acct(req, 0);
1377+ blk_account_io_start(req, false);
13921378 return true;
13931379 }
13941380
13951381 /**
1396− * attempt_plug_merge - try to merge with %current's plugged list
1382+ * blk_attempt_plug_merge - try to merge with %current's plugged list
13971383 * @q: request_queue new bio is being queued at
13981384 * @bio: new bio being queued
13991385 * @request_count: out parameter for number of traversed plugged requests
@@ -1409,8 +1395,8 @@ static bool bio_attempt_front_merge(struct request_queue *q, · 9 unmodified lines
14091395 * reliable access to the elevator outside queue lock. Only check basic
14101396 * merging parameters without querying the elevator.
14111397 */
1412−static bool attempt_plug_merge(struct request_queue *q, struct bio *bio,
1413− unsigned int *request_count)
1398+bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1399+ unsigned int *request_count)
14141400 {
14151401 struct blk_plug *plug;
14161402 struct request *rq;
@@ -1489,7 +1475,7 @@ void blk_queue_bio(struct request_queue *q, struct bio *bio) · 72 unmodified lines
14891475 * Check if we can merge with the plugged list before grabbing
14901476 * any locks.
14911477 */
1492− if (attempt_plug_merge(q, bio, &request_count))
1478+ if (blk_attempt_plug_merge(q, bio, &request_count))
14931479 return;
14941480
14951481 spin_lock_irq(q->queue_lock);
@@ -1557,7 +1543,7 @@ get_rq: · 61 unmodified lines
15571543 }
15581544 }
15591545 list_add_tail(&req->queuelist, &plug->list);
1560− drive_stat_acct(req, 1);
1546+ blk_account_io_start(req, true);
15611547 } else {
15621548 spin_lock_irq(q->queue_lock);
15631549 add_acct_request(q, req, where);
@@ -2011,7 +1997,7 @@ unsigned int blk_rq_err_bytes(const struct request *rq) · 447 unmodified lines
20111997 }
20121998 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
20131999
2014−static void blk_account_io_completion(struct request *req, unsigned int bytes)
2000+void blk_account_io_completion(struct request *req, unsigned int bytes)
20152001 {
20162002 if (blk_do_io_stat(req)) {
20172003 const int rw = rq_data_dir(req);
@@ -2025,7 +2011,7 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes) · 7 unmodified lines
20252011 }
20262012 }
20272013
2028−static void blk_account_io_done(struct request *req)
2014+void blk_account_io_done(struct request *req)
20292015 {
20302016 /*
20312017 * Account IO completion. flush_rq isn't accounted as a
@@ -2073,6 +2059,42 @@ static inline struct request *blk_pm_peek_request(struct request_queue *q, · 41 unmodified lines
20732059 }
20742060 #endif
20752061
2062+void blk_account_io_start(struct request *rq, bool new_io)
2063+{
2064+ struct hd_struct *part;
2065+ int rw = rq_data_dir(rq);
2066+ int cpu;
2067+
2068+ if (!blk_do_io_stat(rq))
2069+ return;
2070+
2071+ cpu = part_stat_lock();
2072+
2073+ if (!new_io) {
2074+ part = rq->part;
2075+ part_stat_inc(cpu, part, merges[rw]);
2076+ } else {
2077+ part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
2078+ if (!hd_struct_try_get(part)) {
2079+ /*
2080+ * The partition is already being removed,
2081+ * the request will be accounted on the disk only
2082+ *
2083+ * We take a reference on disk->part0 although that
2084+ * partition will never be deleted, so we can treat
2085+ * it as any other partition.
2086+ */
2087+ part = &rq->rq_disk->part0;
2088+ hd_struct_get(part);
2089+ }
2090+ part_round_stats(cpu, part);
2091+ part_inc_in_flight(part, rw);
2092+ rq->part = part;
2093+ }
2094+
2095+ part_stat_unlock();
2096+}
2097+
20762098 /**
20772099 * blk_peek_request - peek at the top of a request queue
20782100 * @q: request queue to peek at
@@ -2448,7 +2470,6 @@ static void blk_finish_request(struct request *req, int error) · 369 unmodified lines
24482470 if (req->cmd_flags & REQ_DONTPREP)
24492471 blk_unprep_request(req);
24502472
2451−
24522473 blk_account_io_done(req);
24532474
24542475 if (req->end_io)
@@ -2870,6 +2891,7 @@ void blk_start_plug(struct blk_plug *plug) · 415 unmodified lines
28702891
28712892 plug->magic = PLUG_MAGIC;
28722893 INIT_LIST_HEAD(&plug->list);
2894+ INIT_LIST_HEAD(&plug->mq_list);
28732895 INIT_LIST_HEAD(&plug->cb_list);
28742896
28752897 /*
@@ -2967,6 +2989,10 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) · 91 unmodified lines
29672989 BUG_ON(plug->magic != PLUG_MAGIC);
29682990
29692991 flush_plug_callbacks(plug, from_schedule);
2992+
2993+ if (!list_empty(&plug->mq_list))
2994+ blk_mq_flush_plug_list(plug, from_schedule);
2995+
29702996 if (list_empty(&plug->list))
29712997 return;
29722998
block/blk-exec.c+7−0
@@ -5,6 +5,7 @@ · 4 unmodified lines
55 #include <linux/module.h>
66 #include <linux/bio.h>
77 #include <linux/blkdev.h>
8+#include <linux/blk-mq.h>
89 #include <linux/sched/sysctl.h>
910
1011 #include "blk.h"
@@ -58,6 +59,12 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, · 47 unmodified lines
5859
5960 rq->rq_disk = bd_disk;
6061 rq->end_io = done;
62+
63+ if (q->mq_ops) {
64+ blk_mq_insert_request(q, rq, true);
65+ return;
66+ }
67+
6168 /*
6269 * need to check this before __blk_run_queue(), because rq can
6370 * be freed before that returns.
block/blk-flush.c+139−15
@@ -69,8 +69,10 @@ · 68 unmodified lines
6969 #include <linux/bio.h>
7070 #include <linux/blkdev.h>
7171 #include <linux/gfp.h>
72+#include <linux/blk-mq.h>
7273
7374 #include "blk.h"
75+#include "blk-mq.h"
7476
7577 /* FLUSH/FUA sequences */
7678 enum {
@@ -124,6 +126,24 @@ static void blk_flush_restore_request(struct request *rq) · 47 unmodified lines
124126 /* make @rq a normal request */
125127 rq->cmd_flags &= ~REQ_FLUSH_SEQ;
126128 rq->end_io = rq->flush.saved_end_io;
129+
130+ blk_clear_rq_complete(rq);
131+}
132+
133+static void mq_flush_data_run(struct work_struct *work)
134+{
135+ struct request *rq;
136+
137+ rq = container_of(work, struct request, mq_flush_data);
138+
139+ memset(&rq->csd, 0, sizeof(rq->csd));
140+ blk_mq_run_request(rq, true, false);
141+}
142+
143+static void blk_mq_flush_data_insert(struct request *rq)
144+{
145+ INIT_WORK(&rq->mq_flush_data, mq_flush_data_run);
146+ kblockd_schedule_work(rq->q, &rq->mq_flush_data);
127147 }
128148
129149 /**
@@ -136,7 +156,7 @@ static void blk_flush_restore_request(struct request *rq) · 6 unmodified lines
136156 * completion and trigger the next step.
137157 *
138158 * CONTEXT:
139− * spin_lock_irq(q->queue_lock)
159+ * spin_lock_irq(q->queue_lock or q->mq_flush_lock)
140160 *
141161 * RETURNS:
142162 * %true if requests were added to the dispatch queue, %false otherwise.
@@ -146,7 +166,7 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, · 3 unmodified lines
146166 {
147167 struct request_queue *q = rq->q;
148168 struct list_head *pending = &q->flush_queue[q->flush_pending_idx];
149− bool queued = false;
169+ bool queued = false, kicked;
150170
151171 BUG_ON(rq->flush.seq & seq);
152172 rq->flush.seq |= seq;
@@ -167,8 +187,12 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, · 14 unmodified lines
167187
168188 case REQ_FSEQ_DATA:
169189 list_move_tail(&rq->flush.list, &q->flush_data_in_flight);
170− list_add(&rq->queuelist, &q->queue_head);
171− queued = true;
190+ if (q->mq_ops)
191+ blk_mq_flush_data_insert(rq);
192+ else {
193+ list_add(&rq->queuelist, &q->queue_head);
194+ queued = true;
195+ }
172196 break;
173197
174198 case REQ_FSEQ_DONE:
@@ -181,28 +205,43 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, · 6 unmodified lines
181205 BUG_ON(!list_empty(&rq->queuelist));
182206 list_del_init(&rq->flush.list);
183207 blk_flush_restore_request(rq);
184− __blk_end_request_all(rq, error);
208+ if (q->mq_ops)
209+ blk_mq_end_io(rq, error);
210+ else
211+ __blk_end_request_all(rq, error);
185212 break;
186213
187214 default:
188215 BUG();
189216 }
190217
191− return blk_kick_flush(q) | queued;
218+ kicked = blk_kick_flush(q);
219+ /* blk_mq_run_flush will run queue */
220+ if (q->mq_ops)
221+ return queued;
222+ return kicked | queued;
192223 }
193224
194225 static void flush_end_io(struct request *flush_rq, int error)
195226 {
196227 struct request_queue *q = flush_rq->q;
197− struct list_head *running = &q->flush_queue[q->flush_running_idx];
228+ struct list_head *running;
198229 bool queued = false;
199230 struct request *rq, *n;
231+ unsigned long flags = 0;
200232
233+ if (q->mq_ops) {
234+ blk_mq_free_request(flush_rq);
235+ spin_lock_irqsave(&q->mq_flush_lock, flags);
236+ }
237+ running = &q->flush_queue[q->flush_running_idx];
201238 BUG_ON(q->flush_pending_idx == q->flush_running_idx);
202239
203240 /* account completion of the flush request */
204241 q->flush_running_idx ^= 1;
205− elv_completed_request(q, flush_rq);
242+
243+ if (!q->mq_ops)
244+ elv_completed_request(q, flush_rq);
206245
207246 /* and push the waiting requests to the next stage */
208247 list_for_each_entry_safe(rq, n, running, flush.list) {
@@ -223,9 +262,48 @@ static void flush_end_io(struct request *flush_rq, int error) · 14 unmodified lines
223262 * directly into request_fn may confuse the driver. Always use
224263 * kblockd.
225264 */
226− if (queued || q->flush_queue_delayed)
227− blk_run_queue_async(q);
265+ if (queued || q->flush_queue_delayed) {
266+ if (!q->mq_ops)
267+ blk_run_queue_async(q);
268+ else
269+ /*
270+ * This can be optimized to only run queues with requests
271+ * queued if necessary.
272+ */
273+ blk_mq_run_queues(q, true);
274+ }
228275 q->flush_queue_delayed = 0;
276+ if (q->mq_ops)
277+ spin_unlock_irqrestore(&q->mq_flush_lock, flags);
278+}
279+
280+static void mq_flush_work(struct work_struct *work)
281+{
282+ struct request_queue *q;
283+ struct request *rq;
284+
285+ q = container_of(work, struct request_queue, mq_flush_work);
286+
287+ /* We don't need set REQ_FLUSH_SEQ, it's for consistency */
288+ rq = blk_mq_alloc_request(q, WRITE_FLUSH|REQ_FLUSH_SEQ,
289+ __GFP_WAIT|GFP_ATOMIC);
290+ rq->cmd_type = REQ_TYPE_FS;
291+ rq->end_io = flush_end_io;
292+
293+ blk_mq_run_request(rq, true, false);
294+}
295+
296+/*
297+ * We can't directly use q->flush_rq, because it doesn't have tag and is not in
298+ * hctx->rqs[]. so we must allocate a new request, since we can't sleep here,
299+ * so offload the work to workqueue.
300+ *
301+ * Note: we assume a flush request finished in any hardware queue will flush
302+ * the whole disk cache.
303+ */
304+static void mq_run_flush(struct request_queue *q)
305+{
306+ kblockd_schedule_work(q, &q->mq_flush_work);
229307 }
230308
231309 /**
@@ -236,7 +314,7 @@ static void flush_end_io(struct request *flush_rq, int error) · 4 unmodified lines
236314 * Please read the comment at the top of this file for more info.
237315 *
238316 * CONTEXT:
239− * spin_lock_irq(q->queue_lock)
317+ * spin_lock_irq(q->queue_lock or q->mq_flush_lock)
240318 *
241319 * RETURNS:
242320 * %true if flush was issued, %false otherwise.
@@ -261,13 +339,18 @@ static bool blk_kick_flush(struct request_queue *q) · 18 unmodified lines
261339 * Issue flush and toggle pending_idx. This makes pending_idx
262340 * different from running_idx, which means flush is in flight.
263341 */
342+ q->flush_pending_idx ^= 1;
343+ if (q->mq_ops) {
344+ mq_run_flush(q);
345+ return true;
346+ }
347+
264348 blk_rq_init(q, &q->flush_rq);
265349 q->flush_rq.cmd_type = REQ_TYPE_FS;
266350 q->flush_rq.cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ;
267351 q->flush_rq.rq_disk = first_rq->rq_disk;
268352 q->flush_rq.end_io = flush_end_io;
269353
270− q->flush_pending_idx ^= 1;
271354 list_add_tail(&q->flush_rq.queuelist, &q->queue_head);
272355 return true;
273356 }
@@ -284,16 +367,37 @@ static void flush_data_end_io(struct request *rq, int error) · 10 unmodified lines
284367 blk_run_queue_async(q);
285368 }
286369
370+static void mq_flush_data_end_io(struct request *rq, int error)
371+{
372+ struct request_queue *q = rq->q;
373+ struct blk_mq_hw_ctx *hctx;
374+ struct blk_mq_ctx *ctx;
375+ unsigned long flags;
376+
377+ ctx = rq->mq_ctx;
378+ hctx = q->mq_ops->map_queue(q, ctx->cpu);
379+
380+ /*
381+ * After populating an empty queue, kick it to avoid stall. Read
382+ * the comment in flush_end_io().
383+ */
384+ spin_lock_irqsave(&q->mq_flush_lock, flags);
385+ if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error))
386+ blk_mq_run_hw_queue(hctx, true);
387+ spin_unlock_irqrestore(&q->mq_flush_lock, flags);
388+}
389+
287390 /**
288391 * blk_insert_flush - insert a new FLUSH/FUA request
289392 * @rq: request to insert
290393 *
291394 * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions.
395+ * or __blk_mq_run_hw_queue() to dispatch request.
292396 * @rq is being submitted. Analyze what needs to be done and put it on the
293397 * right queue.
294398 *
295399 * CONTEXT:
296− * spin_lock_irq(q->queue_lock)
400+ * spin_lock_irq(q->queue_lock) in !mq case
297401 */
298402 void blk_insert_flush(struct request *rq)
299403 {
@@ -316,7 +420,10 @@ void blk_insert_flush(struct request *rq) · 16 unmodified lines
316420 * complete the request.
317421 */
318422 if (!policy) {
319− __blk_end_bidi_request(rq, 0, 0, 0);
423+ if (q->mq_ops)
424+ blk_mq_end_io(rq, 0);
425+ else
426+ __blk_end_bidi_request(rq, 0, 0, 0);
320427 return;
321428 }
322429
@@ -329,7 +436,10 @@ void blk_insert_flush(struct request *rq) · 6 unmodified lines
329436 */
330437 if ((policy & REQ_FSEQ_DATA) &&
331438 !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
332− list_add_tail(&rq->queuelist, &q->queue_head);
439+ if (q->mq_ops) {
440+ blk_mq_run_request(rq, false, true);
441+ } else
442+ list_add_tail(&rq->queuelist, &q->queue_head);
333443 return;
334444 }
335445
@@ -341,6 +451,14 @@ void blk_insert_flush(struct request *rq) · 5 unmodified lines
341451 INIT_LIST_HEAD(&rq->flush.list);
342452 rq->cmd_flags |= REQ_FLUSH_SEQ;
343453 rq->flush.saved_end_io = rq->end_io; /* Usually NULL */
454+ if (q->mq_ops) {
455+ rq->end_io = mq_flush_data_end_io;
456+
457+ spin_lock_irq(&q->mq_flush_lock);
458+ blk_flush_complete_seq(rq, REQ_FSEQ_ACTIONS & ~policy, 0);
459+ spin_unlock_irq(&q->mq_flush_lock);
460+ return;
461+ }
344462 rq->end_io = flush_data_end_io;
345463
346464 blk_flush_complete_seq(rq, REQ_FSEQ_ACTIONS & ~policy, 0);
@@ -453,3 +571,9 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, · 106 unmodified lines
453571 return ret;
454572 }
455573 EXPORT_SYMBOL(blkdev_issue_flush);
574+
575+void blk_mq_init_flush(struct request_queue *q)
576+{
577+ spin_lock_init(&q->mq_flush_lock);
578+ INIT_WORK(&q->mq_flush_work, mq_flush_work);
579+}
block/blk-mq-cpu.c+93−0
@@ -0,0 +1,93 @@
1+#include <linux/kernel.h>
2+#include <linux/module.h>
3+#include <linux/init.h>
4+#include <linux/blkdev.h>
5+#include <linux/list.h>
6+#include <linux/llist.h>
7+#include <linux/smp.h>
8+#include <linux/cpu.h>
9+
10+#include <linux/blk-mq.h>
11+#include "blk-mq.h"
12+
13+static LIST_HEAD(blk_mq_cpu_notify_list);
14+static DEFINE_SPINLOCK(blk_mq_cpu_notify_lock);
15+
16+static int __cpuinit blk_mq_main_cpu_notify(struct notifier_block *self,
17+ unsigned long action, void *hcpu)
18+{
19+ unsigned int cpu = (unsigned long) hcpu;
20+ struct blk_mq_cpu_notifier *notify;
21+
22+ spin_lock(&blk_mq_cpu_notify_lock);
23+
24+ list_for_each_entry(notify, &blk_mq_cpu_notify_list, list)
25+ notify->notify(notify->data, action, cpu);
26+
27+ spin_unlock(&blk_mq_cpu_notify_lock);
28+ return NOTIFY_OK;
29+}
30+
31+static void __cpuinit blk_mq_cpu_notify(void *data, unsigned long action,
32+ unsigned int cpu)
33+{
34+ if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
35+ /*
36+ * If the CPU goes away, ensure that we run any pending
37+ * completions.
38+ */
39+ struct llist_node *node;
40+ struct request *rq;
41+
42+ local_irq_disable();
43+
44+ node = llist_del_all(&per_cpu(ipi_lists, cpu));
45+ while (node) {
46+ struct llist_node *next = node->next;
47+
48+ rq = llist_entry(node, struct request, ll_list);
49+ __blk_mq_end_io(rq, rq->errors);
50+ node = next;
51+ }
52+
53+ local_irq_enable();
54+ }
55+}
56+
57+static struct notifier_block __cpuinitdata blk_mq_main_cpu_notifier = {
58+ .notifier_call = blk_mq_main_cpu_notify,
59+};
60+
61+void blk_mq_register_cpu_notifier(struct blk_mq_cpu_notifier *notifier)
62+{
63+ BUG_ON(!notifier->notify);
64+
65+ spin_lock(&blk_mq_cpu_notify_lock);
66+ list_add_tail(&notifier->list, &blk_mq_cpu_notify_list);
67+ spin_unlock(&blk_mq_cpu_notify_lock);
68+}
69+
70+void blk_mq_unregister_cpu_notifier(struct blk_mq_cpu_notifier *notifier)
71+{
72+ spin_lock(&blk_mq_cpu_notify_lock);
73+ list_del(&notifier->list);
74+ spin_unlock(&blk_mq_cpu_notify_lock);
75+}
76+
77+void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier,
78+ void (*fn)(void *, unsigned long, unsigned int),
79+ void *data)
80+{
81+ notifier->notify = fn;
82+ notifier->data = data;
83+}
84+
85+static struct blk_mq_cpu_notifier __cpuinitdata cpu_notifier = {
86+ .notify = blk_mq_cpu_notify,
87+};
88+
89+void __init blk_mq_cpu_init(void)
90+{
91+ register_hotcpu_notifier(&blk_mq_main_cpu_notifier);
92+ blk_mq_register_cpu_notifier(&cpu_notifier);
93+}
block/blk-mq-cpumap.c+108−0
@@ -0,0 +1,108 @@
1+#include <linux/kernel.h>
2+#include <linux/threads.h>
3+#include <linux/module.h>
4+#include <linux/mm.h>
5+#include <linux/smp.h>
6+#include <linux/cpu.h>
7+
8+#include <linux/blk-mq.h>
9+#include "blk.h"
10+#include "blk-mq.h"
11+
12+static void show_map(unsigned int *map, unsigned int nr)
13+{
14+ int i;
15+
16+ pr_info("blk-mq: CPU -> queue map\n");
17+ for_each_online_cpu(i)
18+ pr_info(" CPU%2u -> Queue %u\n", i, map[i]);
19+}
20+
21+static int cpu_to_queue_index(unsigned int nr_cpus, unsigned int nr_queues,
22+ const int cpu)
23+{
24+ return cpu / ((nr_cpus + nr_queues - 1) / nr_queues);
25+}
26+
27+static int get_first_sibling(unsigned int cpu)
28+{
29+ unsigned int ret;
30+
31+ ret = cpumask_first(topology_thread_cpumask(cpu));
32+ if (ret < nr_cpu_ids)
33+ return ret;
34+
35+ return cpu;
36+}
37+
38+int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues)
39+{
40+ unsigned int i, nr_cpus, nr_uniq_cpus, queue, first_sibling;
41+ cpumask_var_t cpus;
42+
43+ if (!alloc_cpumask_var(&cpus, GFP_ATOMIC))
44+ return 1;
45+
46+ cpumask_clear(cpus);
47+ nr_cpus = nr_uniq_cpus = 0;
48+ for_each_online_cpu(i) {
49+ nr_cpus++;
50+ first_sibling = get_first_sibling(i);
51+ if (!cpumask_test_cpu(first_sibling, cpus))
52+ nr_uniq_cpus++;
53+ cpumask_set_cpu(i, cpus);
54+ }
55+
56+ queue = 0;
57+ for_each_possible_cpu(i) {
58+ if (!cpu_online(i)) {
59+ map[i] = 0;
60+ continue;
61+ }
62+
63+ /*
64+ * Easy case - we have equal or more hardware queues. Or
65+ * there are no thread siblings to take into account. Do
66+ * 1:1 if enough, or sequential mapping if less.
67+ */
68+ if (nr_queues >= nr_cpus || nr_cpus == nr_uniq_cpus) {
69+ map[i] = cpu_to_queue_index(nr_cpus, nr_queues, queue);
70+ queue++;
71+ continue;
72+ }
73+
74+ /*
75+ * Less then nr_cpus queues, and we have some number of
76+ * threads per cores. Map sibling threads to the same
77+ * queue.
78+ */
79+ first_sibling = get_first_sibling(i);
80+ if (first_sibling == i) {
81+ map[i] = cpu_to_queue_index(nr_uniq_cpus, nr_queues,
82+ queue);
83+ queue++;
84+ } else
85+ map[i] = map[first_sibling];
86+ }
87+
88+ show_map(map, nr_cpus);
89+ free_cpumask_var(cpus);
90+ return 0;
91+}
92+
93+unsigned int *blk_mq_make_queue_map(struct blk_mq_reg *reg)
94+{
95+ unsigned int *map;
96+
97+ /* If cpus are offline, map them to first hctx */
98+ map = kzalloc_node(sizeof(*map) * num_possible_cpus(), GFP_KERNEL,
99+ reg->numa_node);
100+ if (!map)
101+ return NULL;
102+
103+ if (!blk_mq_update_queue_map(map, reg->nr_hw_queues))
104+ return map;
105+
106+ kfree(map);
107+ return NULL;
108+}
block/blk-mq-sysfs.c+384−0
@@ -0,0 +1,384 @@
1+#include <linux/kernel.h>
2+#include <linux/module.h>
3+#include <linux/backing-dev.h>
4+#include <linux/bio.h>
5+#include <linux/blkdev.h>
6+#include <linux/mm.h>
7+#include <linux/init.h>
8+#include <linux/slab.h>
9+#include <linux/workqueue.h>
10+#include <linux/smp.h>
11+
12+#include <linux/blk-mq.h>
13+#include "blk-mq.h"
14+#include "blk-mq-tag.h"
15+
16+static void blk_mq_sysfs_release(struct kobject *kobj)
17+{
18+}
19+
20+struct blk_mq_ctx_sysfs_entry {
21+ struct attribute attr;
22+ ssize_t (*show)(struct blk_mq_ctx *, char *);
23+ ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
24+};
25+
26+struct blk_mq_hw_ctx_sysfs_entry {
27+ struct attribute attr;
28+ ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
29+ ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
30+};
31+
32+static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
33+ char *page)
34+{
35+ struct blk_mq_ctx_sysfs_entry *entry;
36+ struct blk_mq_ctx *ctx;
37+ struct request_queue *q;
38+ ssize_t res;
39+
40+ entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
41+ ctx = container_of(kobj, struct blk_mq_ctx, kobj);
42+ q = ctx->queue;
43+
44+ if (!entry->show)
45+ return -EIO;
46+
47+ res = -ENOENT;
48+ mutex_lock(&q->sysfs_lock);
49+ if (!blk_queue_dying(q))
50+ res = entry->show(ctx, page);
51+ mutex_unlock(&q->sysfs_lock);
52+ return res;
53+}
54+
55+static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
56+ const char *page, size_t length)
57+{
58+ struct blk_mq_ctx_sysfs_entry *entry;
59+ struct blk_mq_ctx *ctx;
60+ struct request_queue *q;
61+ ssize_t res;
62+
63+ entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
64+ ctx = container_of(kobj, struct blk_mq_ctx, kobj);
65+ q = ctx->queue;
66+
67+ if (!entry->store)
68+ return -EIO;
69+
70+ res = -ENOENT;
71+ mutex_lock(&q->sysfs_lock);
72+ if (!blk_queue_dying(q))
73+ res = entry->store(ctx, page, length);
74+ mutex_unlock(&q->sysfs_lock);
75+ return res;
76+}
77+
78+static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
79+ struct attribute *attr, char *page)
80+{
81+ struct blk_mq_hw_ctx_sysfs_entry *entry;
82+ struct blk_mq_hw_ctx *hctx;
83+ struct request_queue *q;
84+ ssize_t res;
85+
86+ entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
87+ hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
88+ q = hctx->queue;
89+
90+ if (!entry->show)
91+ return -EIO;
92+
93+ res = -ENOENT;
94+ mutex_lock(&q->sysfs_lock);
95+ if (!blk_queue_dying(q))
96+ res = entry->show(hctx, page);
97+ mutex_unlock(&q->sysfs_lock);
98+ return res;
99+}
100+
101+static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
102+ struct attribute *attr, const char *page,
103+ size_t length)
104+{
105+ struct blk_mq_hw_ctx_sysfs_entry *entry;
106+ struct blk_mq_hw_ctx *hctx;
107+ struct request_queue *q;
108+ ssize_t res;
109+
110+ entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
111+ hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
112+ q = hctx->queue;
113+
114+ if (!entry->store)
115+ return -EIO;
116+
117+ res = -ENOENT;
118+ mutex_lock(&q->sysfs_lock);
119+ if (!blk_queue_dying(q))
120+ res = entry->store(hctx, page, length);
121+ mutex_unlock(&q->sysfs_lock);
122+ return res;
123+}
124+
125+static ssize_t blk_mq_sysfs_dispatched_show(struct blk_mq_ctx *ctx, char *page)
126+{
127+ return sprintf(page, "%lu %lu\n", ctx->rq_dispatched[1],
128+ ctx->rq_dispatched[0]);
129+}
130+
131+static ssize_t blk_mq_sysfs_merged_show(struct blk_mq_ctx *ctx, char *page)
132+{
133+ return sprintf(page, "%lu\n", ctx->rq_merged);
134+}
135+
136+static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx *ctx, char *page)
137+{
138+ return sprintf(page, "%lu %lu\n", ctx->rq_completed[1],
139+ ctx->rq_completed[0]);
140+}
141+
142+static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg)
143+{
144+ char *start_page = page;
145+ struct request *rq;
146+
147+ page += sprintf(page, "%s:\n", msg);
148+
149+ list_for_each_entry(rq, list, queuelist)
150+ page += sprintf(page, "\t%p\n", rq);
151+
152+ return page - start_page;
153+}
154+
155+static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page)
156+{
157+ ssize_t ret;
158+
159+ spin_lock(&ctx->lock);
160+ ret = sysfs_list_show(page, &ctx->rq_list, "CTX pending");
161+ spin_unlock(&ctx->lock);
162+
163+ return ret;
164+}
165+
166+static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
167+ char *page)
168+{
169+ return sprintf(page, "%lu\n", hctx->queued);
170+}
171+
172+static ssize_t blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx *hctx, char *page)
173+{
174+ return sprintf(page, "%lu\n", hctx->run);
175+}
176+
177+static ssize_t blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx *hctx,
178+ char *page)
179+{
180+ char *start_page = page;
181+ int i;
182+
183+ page += sprintf(page, "%8u\t%lu\n", 0U, hctx->dispatched[0]);
184+
185+ for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER; i++) {
186+ unsigned long d = 1U << (i - 1);
187+
188+ page += sprintf(page, "%8lu\t%lu\n", d, hctx->dispatched[i]);
189+ }
190+
191+ return page - start_page;
192+}
193+
194+static ssize_t blk_mq_hw_sysfs_rq_list_show(struct blk_mq_hw_ctx *hctx,
195+ char *page)
196+{
197+ ssize_t ret;
198+
199+ spin_lock(&hctx->lock);
200+ ret = sysfs_list_show(page, &hctx->dispatch, "HCTX pending");
201+ spin_unlock(&hctx->lock);
202+
203+ return ret;
204+}
205+
206+static ssize_t blk_mq_hw_sysfs_ipi_show(struct blk_mq_hw_ctx *hctx, char *page)
207+{
208+ ssize_t ret;
209+
210+ spin_lock(&hctx->lock);
211+ ret = sprintf(page, "%u\n", !!(hctx->flags & BLK_MQ_F_SHOULD_IPI));
212+ spin_unlock(&hctx->lock);
213+
214+ return ret;
215+}
216+
217+static ssize_t blk_mq_hw_sysfs_ipi_store(struct blk_mq_hw_ctx *hctx,
218+ const char *page, size_t len)
219+{
220+ struct blk_mq_ctx *ctx;
221+ unsigned long ret;
222+ unsigned int i;
223+
224+ if (kstrtoul(page, 10, &ret)) {
225+ pr_err("blk-mq-sysfs: invalid input '%s'\n", page);
226+ return -EINVAL;
227+ }
228+
229+ spin_lock(&hctx->lock);
230+ if (ret)
231+ hctx->flags |= BLK_MQ_F_SHOULD_IPI;
232+ else
233+ hctx->flags &= ~BLK_MQ_F_SHOULD_IPI;
234+ spin_unlock(&hctx->lock);
235+
236+ hctx_for_each_ctx(hctx, ctx, i)
237+ ctx->ipi_redirect = !!ret;
238+
239+ return len;
240+}
241+
242+static ssize_t blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx *hctx, char *page)
243+{
244+ return blk_mq_tag_sysfs_show(hctx->tags, page);
245+}
246+
247+static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
248+ .attr = {.name = "dispatched", .mode = S_IRUGO },
249+ .show = blk_mq_sysfs_dispatched_show,
250+};
251+static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged = {
252+ .attr = {.name = "merged", .mode = S_IRUGO },
253+ .show = blk_mq_sysfs_merged_show,
254+};
255+static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed = {
256+ .attr = {.name = "completed", .mode = S_IRUGO },
257+ .show = blk_mq_sysfs_completed_show,
258+};
259+static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_rq_list = {
260+ .attr = {.name = "rq_list", .mode = S_IRUGO },
261+ .show = blk_mq_sysfs_rq_list_show,
262+};
263+
264+static struct attribute *default_ctx_attrs[] = {
265+ &blk_mq_sysfs_dispatched.attr,
266+ &blk_mq_sysfs_merged.attr,
267+ &blk_mq_sysfs_completed.attr,
268+ &blk_mq_sysfs_rq_list.attr,
269+ NULL,
270+};
271+
272+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued = {
273+ .attr = {.name = "queued", .mode = S_IRUGO },
274+ .show = blk_mq_hw_sysfs_queued_show,
275+};
276+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run = {
277+ .attr = {.name = "run", .mode = S_IRUGO },
278+ .show = blk_mq_hw_sysfs_run_show,
279+};
280+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched = {
281+ .attr = {.name = "dispatched", .mode = S_IRUGO },
282+ .show = blk_mq_hw_sysfs_dispatched_show,
283+};
284+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_pending = {
285+ .attr = {.name = "pending", .mode = S_IRUGO },
286+ .show = blk_mq_hw_sysfs_rq_list_show,
287+};
288+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_ipi = {
289+ .attr = {.name = "ipi_redirect", .mode = S_IRUGO | S_IWUSR},
290+ .show = blk_mq_hw_sysfs_ipi_show,
291+ .store = blk_mq_hw_sysfs_ipi_store,
292+};
293+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags = {
294+ .attr = {.name = "tags", .mode = S_IRUGO },
295+ .show = blk_mq_hw_sysfs_tags_show,
296+};
297+
298+static struct attribute *default_hw_ctx_attrs[] = {
299+ &blk_mq_hw_sysfs_queued.attr,
300+ &blk_mq_hw_sysfs_run.attr,
301+ &blk_mq_hw_sysfs_dispatched.attr,
302+ &blk_mq_hw_sysfs_pending.attr,
303+ &blk_mq_hw_sysfs_ipi.attr,
304+ &blk_mq_hw_sysfs_tags.attr,
305+ NULL,
306+};
307+
308+static const struct sysfs_ops blk_mq_sysfs_ops = {
309+ .show = blk_mq_sysfs_show,
310+ .store = blk_mq_sysfs_store,
311+};
312+
313+static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
314+ .show = blk_mq_hw_sysfs_show,
315+ .store = blk_mq_hw_sysfs_store,
316+};
317+
318+static struct kobj_type blk_mq_ktype = {
319+ .sysfs_ops = &blk_mq_sysfs_ops,
320+ .release = blk_mq_sysfs_release,
321+};
322+
323+static struct kobj_type blk_mq_ctx_ktype = {
324+ .sysfs_ops = &blk_mq_sysfs_ops,
325+ .default_attrs = default_ctx_attrs,
326+ .release = blk_mq_sysfs_release,
327+};
328+
329+static struct kobj_type blk_mq_hw_ktype = {
330+ .sysfs_ops = &blk_mq_hw_sysfs_ops,
331+ .default_attrs = default_hw_ctx_attrs,
332+ .release = blk_mq_sysfs_release,
333+};
334+
335+void blk_mq_unregister_disk(struct gendisk *disk)
336+{
337+ struct request_queue *q = disk->queue;
338+
339+ kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
340+ kobject_del(&q->mq_kobj);
341+
342+ kobject_put(&disk_to_dev(disk)->kobj);
343+}
344+
345+int blk_mq_register_disk(struct gendisk *disk)
346+{
347+ struct device *dev = disk_to_dev(disk);
348+ struct request_queue *q = disk->queue;
349+ struct blk_mq_hw_ctx *hctx;
350+ struct blk_mq_ctx *ctx;
351+ int ret, i, j;
352+
353+ kobject_init(&q->mq_kobj, &blk_mq_ktype);
354+
355+ ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
356+ if (ret < 0)
357+ return ret;
358+
359+ kobject_uevent(&q->mq_kobj, KOBJ_ADD);
360+
361+ queue_for_each_hw_ctx(q, hctx, i) {
362+ kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
363+ ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", i);
364+ if (ret)
365+ break;
366+
367+ if (!hctx->nr_ctx)
368+ continue;
369+
370+ hctx_for_each_ctx(hctx, ctx, j) {
371+ kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
372+ ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
373+ if (ret)
374+ break;
375+ }
376+ }
377+
378+ if (ret) {
379+ blk_mq_unregister_disk(disk);
380+ return ret;
381+ }
382+
383+ return 0;
384+}
block/blk-mq-tag.c+204−0
@@ -0,0 +1,204 @@
1+#include <linux/kernel.h>
2+#include <linux/module.h>
3+#include <linux/percpu_ida.h>
4+
5+#include <linux/blk-mq.h>
6+#include "blk.h"
7+#include "blk-mq.h"
8+#include "blk-mq-tag.h"
9+
10+/*
11+ * Per tagged queue (tag address space) map
12+ */
13+struct blk_mq_tags {
14+ unsigned int nr_tags;
15+ unsigned int nr_reserved_tags;
16+ unsigned int nr_batch_move;
17+ unsigned int nr_max_cache;
18+
19+ struct percpu_ida free_tags;
20+ struct percpu_ida reserved_tags;
21+};
22+
23+void blk_mq_wait_for_tags(struct blk_mq_tags *tags)
24+{
25+ int tag = blk_mq_get_tag(tags, __GFP_WAIT, false);
26+ blk_mq_put_tag(tags, tag);
27+}
28+
29+bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
30+{
31+ return !tags ||
32+ percpu_ida_free_tags(&tags->free_tags, nr_cpu_ids) != 0;
33+}
34+
35+static unsigned int __blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp)
36+{
37+ int tag;
38+
39+ tag = percpu_ida_alloc(&tags->free_tags, gfp);
40+ if (tag < 0)
41+ return BLK_MQ_TAG_FAIL;
42+ return tag + tags->nr_reserved_tags;
43+}
44+
45+static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_tags *tags,
46+ gfp_t gfp)
47+{
48+ int tag;
49+
50+ if (unlikely(!tags->nr_reserved_tags)) {
51+ WARN_ON_ONCE(1);
52+ return BLK_MQ_TAG_FAIL;
53+ }
54+
55+ tag = percpu_ida_alloc(&tags->reserved_tags, gfp);
56+ if (tag < 0)
57+ return BLK_MQ_TAG_FAIL;
58+ return tag;
59+}
60+
61+unsigned int blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp, bool reserved)
62+{
63+ if (!reserved)
64+ return __blk_mq_get_tag(tags, gfp);
65+
66+ return __blk_mq_get_reserved_tag(tags, gfp);
67+}
68+
69+static void __blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag)
70+{
71+ BUG_ON(tag >= tags->nr_tags);
72+
73+ percpu_ida_free(&tags->free_tags, tag - tags->nr_reserved_tags);
74+}
75+
76+static void __blk_mq_put_reserved_tag(struct blk_mq_tags *tags,
77+ unsigned int tag)
78+{
79+ BUG_ON(tag >= tags->nr_reserved_tags);
80+
81+ percpu_ida_free(&tags->reserved_tags, tag);
82+}
83+
84+void blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag)
85+{
86+ if (tag >= tags->nr_reserved_tags)
87+ __blk_mq_put_tag(tags, tag);
88+ else
89+ __blk_mq_put_reserved_tag(tags, tag);
90+}
91+
92+static int __blk_mq_tag_iter(unsigned id, void *data)
93+{
94+ unsigned long *tag_map = data;
95+ __set_bit(id, tag_map);
96+ return 0;
97+}
98+
99+void blk_mq_tag_busy_iter(struct blk_mq_tags *tags,
100+ void (*fn)(void *, unsigned long *), void *data)
101+{
102+ unsigned long *tag_map;
103+ size_t map_size;
104+
105+ map_size = ALIGN(tags->nr_tags, BITS_PER_LONG) / BITS_PER_LONG;
106+ tag_map = kzalloc(map_size * sizeof(unsigned long), GFP_ATOMIC);
107+ if (!tag_map)
108+ return;
109+
110+ percpu_ida_for_each_free(&tags->free_tags, __blk_mq_tag_iter, tag_map);
111+ if (tags->nr_reserved_tags)
112+ percpu_ida_for_each_free(&tags->reserved_tags, __blk_mq_tag_iter,
113+ tag_map);
114+
115+ fn(data, tag_map);
116+ kfree(tag_map);
117+}
118+
119+struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
120+ unsigned int reserved_tags, int node)
121+{
122+ unsigned int nr_tags, nr_cache;
123+ struct blk_mq_tags *tags;
124+ int ret;
125+
126+ if (total_tags > BLK_MQ_TAG_MAX) {
127+ pr_err("blk-mq: tag depth too large\n");
128+ return NULL;
129+ }
130+
131+ tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
132+ if (!tags)
133+ return NULL;
134+
135+ nr_tags = total_tags - reserved_tags;
136+ nr_cache = nr_tags / num_possible_cpus();
137+
138+ if (nr_cache < BLK_MQ_TAG_CACHE_MIN)
139+ nr_cache = BLK_MQ_TAG_CACHE_MIN;
140+ else if (nr_cache > BLK_MQ_TAG_CACHE_MAX)
141+ nr_cache = BLK_MQ_TAG_CACHE_MAX;
142+
143+ tags->nr_tags = total_tags;
144+ tags->nr_reserved_tags = reserved_tags;
145+ tags->nr_max_cache = nr_cache;
146+ tags->nr_batch_move = max(1u, nr_cache / 2);
147+
148+ ret = __percpu_ida_init(&tags->free_tags, tags->nr_tags -
149+ tags->nr_reserved_tags,
150+ tags->nr_max_cache,
151+ tags->nr_batch_move);
152+ if (ret)
153+ goto err_free_tags;
154+
155+ if (reserved_tags) {
156+ /*
157+ * With max_cahe and batch set to 1, the allocator fallbacks to
158+ * no cached. It's fine reserved tags allocation is slow.
159+ */
160+ ret = __percpu_ida_init(&tags->reserved_tags, reserved_tags,
161+ 1, 1);
162+ if (ret)
163+ goto err_reserved_tags;
164+ }
165+
166+ return tags;
167+
168+err_reserved_tags:
169+ percpu_ida_destroy(&tags->free_tags);
170+err_free_tags:
171+ kfree(tags);
172+ return NULL;
173+}
174+
175+void blk_mq_free_tags(struct blk_mq_tags *tags)
176+{
177+ percpu_ida_destroy(&tags->free_tags);
178+ percpu_ida_destroy(&tags->reserved_tags);
179+ kfree(tags);
180+}
181+
182+ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page)
183+{
184+ char *orig_page = page;
185+ int cpu;
186+
187+ if (!tags)
188+ return 0;
189+
190+ page += sprintf(page, "nr_tags=%u, reserved_tags=%u, batch_move=%u,"
191+ " max_cache=%u\n", tags->nr_tags, tags->nr_reserved_tags,
192+ tags->nr_batch_move, tags->nr_max_cache);
193+
194+ page += sprintf(page, "nr_free=%u, nr_reserved=%u\n",
195+ percpu_ida_free_tags(&tags->free_tags, nr_cpu_ids),
196+ percpu_ida_free_tags(&tags->reserved_tags, nr_cpu_ids));
197+
198+ for_each_possible_cpu(cpu) {
199+ page += sprintf(page, " cpu%02u: nr_free=%u\n", cpu,
200+ percpu_ida_free_tags(&tags->free_tags, cpu));
201+ }
202+
203+ return page - orig_page;
204+}
block/blk-mq-tag.h+27−0
@@ -0,0 +1,27 @@
1+#ifndef INT_BLK_MQ_TAG_H
2+#define INT_BLK_MQ_TAG_H
3+
4+struct blk_mq_tags;
5+
6+extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node);
7+extern void blk_mq_free_tags(struct blk_mq_tags *tags);
8+
9+extern unsigned int blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp, bool reserved);
10+extern void blk_mq_wait_for_tags(struct blk_mq_tags *tags);
11+extern void blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag);
12+extern void blk_mq_tag_busy_iter(struct blk_mq_tags *tags, void (*fn)(void *data, unsigned long *), void *data);
13+extern bool blk_mq_has_free_tags(struct blk_mq_tags *tags);
14+extern ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page);
15+
16+enum {
17+ BLK_MQ_TAG_CACHE_MIN = 1,
18+ BLK_MQ_TAG_CACHE_MAX = 64,
19+};
20+
21+enum {
22+ BLK_MQ_TAG_FAIL = -1U,
23+ BLK_MQ_TAG_MIN = BLK_MQ_TAG_CACHE_MIN,
24+ BLK_MQ_TAG_MAX = BLK_MQ_TAG_FAIL - 1,
25+};
26+
27+#endif
block/blk-mq.c+1480−0
@@ -0,0 +1,1480 @@
1+#include <linux/kernel.h>
2+#include <linux/module.h>
3+#include <linux/backing-dev.h>
4+#include <linux/bio.h>
5+#include <linux/blkdev.h>
6+#include <linux/mm.h>
7+#include <linux/init.h>
8+#include <linux/slab.h>
9+#include <linux/workqueue.h>
10+#include <linux/smp.h>
11+#include <linux/llist.h>
12+#include <linux/list_sort.h>
13+#include <linux/cpu.h>
14+#include <linux/cache.h>
15+#include <linux/sched/sysctl.h>
16+#include <linux/delay.h>
17+
18+#include <trace/events/block.h>
19+
20+#include <linux/blk-mq.h>
21+#include "blk.h"
22+#include "blk-mq.h"
23+#include "blk-mq-tag.h"
24+
25+static DEFINE_MUTEX(all_q_mutex);
26+static LIST_HEAD(all_q_list);
27+
28+static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
29+
30+DEFINE_PER_CPU(struct llist_head, ipi_lists);
31+
32+static struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
33+ unsigned int cpu)
34+{
35+ return per_cpu_ptr(q->queue_ctx, cpu);
36+}
37+
38+/*
39+ * This assumes per-cpu software queueing queues. They could be per-node
40+ * as well, for instance. For now this is hardcoded as-is. Note that we don't
41+ * care about preemption, since we know the ctx's are persistent. This does
42+ * mean that we can't rely on ctx always matching the currently running CPU.
43+ */
44+static struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
45+{
46+ return __blk_mq_get_ctx(q, get_cpu());
47+}
48+
49+static void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
50+{
51+ put_cpu();
52+}
53+
54+/*
55+ * Check if any of the ctx's have pending work in this hardware queue
56+ */
57+static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
58+{
59+ unsigned int i;
60+
61+ for (i = 0; i < hctx->nr_ctx_map; i++)
62+ if (hctx->ctx_map[i])
63+ return true;
64+
65+ return false;
66+}
67+
68+/*
69+ * Mark this ctx as having pending work in this hardware queue
70+ */
71+static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
72+ struct blk_mq_ctx *ctx)
73+{
74+ if (!test_bit(ctx->index_hw, hctx->ctx_map))
75+ set_bit(ctx->index_hw, hctx->ctx_map);
76+}
77+
78+static struct request *blk_mq_alloc_rq(struct blk_mq_hw_ctx *hctx, gfp_t gfp,
79+ bool reserved)
80+{
81+ struct request *rq;
82+ unsigned int tag;
83+
84+ tag = blk_mq_get_tag(hctx->tags, gfp, reserved);
85+ if (tag != BLK_MQ_TAG_FAIL) {
86+ rq = hctx->rqs[tag];
87+ rq->tag = tag;
88+
89+ return rq;
90+ }
91+
92+ return NULL;
93+}
94+
95+static int blk_mq_queue_enter(struct request_queue *q)
96+{
97+ int ret;
98+
99+ __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
100+ smp_wmb();
101+ /* we have problems to freeze the queue if it's initializing */
102+ if (!blk_queue_bypass(q) || !blk_queue_init_done(q))
103+ return 0;
104+
105+ __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
106+
107+ spin_lock_irq(q->queue_lock);
108+ ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
109+ !blk_queue_bypass(q), *q->queue_lock);
110+ /* inc usage with lock hold to avoid freeze_queue runs here */
111+ if (!ret)
112+ __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
113+ spin_unlock_irq(q->queue_lock);
114+
115+ return ret;
116+}
117+
118+static void blk_mq_queue_exit(struct request_queue *q)
119+{
120+ __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
121+}
122+
123+/*
124+ * Guarantee no request is in use, so we can change any data structure of
125+ * the queue afterward.
126+ */
127+static void blk_mq_freeze_queue(struct request_queue *q)
128+{
129+ bool drain;
130+
131+ spin_lock_irq(q->queue_lock);
132+ drain = !q->bypass_depth++;
133+ queue_flag_set(QUEUE_FLAG_BYPASS, q);
134+ spin_unlock_irq(q->queue_lock);
135+
136+ if (!drain)
137+ return;
138+
139+ while (true) {
140+ s64 count;
141+
142+ spin_lock_irq(q->queue_lock);
143+ count = percpu_counter_sum(&q->mq_usage_counter);
144+ spin_unlock_irq(q->queue_lock);
145+
146+ if (count == 0)
147+ break;
148+ blk_mq_run_queues(q, false);
149+ msleep(10);
150+ }
151+}
152+
153+static void blk_mq_unfreeze_queue(struct request_queue *q)
154+{
155+ bool wake = false;
156+
157+ spin_lock_irq(q->queue_lock);
158+ if (!--q->bypass_depth) {
159+ queue_flag_clear(QUEUE_FLAG_BYPASS, q);
160+ wake = true;
161+ }
162+ WARN_ON_ONCE(q->bypass_depth < 0);
163+ spin_unlock_irq(q->queue_lock);
164+ if (wake)
165+ wake_up_all(&q->mq_freeze_wq);
166+}
167+
168+bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
169+{
170+ return blk_mq_has_free_tags(hctx->tags);
171+}
172+EXPORT_SYMBOL(blk_mq_can_queue);
173+
174+static void blk_mq_rq_ctx_init(struct blk_mq_ctx *ctx, struct request *rq,
175+ unsigned int rw_flags)
176+{
177+ rq->mq_ctx = ctx;
178+ rq->cmd_flags = rw_flags;
179+ ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
180+}
181+
182+static struct request *__blk_mq_alloc_request(struct blk_mq_hw_ctx *hctx,
183+ gfp_t gfp, bool reserved)
184+{
185+ return blk_mq_alloc_rq(hctx, gfp, reserved);
186+}
187+
188+static struct request *blk_mq_alloc_request_pinned(struct request_queue *q,
189+ int rw, gfp_t gfp,
190+ bool reserved)
191+{
192+ struct request *rq;
193+
194+ do {
195+ struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
196+ struct blk_mq_hw_ctx *hctx = q->mq_ops->map_queue(q, ctx->cpu);
197+
198+ rq = __blk_mq_alloc_request(hctx, gfp & ~__GFP_WAIT, reserved);
199+ if (rq) {
200+ blk_mq_rq_ctx_init(ctx, rq, rw);
201+ break;
202+ } else if (!(gfp & __GFP_WAIT))
203+ break;
204+
205+ blk_mq_put_ctx(ctx);
206+ __blk_mq_run_hw_queue(hctx);
207+ blk_mq_wait_for_tags(hctx->tags);
208+ } while (1);
209+
210+ return rq;
211+}
212+
213+struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp)
214+{
215+ struct request *rq;
216+
217+ if (blk_mq_queue_enter(q))
218+ return NULL;
219+
220+ rq = blk_mq_alloc_request_pinned(q, rw, gfp, false);
221+ blk_mq_put_ctx(rq->mq_ctx);
222+ return rq;
223+}
224+
225+struct request *blk_mq_alloc_reserved_request(struct request_queue *q, int rw,
226+ gfp_t gfp)
227+{
228+ struct request *rq;
229+
230+ if (blk_mq_queue_enter(q))
231+ return NULL;
232+
233+ rq = blk_mq_alloc_request_pinned(q, rw, gfp, true);
234+ blk_mq_put_ctx(rq->mq_ctx);
235+ return rq;
236+}
237+EXPORT_SYMBOL(blk_mq_alloc_reserved_request);
238+
239+/*
240+ * Re-init and set pdu, if we have it
241+ */
242+static void blk_mq_rq_init(struct blk_mq_hw_ctx *hctx, struct request *rq)
243+{
244+ blk_rq_init(hctx->queue, rq);
245+
246+ if (hctx->cmd_size)
247+ rq->special = blk_mq_rq_to_pdu(rq);
248+}
249+
250+static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
251+ struct blk_mq_ctx *ctx, struct request *rq)
252+{
253+ const int tag = rq->tag;
254+ struct request_queue *q = rq->q;
255+
256+ blk_mq_rq_init(hctx, rq);
257+ blk_mq_put_tag(hctx->tags, tag);
258+
259+ blk_mq_queue_exit(q);
260+}
261+
262+void blk_mq_free_request(struct request *rq)
263+{
264+ struct blk_mq_ctx *ctx = rq->mq_ctx;
265+ struct blk_mq_hw_ctx *hctx;
266+ struct request_queue *q = rq->q;
267+
268+ ctx->rq_completed[rq_is_sync(rq)]++;
269+
270+ hctx = q->mq_ops->map_queue(q, ctx->cpu);
271+ __blk_mq_free_request(hctx, ctx, rq);
272+}
273+
274+static void blk_mq_bio_endio(struct request *rq, struct bio *bio, int error)
275+{
276+ if (error)
277+ clear_bit(BIO_UPTODATE, &bio->bi_flags);
278+ else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
279+ error = -EIO;
280+
281+ if (unlikely(rq->cmd_flags & REQ_QUIET))
282+ set_bit(BIO_QUIET, &bio->bi_flags);
283+
284+ /* don't actually finish bio if it's part of flush sequence */
285+ if (!(rq->cmd_flags & REQ_FLUSH_SEQ))
286+ bio_endio(bio, error);
287+}
288+
289+void blk_mq_complete_request(struct request *rq, int error)
290+{
291+ struct bio *bio = rq->bio;
292+ unsigned int bytes = 0;
293+
294+ trace_block_rq_complete(rq->q, rq);
295+
296+ while (bio) {
297+ struct bio *next = bio->bi_next;
298+
299+ bio->bi_next = NULL;
300+ bytes += bio->bi_size;
301+ blk_mq_bio_endio(rq, bio, error);
302+ bio = next;
303+ }
304+
305+ blk_account_io_completion(rq, bytes);
306+
307+ if (rq->end_io)
308+ rq->end_io(rq, error);
309+ else
310+ blk_mq_free_request(rq);
311+
312+ blk_account_io_done(rq);
313+}
314+
315+void __blk_mq_end_io(struct request *rq, int error)
316+{
317+ if (!blk_mark_rq_complete(rq))
318+ blk_mq_complete_request(rq, error);
319+}
320+
321+#if defined(CONFIG_SMP) && defined(CONFIG_USE_GENERIC_SMP_HELPERS)
322+
323+/*
324+ * Called with interrupts disabled.
325+ */
326+static void ipi_end_io(void *data)
327+{
328+ struct llist_head *list = &per_cpu(ipi_lists, smp_processor_id());
329+ struct llist_node *entry, *next;
330+ struct request *rq;
331+
332+ entry = llist_del_all(list);
333+
334+ while (entry) {
335+ next = entry->next;
336+ rq = llist_entry(entry, struct request, ll_list);
337+ __blk_mq_end_io(rq, rq->errors);
338+ entry = next;
339+ }
340+}
341+
342+static int ipi_remote_cpu(struct blk_mq_ctx *ctx, const int cpu,
343+ struct request *rq, const int error)
344+{
345+ struct call_single_data *data = &rq->csd;
346+
347+ rq->errors = error;
348+ rq->ll_list.next = NULL;
349+
350+ /*
351+ * If the list is non-empty, an existing IPI must already
352+ * be "in flight". If that is the case, we need not schedule
353+ * a new one.
354+ */
355+ if (llist_add(&rq->ll_list, &per_cpu(ipi_lists, ctx->cpu))) {
356+ data->func = ipi_end_io;
357+ data->flags = 0;
358+ __smp_call_function_single(ctx->cpu, data, 0);
359+ }
360+
361+ return true;
362+}
363+#else /* CONFIG_SMP && CONFIG_USE_GENERIC_SMP_HELPERS */
364+static int ipi_remote_cpu(struct blk_mq_ctx *ctx, const int cpu,
365+ struct request *rq, const int error)
366+{
367+ return false;
368+}
369+#endif
370+
371+/*
372+ * End IO on this request on a multiqueue enabled driver. We'll either do
373+ * it directly inline, or punt to a local IPI handler on the matching
374+ * remote CPU.
375+ */
376+void blk_mq_end_io(struct request *rq, int error)
377+{
378+ struct blk_mq_ctx *ctx = rq->mq_ctx;
379+ int cpu;
380+
381+ if (!ctx->ipi_redirect)
382+ return __blk_mq_end_io(rq, error);
383+
384+ cpu = get_cpu();
385+
386+ if (cpu == ctx->cpu || !cpu_online(ctx->cpu) ||
387+ !ipi_remote_cpu(ctx, cpu, rq, error))
388+ __blk_mq_end_io(rq, error);
389+
390+ put_cpu();
391+}
392+EXPORT_SYMBOL(blk_mq_end_io);
393+
394+static void blk_mq_start_request(struct request *rq)
395+{
396+ struct request_queue *q = rq->q;
397+
398+ trace_block_rq_issue(q, rq);
399+
400+ /*
401+ * Just mark start time and set the started bit. Due to memory
402+ * ordering, we know we'll see the correct deadline as long as
403+ * REQ_ATOMIC_STARTED is seen.
404+ */
405+ rq->deadline = jiffies + q->rq_timeout;
406+ set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
407+}
408+
409+static void blk_mq_requeue_request(struct request *rq)
410+{
411+ struct request_queue *q = rq->q;
412+
413+ trace_block_rq_requeue(q, rq);
414+ clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
415+}
416+
417+struct blk_mq_timeout_data {
418+ struct blk_mq_hw_ctx *hctx;
419+ unsigned long *next;
420+ unsigned int *next_set;
421+};
422+
423+static void blk_mq_timeout_check(void *__data, unsigned long *free_tags)
424+{
425+ struct blk_mq_timeout_data *data = __data;
426+ struct blk_mq_hw_ctx *hctx = data->hctx;
427+ unsigned int tag;
428+
429+ /* It may not be in flight yet (this is where
430+ * the REQ_ATOMIC_STARTED flag comes in). The requests are
431+ * statically allocated, so we know it's always safe to access the
432+ * memory associated with a bit offset into ->rqs[].
433+ */
434+ tag = 0;
435+ do {
436+ struct request *rq;
437+
438+ tag = find_next_zero_bit(free_tags, hctx->queue_depth, tag);
439+ if (tag >= hctx->queue_depth)
440+ break;
441+
442+ rq = hctx->rqs[tag++];
443+
444+ if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
445+ continue;
446+
447+ blk_rq_check_expired(rq, data->next, data->next_set);
448+ } while (1);
449+}
450+
451+static void blk_mq_hw_ctx_check_timeout(struct blk_mq_hw_ctx *hctx,
452+ unsigned long *next,
453+ unsigned int *next_set)
454+{
455+ struct blk_mq_timeout_data data = {
456+ .hctx = hctx,
457+ .next = next,
458+ .next_set = next_set,
459+ };
460+
461+ /*
462+ * Ask the tagging code to iterate busy requests, so we can
463+ * check them for timeout.
464+ */
465+ blk_mq_tag_busy_iter(hctx->tags, blk_mq_timeout_check, &data);
466+}
467+
468+static void blk_mq_rq_timer(unsigned long data)
469+{
470+ struct request_queue *q = (struct request_queue *) data;
471+ struct blk_mq_hw_ctx *hctx;
472+ unsigned long next = 0;
473+ int i, next_set = 0;
474+
475+ queue_for_each_hw_ctx(q, hctx, i)
476+ blk_mq_hw_ctx_check_timeout(hctx, &next, &next_set);
477+
478+ if (next_set)
479+ mod_timer(&q->timeout, round_jiffies_up(next));
480+}
481+
482+/*
483+ * Reverse check our software queue for entries that we could potentially
484+ * merge with. Currently includes a hand-wavy stop count of 8, to not spend
485+ * too much time checking for merges.
486+ */
487+static bool blk_mq_attempt_merge(struct request_queue *q,
488+ struct blk_mq_ctx *ctx, struct bio *bio)
489+{
490+ struct request *rq;
491+ int checked = 8;
492+
493+ list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
494+ int el_ret;
495+
496+ if (!checked--)
497+ break;
498+
499+ if (!blk_rq_merge_ok(rq, bio))
500+ continue;
501+
502+ el_ret = blk_try_merge(rq, bio);
503+ if (el_ret == ELEVATOR_BACK_MERGE) {
504+ if (bio_attempt_back_merge(q, rq, bio)) {
505+ ctx->rq_merged++;
506+ return true;
507+ }
508+ break;
509+ } else if (el_ret == ELEVATOR_FRONT_MERGE) {
510+ if (bio_attempt_front_merge(q, rq, bio)) {
511+ ctx->rq_merged++;
512+ return true;
513+ }
514+ break;
515+ }
516+ }
517+
518+ return false;
519+}
520+
521+void blk_mq_add_timer(struct request *rq)
522+{
523+ __blk_add_timer(rq, NULL);
524+}
525+
526+/*
527+ * Run this hardware queue, pulling any software queues mapped to it in.
528+ * Note that this function currently has various problems around ordering
529+ * of IO. In particular, we'd like FIFO behaviour on handling existing
530+ * items on the hctx->dispatch list. Ignore that for now.
531+ */
532+static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
533+{
534+ struct request_queue *q = hctx->queue;
535+ struct blk_mq_ctx *ctx;
536+ struct request *rq;
537+ LIST_HEAD(rq_list);
538+ int bit, queued;
539+
540+ if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->flags)))
541+ return;
542+
543+ hctx->run++;
544+
545+ /*
546+ * Touch any software queue that has pending entries.
547+ */
548+ for_each_set_bit(bit, hctx->ctx_map, hctx->nr_ctx) {
549+ clear_bit(bit, hctx->ctx_map);
550+ ctx = hctx->ctxs[bit];
551+ BUG_ON(bit != ctx->index_hw);
552+
553+ spin_lock(&ctx->lock);
554+ list_splice_tail_init(&ctx->rq_list, &rq_list);
555+ spin_unlock(&ctx->lock);
556+ }
557+
558+ /*
559+ * If we have previous entries on our dispatch list, grab them
560+ * and stuff them at the front for more fair dispatch.
561+ */
562+ if (!list_empty_careful(&hctx->dispatch)) {
563+ spin_lock(&hctx->lock);
564+ if (!list_empty(&hctx->dispatch))
565+ list_splice_init(&hctx->dispatch, &rq_list);
566+ spin_unlock(&hctx->lock);
567+ }
568+
569+ /*
570+ * Delete and return all entries from our dispatch list
571+ */
572+ queued = 0;
573+
574+ /*
575+ * Now process all the entries, sending them to the driver.
576+ */
577+ while (!list_empty(&rq_list)) {
578+ int ret;
579+
580+ rq = list_first_entry(&rq_list, struct request, queuelist);
581+ list_del_init(&rq->queuelist);
582+ blk_mq_start_request(rq);
583+
584+ /*
585+ * Last request in the series. Flag it as such, this
586+ * enables drivers to know when IO should be kicked off,
587+ * if they don't do it on a per-request basis.
588+ *
589+ * Note: the flag isn't the only condition drivers
590+ * should do kick off. If drive is busy, the last
591+ * request might not have the bit set.
592+ */
593+ if (list_empty(&rq_list))
594+ rq->cmd_flags |= REQ_END;
595+
596+ ret = q->mq_ops->queue_rq(hctx, rq);
597+ switch (ret) {
598+ case BLK_MQ_RQ_QUEUE_OK:
599+ queued++;
600+ continue;
601+ case BLK_MQ_RQ_QUEUE_BUSY:
602+ /*
603+ * FIXME: we should have a mechanism to stop the queue
604+ * like blk_stop_queue, otherwise we will waste cpu
605+ * time
606+ */
607+ list_add(&rq->queuelist, &rq_list);
608+ blk_mq_requeue_request(rq);
609+ break;
610+ default:
611+ pr_err("blk-mq: bad return on queue: %d\n", ret);
612+ rq->errors = -EIO;
613+ case BLK_MQ_RQ_QUEUE_ERROR:
614+ blk_mq_end_io(rq, rq->errors);
615+ break;
616+ }
617+
618+ if (ret == BLK_MQ_RQ_QUEUE_BUSY)
619+ break;
620+ }
621+
622+ if (!queued)
623+ hctx->dispatched[0]++;
624+ else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
625+ hctx->dispatched[ilog2(queued) + 1]++;
626+
627+ /*
628+ * Any items that need requeuing? Stuff them into hctx->dispatch,
629+ * that is where we will continue on next queue run.
630+ */
631+ if (!list_empty(&rq_list)) {
632+ spin_lock(&hctx->lock);
633+ list_splice(&rq_list, &hctx->dispatch);
634+ spin_unlock(&hctx->lock);
635+ }
636+}
637+
638+void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
639+{
640+ if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->flags)))
641+ return;
642+
643+ if (!async)
644+ __blk_mq_run_hw_queue(hctx);
645+ else {
646+ struct request_queue *q = hctx->queue;
647+
648+ kblockd_schedule_delayed_work(q, &hctx->delayed_work, 0);
649+ }
650+}
651+
652+void blk_mq_run_queues(struct request_queue *q, bool async)
653+{
654+ struct blk_mq_hw_ctx *hctx;
655+ int i;
656+
657+ queue_for_each_hw_ctx(q, hctx, i) {
658+ if ((!blk_mq_hctx_has_pending(hctx) &&
659+ list_empty_careful(&hctx->dispatch)) ||
660+ test_bit(BLK_MQ_S_STOPPED, &hctx->flags))
661+ continue;
662+
663+ blk_mq_run_hw_queue(hctx, async);
664+ }
665+}
666+EXPORT_SYMBOL(blk_mq_run_queues);
667+
668+void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
669+{
670+ cancel_delayed_work(&hctx->delayed_work);
671+ set_bit(BLK_MQ_S_STOPPED, &hctx->state);
672+}
673+EXPORT_SYMBOL(blk_mq_stop_hw_queue);
674+
675+void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
676+{
677+ clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
678+ __blk_mq_run_hw_queue(hctx);
679+}
680+EXPORT_SYMBOL(blk_mq_start_hw_queue);
681+
682+void blk_mq_start_stopped_hw_queues(struct request_queue *q)
683+{
684+ struct blk_mq_hw_ctx *hctx;
685+ int i;
686+
687+ queue_for_each_hw_ctx(q, hctx, i) {
688+ if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
689+ continue;
690+
691+ clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
692+ blk_mq_run_hw_queue(hctx, true);
693+ }
694+}
695+EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
696+
697+static void blk_mq_work_fn(struct work_struct *work)
698+{
699+ struct blk_mq_hw_ctx *hctx;
700+
701+ hctx = container_of(work, struct blk_mq_hw_ctx, delayed_work.work);
702+ __blk_mq_run_hw_queue(hctx);
703+}
704+
705+static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
706+ struct request *rq)
707+{
708+ struct blk_mq_ctx *ctx = rq->mq_ctx;
709+
710+ list_add_tail(&rq->queuelist, &ctx->rq_list);
711+ blk_mq_hctx_mark_pending(hctx, ctx);
712+
713+ /*
714+ * We do this early, to ensure we are on the right CPU.
715+ */
716+ blk_mq_add_timer(rq);
717+}
718+
719+void blk_mq_insert_request(struct request_queue *q, struct request *rq,
720+ bool run_queue)
721+{
722+ struct blk_mq_hw_ctx *hctx;
723+ struct blk_mq_ctx *ctx, *current_ctx;
724+
725+ ctx = rq->mq_ctx;
726+ hctx = q->mq_ops->map_queue(q, ctx->cpu);
727+
728+ if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA)) {
729+ blk_insert_flush(rq);
730+ } else {
731+ current_ctx = blk_mq_get_ctx(q);
732+
733+ if (!cpu_online(ctx->cpu)) {
734+ ctx = current_ctx;
735+ hctx = q->mq_ops->map_queue(q, ctx->cpu);
736+ rq->mq_ctx = ctx;
737+ }
738+ spin_lock(&ctx->lock);
739+ __blk_mq_insert_request(hctx, rq);
740+ spin_unlock(&ctx->lock);
741+
742+ blk_mq_put_ctx(current_ctx);
743+ }
744+
745+ if (run_queue)
746+ __blk_mq_run_hw_queue(hctx);
747+}
748+EXPORT_SYMBOL(blk_mq_insert_request);
749+
750+/*
751+ * This is a special version of blk_mq_insert_request to bypass FLUSH request
752+ * check. Should only be used internally.
753+ */
754+void blk_mq_run_request(struct request *rq, bool run_queue, bool async)
755+{
756+ struct request_queue *q = rq->q;
757+ struct blk_mq_hw_ctx *hctx;
758+ struct blk_mq_ctx *ctx, *current_ctx;
759+
760+ current_ctx = blk_mq_get_ctx(q);
761+
762+ ctx = rq->mq_ctx;
763+ if (!cpu_online(ctx->cpu)) {
764+ ctx = current_ctx;
765+ rq->mq_ctx = ctx;
766+ }
767+ hctx = q->mq_ops->map_queue(q, ctx->cpu);
768+
769+ /* ctx->cpu might be offline */
770+ spin_lock(&ctx->lock);
771+ __blk_mq_insert_request(hctx, rq);
772+ spin_unlock(&ctx->lock);
773+
774+ blk_mq_put_ctx(current_ctx);
775+
776+ if (run_queue)
777+ blk_mq_run_hw_queue(hctx, async);
778+}
779+
780+static void blk_mq_insert_requests(struct request_queue *q,
781+ struct blk_mq_ctx *ctx,
782+ struct list_head *list,
783+ int depth,
784+ bool from_schedule)
785+
786+{
787+ struct blk_mq_hw_ctx *hctx;
788+ struct blk_mq_ctx *current_ctx;
789+
790+ trace_block_unplug(q, depth, !from_schedule);
791+
792+ current_ctx = blk_mq_get_ctx(q);
793+
794+ if (!cpu_online(ctx->cpu))
795+ ctx = current_ctx;
796+ hctx = q->mq_ops->map_queue(q, ctx->cpu);
797+
798+ /*
799+ * preemption doesn't flush plug list, so it's possible ctx->cpu is
800+ * offline now
801+ */
802+ spin_lock(&ctx->lock);
803+ while (!list_empty(list)) {
804+ struct request *rq;
805+
806+ rq = list_first_entry(list, struct request, queuelist);
807+ list_del_init(&rq->queuelist);
808+ rq->mq_ctx = ctx;
809+ __blk_mq_insert_request(hctx, rq);
810+ }
811+ spin_unlock(&ctx->lock);
812+
813+ blk_mq_put_ctx(current_ctx);
814+
815+ blk_mq_run_hw_queue(hctx, from_schedule);
816+}
817+
818+static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
819+{
820+ struct request *rqa = container_of(a, struct request, queuelist);
821+ struct request *rqb = container_of(b, struct request, queuelist);
822+
823+ return !(rqa->mq_ctx < rqb->mq_ctx ||
824+ (rqa->mq_ctx == rqb->mq_ctx &&
825+ blk_rq_pos(rqa) < blk_rq_pos(rqb)));
826+}
827+
828+void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
829+{
830+ struct blk_mq_ctx *this_ctx;
831+ struct request_queue *this_q;
832+ struct request *rq;
833+ LIST_HEAD(list);
834+ LIST_HEAD(ctx_list);
835+ unsigned int depth;
836+
837+ list_splice_init(&plug->mq_list, &list);
838+
839+ list_sort(NULL, &list, plug_ctx_cmp);
840+
841+ this_q = NULL;
842+ this_ctx = NULL;
843+ depth = 0;
844+
845+ while (!list_empty(&list)) {
846+ rq = list_entry_rq(list.next);
847+ list_del_init(&rq->queuelist);
848+ BUG_ON(!rq->q);
849+ if (rq->mq_ctx != this_ctx) {
850+ if (this_ctx) {
851+ blk_mq_insert_requests(this_q, this_ctx,
852+ &ctx_list, depth,
853+ from_schedule);
854+ }
855+
856+ this_ctx = rq->mq_ctx;
857+ this_q = rq->q;
858+ depth = 0;
859+ }
860+
861+ depth++;
862+ list_add_tail(&rq->queuelist, &ctx_list);
863+ }
864+
865+ /*
866+ * If 'this_ctx' is set, we know we have entries to complete
867+ * on 'ctx_list'. Do those.
868+ */
869+ if (this_ctx) {
870+ blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
871+ from_schedule);
872+ }
873+}
874+
875+static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
876+{
877+ init_request_from_bio(rq, bio);
878+ blk_account_io_start(rq, 1);
879+}
880+
881+static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
882+{
883+ struct blk_mq_hw_ctx *hctx;
884+ struct blk_mq_ctx *ctx;
885+ const int is_sync = rw_is_sync(bio->bi_rw);
886+ const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
887+ int rw = bio_data_dir(bio);
888+ struct request *rq;
889+ unsigned int use_plug, request_count = 0;
890+
891+ /*
892+ * If we have multiple hardware queues, just go directly to
893+ * one of those for sync IO.
894+ */
895+ use_plug = !is_flush_fua && ((q->nr_hw_queues == 1) || !is_sync);
896+
897+ blk_queue_bounce(q, &bio);
898+
899+ if (use_plug && blk_attempt_plug_merge(q, bio, &request_count))
900+ return;
901+
902+ if (blk_mq_queue_enter(q)) {
903+ bio_endio(bio, -EIO);
904+ return;
905+ }
906+
907+ ctx = blk_mq_get_ctx(q);
908+ hctx = q->mq_ops->map_queue(q, ctx->cpu);
909+
910+ trace_block_getrq(q, bio, rw);
911+ rq = __blk_mq_alloc_request(hctx, GFP_ATOMIC, false);
912+ if (likely(rq))
913+ blk_mq_rq_ctx_init(ctx, rq, rw);
914+ else {
915+ blk_mq_put_ctx(ctx);
916+ trace_block_sleeprq(q, bio, rw);
917+ rq = blk_mq_alloc_request_pinned(q, rw, __GFP_WAIT|GFP_ATOMIC,
918+ false);
919+ ctx = rq->mq_ctx;
920+ hctx = q->mq_ops->map_queue(q, ctx->cpu);
921+ }
922+
923+ hctx->queued++;
924+
925+ if (unlikely(is_flush_fua)) {
926+ blk_mq_bio_to_request(rq, bio);
927+ blk_mq_put_ctx(ctx);
928+ blk_insert_flush(rq);
929+ goto run_queue;
930+ }
931+
932+ /*
933+ * A task plug currently exists. Since this is completely lockless,
934+ * utilize that to temporarily store requests until the task is
935+ * either done or scheduled away.
936+ */
937+ if (use_plug) {
938+ struct blk_plug *plug = current->plug;
939+
940+ if (plug) {
941+ blk_mq_bio_to_request(rq, bio);
942+ if (list_empty(&plug->list))
943+ trace_block_plug(q);
944+ else if (request_count >= BLK_MAX_REQUEST_COUNT) {
945+ blk_flush_plug_list(plug, false);
946+ trace_block_plug(q);
947+ }
948+ list_add_tail(&rq->queuelist, &plug->mq_list);
949+ blk_mq_put_ctx(ctx);
950+ return;
951+ }
952+ }
953+
954+ spin_lock(&ctx->lock);
955+
956+ if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
957+ blk_mq_attempt_merge(q, ctx, bio))
958+ __blk_mq_free_request(hctx, ctx, rq);
959+ else {
960+ blk_mq_bio_to_request(rq, bio);
961+ __blk_mq_insert_request(hctx, rq);
962+ }
963+
964+ spin_unlock(&ctx->lock);
965+ blk_mq_put_ctx(ctx);
966+
967+ /*
968+ * For a SYNC request, send it to the hardware immediately. For an
969+ * ASYNC request, just ensure that we run it later on. The latter
970+ * allows for merging opportunities and more efficient dispatching.
971+ */
972+run_queue:
973+ blk_mq_run_hw_queue(hctx, !is_sync || is_flush_fua);
974+}
975+
976+/*
977+ * Default mapping to a software queue, since we use one per CPU.
978+ */
979+struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
980+{
981+ return q->queue_hw_ctx[q->mq_map[cpu]];
982+}
983+EXPORT_SYMBOL(blk_mq_map_queue);
984+
985+struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_reg *reg,
986+ unsigned int hctx_index)
987+{
988+ return kmalloc_node(sizeof(struct blk_mq_hw_ctx),
989+ GFP_KERNEL | __GFP_ZERO, reg->numa_node);
990+}
991+EXPORT_SYMBOL(blk_mq_alloc_single_hw_queue);
992+
993+void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *hctx,
994+ unsigned int hctx_index)
995+{
996+ kfree(hctx);
997+}
998+EXPORT_SYMBOL(blk_mq_free_single_hw_queue);
999+
1000+static void blk_mq_hctx_notify(void *data, unsigned long action,
1001+ unsigned int cpu)
1002+{
1003+ struct blk_mq_hw_ctx *hctx = data;
1004+ struct blk_mq_ctx *ctx;
1005+ LIST_HEAD(tmp);
1006+
1007+ if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
1008+ return;
1009+
1010+ /*
1011+ * Move ctx entries to new CPU, if this one is going away.
1012+ */
1013+ ctx = __blk_mq_get_ctx(hctx->queue, cpu);
1014+
1015+ spin_lock(&ctx->lock);
1016+ if (!list_empty(&ctx->rq_list)) {
1017+ list_splice_init(&ctx->rq_list, &tmp);
1018+ clear_bit(ctx->index_hw, hctx->ctx_map);
1019+ }
1020+ spin_unlock(&ctx->lock);
1021+
1022+ if (list_empty(&tmp))
1023+ return;
1024+
1025+ ctx = blk_mq_get_ctx(hctx->queue);
1026+ spin_lock(&ctx->lock);
1027+
1028+ while (!list_empty(&tmp)) {
1029+ struct request *rq;
1030+
1031+ rq = list_first_entry(&tmp, struct request, queuelist);
1032+ rq->mq_ctx = ctx;
1033+ list_move_tail(&rq->queuelist, &ctx->rq_list);
1034+ }
1035+
1036+ blk_mq_hctx_mark_pending(hctx, ctx);
1037+
1038+ spin_unlock(&ctx->lock);
1039+ blk_mq_put_ctx(ctx);
1040+}
1041+
1042+static void blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx,
1043+ void (*init)(void *, struct blk_mq_hw_ctx *,
1044+ struct request *, unsigned int),
1045+ void *data)
1046+{
1047+ unsigned int i;
1048+
1049+ for (i = 0; i < hctx->queue_depth; i++) {
1050+ struct request *rq = hctx->rqs[i];
1051+
1052+ init(data, hctx, rq, i);
1053+ }
1054+}
1055+
1056+void blk_mq_init_commands(struct request_queue *q,
1057+ void (*init)(void *, struct blk_mq_hw_ctx *,
1058+ struct request *, unsigned int),
1059+ void *data)
1060+{
1061+ struct blk_mq_hw_ctx *hctx;
1062+ unsigned int i;
1063+
1064+ queue_for_each_hw_ctx(q, hctx, i)
1065+ blk_mq_init_hw_commands(hctx, init, data);
1066+}
1067+EXPORT_SYMBOL(blk_mq_init_commands);
1068+
1069+static void blk_mq_free_rq_map(struct blk_mq_hw_ctx *hctx)
1070+{
1071+ struct page *page;
1072+
1073+ while (!list_empty(&hctx->page_list)) {
1074+ page = list_first_entry(&hctx->page_list, struct page, list);
1075+ list_del_init(&page->list);
1076+ __free_pages(page, page->private);
1077+ }
1078+
1079+ kfree(hctx->rqs);
1080+
1081+ if (hctx->tags)
1082+ blk_mq_free_tags(hctx->tags);
1083+}
1084+
1085+static size_t order_to_size(unsigned int order)
1086+{
1087+ size_t ret = PAGE_SIZE;
1088+
1089+ while (order--)
1090+ ret *= 2;
1091+
1092+ return ret;
1093+}
1094+
1095+static int blk_mq_init_rq_map(struct blk_mq_hw_ctx *hctx,
1096+ unsigned int reserved_tags, int node)
1097+{
1098+ unsigned int i, j, entries_per_page, max_order = 4;
1099+ size_t rq_size, left;
1100+
1101+ INIT_LIST_HEAD(&hctx->page_list);
1102+
1103+ hctx->rqs = kmalloc_node(hctx->queue_depth * sizeof(struct request *),
1104+ GFP_KERNEL, node);
1105+ if (!hctx->rqs)
1106+ return -ENOMEM;
1107+
1108+ /*
1109+ * rq_size is the size of the request plus driver payload, rounded
1110+ * to the cacheline size
1111+ */
1112+ rq_size = round_up(sizeof(struct request) + hctx->cmd_size,
1113+ cache_line_size());
1114+ left = rq_size * hctx->queue_depth;
1115+
1116+ for (i = 0; i < hctx->queue_depth;) {
1117+ int this_order = max_order;
1118+ struct page *page;
1119+ int to_do;
1120+ void *p;
1121+
1122+ while (left < order_to_size(this_order - 1) && this_order)
1123+ this_order--;
1124+
1125+ do {
1126+ page = alloc_pages_node(node, GFP_KERNEL, this_order);
1127+ if (page)
1128+ break;
1129+ if (!this_order--)
1130+ break;
1131+ if (order_to_size(this_order) < rq_size)
1132+ break;
1133+ } while (1);
1134+
1135+ if (!page)
1136+ break;
1137+
1138+ page->private = this_order;
1139+ list_add_tail(&page->list, &hctx->page_list);
1140+
1141+ p = page_address(page);
1142+ entries_per_page = order_to_size(this_order) / rq_size;
1143+ to_do = min(entries_per_page, hctx->queue_depth - i);
1144+ left -= to_do * rq_size;
1145+ for (j = 0; j < to_do; j++) {
1146+ hctx->rqs[i] = p;
1147+ blk_mq_rq_init(hctx, hctx->rqs[i]);
1148+ p += rq_size;
1149+ i++;
1150+ }
1151+ }
1152+
1153+ if (i < (reserved_tags + BLK_MQ_TAG_MIN))
1154+ goto err_rq_map;
1155+ else if (i != hctx->queue_depth) {
1156+ hctx->queue_depth = i;
1157+ pr_warn("%s: queue depth set to %u because of low memory\n",
1158+ __func__, i);
1159+ }
1160+
1161+ hctx->tags = blk_mq_init_tags(hctx->queue_depth, reserved_tags, node);
1162+ if (!hctx->tags) {
1163+err_rq_map:
1164+ blk_mq_free_rq_map(hctx);
1165+ return -ENOMEM;
1166+ }
1167+
1168+ return 0;
1169+}
1170+
1171+static int blk_mq_init_hw_queues(struct request_queue *q,
1172+ struct blk_mq_reg *reg, void *driver_data)
1173+{
1174+ struct blk_mq_hw_ctx *hctx;
1175+ unsigned int i, j;
1176+
1177+ /*
1178+ * Initialize hardware queues
1179+ */
1180+ queue_for_each_hw_ctx(q, hctx, i) {
1181+ unsigned int num_maps;
1182+ int node;
1183+
1184+ node = hctx->numa_node;
1185+ if (node == NUMA_NO_NODE)
1186+ node = hctx->numa_node = reg->numa_node;
1187+
1188+ INIT_DELAYED_WORK(&hctx->delayed_work, blk_mq_work_fn);
1189+ spin_lock_init(&hctx->lock);
1190+ INIT_LIST_HEAD(&hctx->dispatch);
1191+ hctx->queue = q;
1192+ hctx->queue_num = i;
1193+ hctx->flags = reg->flags;
1194+ hctx->queue_depth = reg->queue_depth;
1195+ hctx->cmd_size = reg->cmd_size;
1196+
1197+ blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1198+ blk_mq_hctx_notify, hctx);
1199+ blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1200+
1201+ if (blk_mq_init_rq_map(hctx, reg->reserved_tags, node))
1202+ break;
1203+
1204+ /*
1205+ * Allocate space for all possible cpus to avoid allocation in
1206+ * runtime
1207+ */
1208+ hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1209+ GFP_KERNEL, node);
1210+ if (!hctx->ctxs)
1211+ break;
1212+
1213+ num_maps = ALIGN(nr_cpu_ids, BITS_PER_LONG) / BITS_PER_LONG;
1214+ hctx->ctx_map = kzalloc_node(num_maps * sizeof(unsigned long),
1215+ GFP_KERNEL, node);
1216+ if (!hctx->ctx_map)
1217+ break;
1218+
1219+ hctx->nr_ctx_map = num_maps;
1220+ hctx->nr_ctx = 0;
1221+
1222+ if (reg->ops->init_hctx &&
1223+ reg->ops->init_hctx(hctx, driver_data, i))
1224+ break;
1225+ }
1226+
1227+ if (i == q->nr_hw_queues)
1228+ return 0;
1229+
1230+ /*
1231+ * Init failed
1232+ */
1233+ queue_for_each_hw_ctx(q, hctx, j) {
1234+ if (i == j)
1235+ break;
1236+
1237+ if (reg->ops->exit_hctx)
1238+ reg->ops->exit_hctx(hctx, j);
1239+
1240+ blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1241+ blk_mq_free_rq_map(hctx);
1242+ kfree(hctx->ctxs);
1243+ }
1244+
1245+ return 1;
1246+}
1247+
1248+static void blk_mq_init_cpu_queues(struct request_queue *q,
1249+ unsigned int nr_hw_queues)
1250+{
1251+ unsigned int i;
1252+
1253+ for_each_possible_cpu(i) {
1254+ struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1255+ struct blk_mq_hw_ctx *hctx;
1256+
1257+ memset(__ctx, 0, sizeof(*__ctx));
1258+ __ctx->cpu = i;
1259+ spin_lock_init(&__ctx->lock);
1260+ INIT_LIST_HEAD(&__ctx->rq_list);
1261+ __ctx->queue = q;
1262+
1263+ /* If the cpu isn't online, the cpu is mapped to first hctx */
1264+ hctx = q->mq_ops->map_queue(q, i);
1265+ hctx->nr_ctx++;
1266+
1267+ if (!cpu_online(i))
1268+ continue;
1269+
1270+ /*
1271+ * Set local node, IFF we have more than one hw queue. If
1272+ * not, we remain on the home node of the device
1273+ */
1274+ if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1275+ hctx->numa_node = cpu_to_node(i);
1276+ }
1277+}
1278+
1279+static void blk_mq_map_swqueue(struct request_queue *q)
1280+{
1281+ unsigned int i;
1282+ struct blk_mq_hw_ctx *hctx;
1283+ struct blk_mq_ctx *ctx;
1284+
1285+ queue_for_each_hw_ctx(q, hctx, i) {
1286+ hctx->nr_ctx = 0;
1287+ }
1288+
1289+ /*
1290+ * Map software to hardware queues
1291+ */
1292+ queue_for_each_ctx(q, ctx, i) {
1293+ /* If the cpu isn't online, the cpu is mapped to first hctx */
1294+ hctx = q->mq_ops->map_queue(q, i);
1295+ ctx->index_hw = hctx->nr_ctx;
1296+ hctx->ctxs[hctx->nr_ctx++] = ctx;
1297+ }
1298+}
1299+
1300+struct request_queue *blk_mq_init_queue(struct blk_mq_reg *reg,
1301+ void *driver_data)
1302+{
1303+ struct blk_mq_hw_ctx **hctxs;
1304+ struct blk_mq_ctx *ctx;
1305+ struct request_queue *q;
1306+ int i;
1307+
1308+ if (!reg->nr_hw_queues ||
1309+ !reg->ops->queue_rq || !reg->ops->map_queue ||
1310+ !reg->ops->alloc_hctx || !reg->ops->free_hctx)
1311+ return ERR_PTR(-EINVAL);
1312+
1313+ if (!reg->queue_depth)
1314+ reg->queue_depth = BLK_MQ_MAX_DEPTH;
1315+ else if (reg->queue_depth > BLK_MQ_MAX_DEPTH) {
1316+ pr_err("blk-mq: queuedepth too large (%u)\n", reg->queue_depth);
1317+ reg->queue_depth = BLK_MQ_MAX_DEPTH;
1318+ }
1319+
1320+ if (reg->queue_depth < (reg->reserved_tags + BLK_MQ_TAG_MIN))
1321+ return ERR_PTR(-EINVAL);
1322+
1323+ ctx = alloc_percpu(struct blk_mq_ctx);
1324+ if (!ctx)
1325+ return ERR_PTR(-ENOMEM);
1326+
1327+ hctxs = kmalloc_node(reg->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1328+ reg->numa_node);
1329+
1330+ if (!hctxs)
1331+ goto err_percpu;
1332+
1333+ for (i = 0; i < reg->nr_hw_queues; i++) {
1334+ hctxs[i] = reg->ops->alloc_hctx(reg, i);
1335+ if (!hctxs[i])
1336+ goto err_hctxs;
1337+
1338+ hctxs[i]->numa_node = NUMA_NO_NODE;
1339+ hctxs[i]->queue_num = i;
1340+ }
1341+
1342+ q = blk_alloc_queue_node(GFP_KERNEL, reg->numa_node);
1343+ if (!q)
1344+ goto err_hctxs;
1345+
1346+ q->mq_map = blk_mq_make_queue_map(reg);
1347+ if (!q->mq_map)
1348+ goto err_map;
1349+
1350+ setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1351+ blk_queue_rq_timeout(q, 30000);
1352+
1353+ q->nr_queues = nr_cpu_ids;
1354+ q->nr_hw_queues = reg->nr_hw_queues;
1355+
1356+ q->queue_ctx = ctx;
1357+ q->queue_hw_ctx = hctxs;
1358+
1359+ q->mq_ops = reg->ops;
1360+
1361+ blk_queue_make_request(q, blk_mq_make_request);
1362+ blk_queue_rq_timed_out(q, reg->ops->timeout);
1363+ if (reg->timeout)
1364+ blk_queue_rq_timeout(q, reg->timeout);
1365+
1366+ blk_mq_init_flush(q);
1367+ blk_mq_init_cpu_queues(q, reg->nr_hw_queues);
1368+
1369+ if (blk_mq_init_hw_queues(q, reg, driver_data))
1370+ goto err_hw;
1371+
1372+ blk_mq_map_swqueue(q);
1373+
1374+ mutex_lock(&all_q_mutex);
1375+ list_add_tail(&q->all_q_node, &all_q_list);
1376+ mutex_unlock(&all_q_mutex);
1377+
1378+ return q;
1379+err_hw:
1380+ kfree(q->mq_map);
1381+err_map:
1382+ blk_cleanup_queue(q);
1383+err_hctxs:
1384+ for (i = 0; i < reg->nr_hw_queues; i++) {
1385+ if (!hctxs[i])
1386+ break;
1387+ reg->ops->free_hctx(hctxs[i], i);
1388+ }
1389+ kfree(hctxs);
1390+err_percpu:
1391+ free_percpu(ctx);
1392+ return ERR_PTR(-ENOMEM);
1393+}
1394+EXPORT_SYMBOL(blk_mq_init_queue);
1395+
1396+void blk_mq_free_queue(struct request_queue *q)
1397+{
1398+ struct blk_mq_hw_ctx *hctx;
1399+ int i;
1400+
1401+ queue_for_each_hw_ctx(q, hctx, i) {
1402+ cancel_delayed_work_sync(&hctx->delayed_work);
1403+ kfree(hctx->ctx_map);
1404+ kfree(hctx->ctxs);
1405+ blk_mq_free_rq_map(hctx);
1406+ blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1407+ if (q->mq_ops->exit_hctx)
1408+ q->mq_ops->exit_hctx(hctx, i);
1409+ q->mq_ops->free_hctx(hctx, i);
1410+ }
1411+
1412+ free_percpu(q->queue_ctx);
1413+ kfree(q->queue_hw_ctx);
1414+ kfree(q->mq_map);
1415+
1416+ q->queue_ctx = NULL;
1417+ q->queue_hw_ctx = NULL;
1418+ q->mq_map = NULL;
1419+
1420+ mutex_lock(&all_q_mutex);
1421+ list_del_init(&q->all_q_node);
1422+ mutex_unlock(&all_q_mutex);
1423+}
1424+EXPORT_SYMBOL(blk_mq_free_queue);
1425+
1426+/* Basically redo blk_mq_init_queue with queue frozen */
1427+static void __cpuinit blk_mq_queue_reinit(struct request_queue *q)
1428+{
1429+ blk_mq_freeze_queue(q);
1430+
1431+ blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1432+
1433+ /*
1434+ * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1435+ * we should change hctx numa_node according to new topology (this
1436+ * involves free and re-allocate memory, worthy doing?)
1437+ */
1438+
1439+ blk_mq_map_swqueue(q);
1440+
1441+ blk_mq_unfreeze_queue(q);
1442+}
1443+
1444+static int __cpuinit blk_mq_queue_reinit_notify(struct notifier_block *nb,
1445+ unsigned long action, void *hcpu)
1446+{
1447+ struct request_queue *q;
1448+
1449+ /*
1450+ * Before new mapping is established, hotadded cpu might already start
1451+ * handling requests. This doesn't break anything as we map offline
1452+ * CPUs to first hardware queue. We will re-init queue below to get
1453+ * optimal settings.
1454+ */
1455+ if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1456+ action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1457+ return NOTIFY_OK;
1458+
1459+ mutex_lock(&all_q_mutex);
1460+ list_for_each_entry(q, &all_q_list, all_q_node)
1461+ blk_mq_queue_reinit(q);
1462+ mutex_unlock(&all_q_mutex);
1463+ return NOTIFY_OK;
1464+}
1465+
1466+static int __init blk_mq_init(void)
1467+{
1468+ unsigned int i;
1469+
1470+ for_each_possible_cpu(i)
1471+ init_llist_head(&per_cpu(ipi_lists, i));
1472+
1473+ blk_mq_cpu_init();
1474+
1475+ /* Must be called after percpu_counter_hotcpu_callback() */
1476+ hotcpu_notifier(blk_mq_queue_reinit_notify, -10);
1477+
1478+ return 0;
1479+}
1480+subsys_initcall(blk_mq_init);
block/blk-mq.h+52−0
@@ -0,0 +1,52 @@
1+#ifndef INT_BLK_MQ_H
2+#define INT_BLK_MQ_H
3+
4+struct blk_mq_ctx {
5+ struct {
6+ spinlock_t lock;
7+ struct list_head rq_list;
8+ } ____cacheline_aligned_in_smp;
9+
10+ unsigned int cpu;
11+ unsigned int index_hw;
12+ unsigned int ipi_redirect;
13+
14+ /* incremented at dispatch time */
15+ unsigned long rq_dispatched[2];
16+ unsigned long rq_merged;
17+
18+ /* incremented at completion time */
19+ unsigned long ____cacheline_aligned_in_smp rq_completed[2];
20+
21+ struct request_queue *queue;
22+ struct kobject kobj;
23+};
24+
25+void __blk_mq_end_io(struct request *rq, int error);
26+void blk_mq_complete_request(struct request *rq, int error);
27+void blk_mq_run_request(struct request *rq, bool run_queue, bool async);
28+void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
29+void blk_mq_init_flush(struct request_queue *q);
30+
31+/*
32+ * CPU hotplug helpers
33+ */
34+struct blk_mq_cpu_notifier;
35+void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier,
36+ void (*fn)(void *, unsigned long, unsigned int),
37+ void *data);
38+void blk_mq_register_cpu_notifier(struct blk_mq_cpu_notifier *notifier);
39+void blk_mq_unregister_cpu_notifier(struct blk_mq_cpu_notifier *notifier);
40+void blk_mq_cpu_init(void);
41+DECLARE_PER_CPU(struct llist_head, ipi_lists);
42+
43+/*
44+ * CPU -> queue mappings
45+ */
46+struct blk_mq_reg;
47+extern unsigned int *blk_mq_make_queue_map(struct blk_mq_reg *reg);
48+extern int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues);
49+
50+void blk_mq_add_timer(struct request *rq);
51+
52+#endif
block/blk-sysfs.c+13−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include <linux/bio.h>
88 #include <linux/blkdev.h>
99 #include <linux/blktrace_api.h>
10+#include <linux/blk-mq.h>
1011
1112 #include "blk.h"
1213 #include "blk-cgroup.h"
@@ -542,6 +543,11 @@ static void blk_release_queue(struct kobject *kobj) · 529 unmodified lines
542543 if (q->queue_tags)
543544 __blk_queue_free_tags(q);
544545
546+ percpu_counter_destroy(&q->mq_usage_counter);
547+
548+ if (q->mq_ops)
549+ blk_mq_free_queue(q);
550+
545551 blk_trace_shutdown(q);
546552
547553 bdi_destroy(&q->backing_dev_info);
@@ -575,6 +581,7 @@ int blk_register_queue(struct gendisk *disk) · 27 unmodified lines
575581 * bypass from queue allocation.
576582 */
577583 blk_queue_bypass_end(q);
584+ queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q);
578585
579586 ret = blk_trace_init_sysfs(dev);
580587 if (ret)
@@ -588,6 +595,9 @@ int blk_register_queue(struct gendisk *disk) · 7 unmodified lines
588595
589596 kobject_uevent(&q->kobj, KOBJ_ADD);
590597
598+ if (q->mq_ops)
599+ blk_mq_register_disk(disk);
600+
591601 if (!q->request_fn)
592602 return 0;
593603
@@ -610,6 +620,9 @@ void blk_unregister_queue(struct gendisk *disk) · 16 unmodified lines
610620 if (WARN_ON(!q))
611621 return;
612622
623+ if (q->mq_ops)
624+ blk_mq_unregister_disk(disk);
625+
613626 if (q->request_fn)
614627 elv_unregister_queue(q);
615628
block/blk-timeout.c+46−27
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include <linux/fault-inject.h>
88
99 #include "blk.h"
10+#include "blk-mq.h"
1011
1112 #ifdef CONFIG_FAIL_IO_TIMEOUT
1213
@@ -88,11 +89,18 @@ static void blk_rq_timed_out(struct request *req) · 75 unmodified lines
8889 ret = q->rq_timed_out_fn(req);
8990 switch (ret) {
9091 case BLK_EH_HANDLED:
91− __blk_complete_request(req);
92+ /* Can we use req->errors here? */
93+ if (q->mq_ops)
94+ blk_mq_complete_request(req, req->errors);
95+ else
96+ __blk_complete_request(req);
9297 break;
9398 case BLK_EH_RESET_TIMER:
9499 blk_clear_rq_complete(req);
95− blk_add_timer(req);
100+ if (q->mq_ops)
101+ blk_mq_add_timer(req);
102+ else
103+ blk_add_timer(req);
96104 break;
97105 case BLK_EH_NOT_HANDLED:
98106 /*
@@ -108,6 +116,23 @@ static void blk_rq_timed_out(struct request *req) · 9 unmodified lines
108116 }
109117 }
110118
119+void blk_rq_check_expired(struct request *rq, unsigned long *next_timeout,
120+ unsigned int *next_set)
121+{
122+ if (time_after_eq(jiffies, rq->deadline)) {
123+ list_del_init(&rq->timeout_list);
124+
125+ /*
126+ * Check if we raced with end io completion
127+ */
128+ if (!blk_mark_rq_complete(rq))
129+ blk_rq_timed_out(rq);
130+ } else if (!*next_set || time_after(*next_timeout, rq->deadline)) {
131+ *next_timeout = rq->deadline;
132+ *next_set = 1;
133+ }
134+}
135+
111136 void blk_rq_timed_out_timer(unsigned long data)
112137 {
113138 struct request_queue *q = (struct request_queue *) data;
@@ -117,21 +142,8 @@ void blk_rq_timed_out_timer(unsigned long data) · 3 unmodified lines
117142
118143 spin_lock_irqsave(q->queue_lock, flags);
119144
120− list_for_each_entry_safe(rq, tmp, &q->timeout_list, timeout_list) {
121− if (time_after_eq(jiffies, rq->deadline)) {
122− list_del_init(&rq->timeout_list);
123−
124− /*
125− * Check if we raced with end io completion
126− */
127− if (blk_mark_rq_complete(rq))
128− continue;
129− blk_rq_timed_out(rq);
130− } else if (!next_set || time_after(next, rq->deadline)) {
131− next = rq->deadline;
132− next_set = 1;
133− }
134− }
145+ list_for_each_entry_safe(rq, tmp, &q->timeout_list, timeout_list)
146+ blk_rq_check_expired(rq, &next, &next_set);
135147
136148 if (next_set)
137149 mod_timer(&q->timeout, round_jiffies_up(next));
@@ -157,15 +169,7 @@ void blk_abort_request(struct request *req) · 19 unmodified lines
157169 }
158170 EXPORT_SYMBOL_GPL(blk_abort_request);
159171
160−/**
161− * blk_add_timer - Start timeout timer for a single request
162− * @req: request that is about to start running.
163− *
164− * Notes:
165− * Each request has its own timer, and as it is added to the queue, we
166− * set up the timer. When the request completes, we cancel the timer.
167− */
168−void blk_add_timer(struct request *req)
172+void __blk_add_timer(struct request *req, struct list_head *timeout_list)
169173 {
170174 struct request_queue *q = req->q;
171175 unsigned long expiry;
@@ -184,7 +188,8 @@ void blk_add_timer(struct request *req) · 12 unmodified lines
184188 req->timeout = q->rq_timeout;
185189
186190 req->deadline = jiffies + req->timeout;
187− list_add_tail(&req->timeout_list, &q->timeout_list);
191+ if (timeout_list)
192+ list_add_tail(&req->timeout_list, timeout_list);
188193
189194 /*
190195 * If the timer isn't already pending or this timeout is earlier
@@ -196,5 +201,19 @@ void blk_add_timer(struct request *req) · 5 unmodified lines
196201 if (!timer_pending(&q->timeout) ||
197202 time_before(expiry, q->timeout.expires))
198203 mod_timer(&q->timeout, expiry);
204+
205+}
206+
207+/**
208+ * blk_add_timer - Start timeout timer for a single request
209+ * @req: request that is about to start running.
210+ *
211+ * Notes:
212+ * Each request has its own timer, and as it is added to the queue, we
213+ * set up the timer. When the request completes, we cancel the timer.
214+ */
215+void blk_add_timer(struct request *req)
216+{
217+ __blk_add_timer(req, &req->q->timeout_list);
199218 }
200219
block/blk.h+17−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010 #define BLK_BATCH_REQ 32
1111
1212 extern struct kmem_cache *blk_requestq_cachep;
13+extern struct kmem_cache *request_cachep;
1314 extern struct kobj_type blk_queue_ktype;
1415 extern struct ida blk_queue_ida;
1516
@@ -34,14 +35,30 @@ bool __blk_end_bidi_request(struct request *rq, int error, · 18 unmodified lines
3435 unsigned int nr_bytes, unsigned int bidi_bytes);
3536
3637 void blk_rq_timed_out_timer(unsigned long data);
38+void blk_rq_check_expired(struct request *rq, unsigned long *next_timeout,
39+ unsigned int *next_set);
40+void __blk_add_timer(struct request *req, struct list_head *timeout_list);
3741 void blk_delete_timer(struct request *);
3842 void blk_add_timer(struct request *);
3943
44+
45+bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
46+ struct bio *bio);
47+bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
48+ struct bio *bio);
49+bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
50+ unsigned int *request_count);
51+
52+void blk_account_io_start(struct request *req, bool new_io);
53+void blk_account_io_completion(struct request *req, unsigned int bytes);
54+void blk_account_io_done(struct request *req);
55+
4056 /*
4157 * Internal atomic flags for request handling
4258 */
4359 enum rq_atomic_flags {
4460 REQ_ATOM_COMPLETE = 0,
61+ REQ_ATOM_STARTED,
4562 };
4663
4764 /*
include/linux/bio.h+2−0
@@ -419,6 +419,8 @@ static inline void bio_list_init(struct bio_list *bl) · 418 unmodified lines
419419 bl->head = bl->tail = NULL;
420420 }
421421
422+#define BIO_EMPTY_LIST { NULL, NULL }
423+
422424 #define bio_list_for_each(bio, bl) \
423425 for (bio = (bl)->head; bio; bio = bio->bi_next)
424426
include/linux/blk-mq.h+182−0
@@ -0,0 +1,182 @@
1+#ifndef BLK_MQ_H
2+#define BLK_MQ_H
3+
4+#include <linux/blkdev.h>
5+
6+struct blk_mq_tags;
7+
8+struct blk_mq_cpu_notifier {
9+ struct list_head list;
10+ void *data;
11+ void (*notify)(void *data, unsigned long action, unsigned int cpu);
12+};
13+
14+struct blk_mq_hw_ctx {
15+ struct {
16+ spinlock_t lock;
17+ struct list_head dispatch;
18+ } ____cacheline_aligned_in_smp;
19+
20+ unsigned long state; /* BLK_MQ_S_* flags */
21+ struct delayed_work delayed_work;
22+
23+ unsigned long flags; /* BLK_MQ_F_* flags */
24+
25+ struct request_queue *queue;
26+ unsigned int queue_num;
27+
28+ void *driver_data;
29+
30+ unsigned int nr_ctx;
31+ struct blk_mq_ctx **ctxs;
32+ unsigned int nr_ctx_map;
33+ unsigned long *ctx_map;
34+
35+ struct request **rqs;
36+ struct list_head page_list;
37+ struct blk_mq_tags *tags;
38+
39+ unsigned long queued;
40+ unsigned long run;
41+#define BLK_MQ_MAX_DISPATCH_ORDER 10
42+ unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER];
43+
44+ unsigned int queue_depth;
45+ unsigned int numa_node;
46+ unsigned int cmd_size; /* per-request extra data */
47+
48+ struct blk_mq_cpu_notifier cpu_notifier;
49+ struct kobject kobj;
50+};
51+
52+struct blk_mq_reg {
53+ struct blk_mq_ops *ops;
54+ unsigned int nr_hw_queues;
55+ unsigned int queue_depth;
56+ unsigned int reserved_tags;
57+ unsigned int cmd_size; /* per-request extra data */
58+ int numa_node;
59+ unsigned int timeout;
60+ unsigned int flags; /* BLK_MQ_F_* */
61+};
62+
63+typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, struct request *);
64+typedef struct blk_mq_hw_ctx *(map_queue_fn)(struct request_queue *, const int);
65+typedef struct blk_mq_hw_ctx *(alloc_hctx_fn)(struct blk_mq_reg *,unsigned int);
66+typedef void (free_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int);
67+typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int);
68+typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int);
69+
70+struct blk_mq_ops {
71+ /*
72+ * Queue request
73+ */
74+ queue_rq_fn *queue_rq;
75+
76+ /*
77+ * Map to specific hardware queue
78+ */
79+ map_queue_fn *map_queue;
80+
81+ /*
82+ * Called on request timeout
83+ */
84+ rq_timed_out_fn *timeout;
85+
86+ /*
87+ * Override for hctx allocations (should probably go)
88+ */
89+ alloc_hctx_fn *alloc_hctx;
90+ free_hctx_fn *free_hctx;
91+
92+ /*
93+ * Called when the block layer side of a hardware queue has been
94+ * set up, allowing the driver to allocate/init matching structures.
95+ * Ditto for exit/teardown.
96+ */
97+ init_hctx_fn *init_hctx;
98+ exit_hctx_fn *exit_hctx;
99+};
100+
101+enum {
102+ BLK_MQ_RQ_QUEUE_OK = 0, /* queued fine */
103+ BLK_MQ_RQ_QUEUE_BUSY = 1, /* requeue IO for later */
104+ BLK_MQ_RQ_QUEUE_ERROR = 2, /* end IO with error */
105+
106+ BLK_MQ_F_SHOULD_MERGE = 1 << 0,
107+ BLK_MQ_F_SHOULD_SORT = 1 << 1,
108+ BLK_MQ_F_SHOULD_IPI = 1 << 2,
109+
110+ BLK_MQ_S_STOPPED = 1 << 0,
111+
112+ BLK_MQ_MAX_DEPTH = 2048,
113+};
114+
115+struct request_queue *blk_mq_init_queue(struct blk_mq_reg *, void *);
116+void blk_mq_free_queue(struct request_queue *);
117+int blk_mq_register_disk(struct gendisk *);
118+void blk_mq_unregister_disk(struct gendisk *);
119+void blk_mq_init_commands(struct request_queue *, void (*init)(void *data, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data);
120+
121+void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
122+
123+void blk_mq_insert_request(struct request_queue *, struct request *, bool);
124+void blk_mq_run_queues(struct request_queue *q, bool async);
125+void blk_mq_free_request(struct request *rq);
126+bool blk_mq_can_queue(struct blk_mq_hw_ctx *);
127+struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp);
128+struct request *blk_mq_alloc_reserved_request(struct request_queue *q, int rw, gfp_t gfp);
129+struct request *blk_mq_rq_from_tag(struct request_queue *q, unsigned int tag);
130+
131+struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index);
132+struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_reg *, unsigned int);
133+void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *, unsigned int);
134+
135+void blk_mq_end_io(struct request *rq, int error);
136+
137+void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
138+void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
139+void blk_mq_start_stopped_hw_queues(struct request_queue *q);
140+
141+/*
142+ * Driver command data is immediately after the request. So subtract request
143+ * size to get back to the original request.
144+ */
145+static inline struct request *blk_mq_rq_from_pdu(void *pdu)
146+{
147+ return pdu - sizeof(struct request);
148+}
149+static inline void *blk_mq_rq_to_pdu(struct request *rq)
150+{
151+ return (void *) rq + sizeof(*rq);
152+}
153+
154+static inline struct request *blk_mq_tag_to_rq(struct blk_mq_hw_ctx *hctx,
155+ unsigned int tag)
156+{
157+ return hctx->rqs[tag];
158+}
159+
160+#define queue_for_each_hw_ctx(q, hctx, i) \
161+ for ((i) = 0, hctx = (q)->queue_hw_ctx[0]; \
162+ (i) < (q)->nr_hw_queues; (i)++, hctx = (q)->queue_hw_ctx[i])
163+
164+#define queue_for_each_ctx(q, ctx, i) \
165+ for ((i) = 0, ctx = per_cpu_ptr((q)->queue_ctx, 0); \
166+ (i) < (q)->nr_queues; (i)++, ctx = per_cpu_ptr(q->queue_ctx, (i)))
167+
168+#define hctx_for_each_ctx(hctx, ctx, i) \
169+ for ((i) = 0, ctx = (hctx)->ctxs[0]; \
170+ (i) < (hctx)->nr_ctx; (i)++, ctx = (hctx)->ctxs[(i)])
171+
172+#define blk_ctx_sum(q, sum) \
173+({ \
174+ struct blk_mq_ctx *__x; \
175+ unsigned int __ret = 0, __i; \
176+ \
177+ queue_for_each_ctx((q), __x, __i) \
178+ __ret += sum; \
179+ __ret; \
180+})
181+
182+#endif
include/linux/blk_types.h+2−0
@@ -178,6 +178,7 @@ enum rq_flag_bits { · 177 unmodified lines
178178 __REQ_MIXED_MERGE, /* merge of different types, fail separately */
179179 __REQ_KERNEL, /* direct IO to kernel pages */
180180 __REQ_PM, /* runtime pm request */
181+ __REQ_END, /* last of chain of requests */
181182 __REQ_NR_BITS, /* stops here */
182183 };
183184
@@ -229,5 +230,6 @@ enum rq_flag_bits { · 45 unmodified lines
229230 #define REQ_SECURE (1ULL << __REQ_SECURE)
230231 #define REQ_KERNEL (1ULL << __REQ_KERNEL)
231232 #define REQ_PM (1ULL << __REQ_PM)
233+#define REQ_END (1ULL << __REQ_END)
232234
233235 #endif /* __LINUX_BLK_TYPES_H */
include/linux/blkdev.h+47−7
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include <linux/major.h>
99 #include <linux/genhd.h>
1010 #include <linux/list.h>
11+#include <linux/llist.h>
1112 #include <linux/timer.h>
1213 #include <linux/workqueue.h>
1314 #include <linux/pagemap.h>
@@ -94,10 +95,17 @@ enum rq_cmd_type_bits { · 80 unmodified lines
9495 * as well!
9596 */
9697 struct request {
97− struct list_head queuelist;
98− struct call_single_data csd;
98+ union {
99+ struct list_head queuelist;
100+ struct llist_node ll_list;
101+ };
102+ union {
103+ struct call_single_data csd;
104+ struct work_struct mq_flush_data;
105+ };
99106
100107 struct request_queue *q;
108+ struct blk_mq_ctx *mq_ctx;
101109
102110 u64 cmd_flags;
103111 enum rq_cmd_type_bits cmd_type;
@@ -213,6 +221,8 @@ struct request_pm_state · 109 unmodified lines
213221
214222 #include <linux/elevator.h>
215223
224+struct blk_queue_ctx;
225+
216226 typedef void (request_fn_proc) (struct request_queue *q);
217227 typedef void (make_request_fn) (struct request_queue *q, struct bio *bio);
218228 typedef int (prep_rq_fn) (struct request_queue *, struct request *);
@@ -311,6 +321,18 @@ struct request_queue { · 92 unmodified lines
311321 dma_drain_needed_fn *dma_drain_needed;
312322 lld_busy_fn *lld_busy_fn;
313323
324+ struct blk_mq_ops *mq_ops;
325+
326+ unsigned int *mq_map;
327+
328+ /* sw queues */
329+ struct blk_mq_ctx *queue_ctx;
330+ unsigned int nr_queues;
331+
332+ /* hw dispatch queues */
333+ struct blk_mq_hw_ctx **queue_hw_ctx;
334+ unsigned int nr_hw_queues;
335+
314336 /*
315337 * Dispatch queue sorting
316338 */
@@ -359,6 +381,11 @@ struct request_queue { · 42 unmodified lines
359381 */
360382 struct kobject kobj;
361383
384+ /*
385+ * mq queue kobject
386+ */
387+ struct kobject mq_kobj;
388+
362389 #ifdef CONFIG_PM_RUNTIME
363390 struct device *dev;
364391 int rpm_status;
@@ -423,7 +450,13 @@ struct request_queue { · 58 unmodified lines
423450 unsigned long flush_pending_since;
424451 struct list_head flush_queue[2];
425452 struct list_head flush_data_in_flight;
426− struct request flush_rq;
453+ union {
454+ struct request flush_rq;
455+ struct {
456+ spinlock_t mq_flush_lock;
457+ struct work_struct mq_flush_work;
458+ };
459+ };
427460
428461 struct mutex sysfs_lock;
429462
@@ -435,14 +468,14 @@ struct request_queue { · 5 unmodified lines
435468 struct bsg_class_device bsg_dev;
436469 #endif
437470
438−#ifdef CONFIG_BLK_CGROUP
439− struct list_head all_q_node;
440−#endif
441471 #ifdef CONFIG_BLK_DEV_THROTTLING
442472 /* Throttle data */
443473 struct throtl_data *td;
444474 #endif
445475 struct rcu_head rcu_head;
476+ wait_queue_head_t mq_freeze_wq;
477+ struct percpu_counter mq_usage_counter;
478+ struct list_head all_q_node;
446479 };
447480
448481 #define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */
@@ -465,6 +498,7 @@ struct request_queue { · 16 unmodified lines
465498 #define QUEUE_FLAG_SECDISCARD 17 /* supports SECDISCARD */
466499 #define QUEUE_FLAG_SAME_FORCE 18 /* force complete on same CPU */
467500 #define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */
501+#define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */
468502
469503 #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
470504 (1 << QUEUE_FLAG_STACKABLE) | \
@@ -537,6 +571,7 @@ static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) · 66 unmodified lines
537571 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
538572 #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
539573 #define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
574+#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
540575 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
541576 #define blk_queue_noxmerges(q) \
542577 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
@@ -1011,6 +1046,7 @@ static inline void blk_post_runtime_resume(struct request_queue *q, int err) {} · 468 unmodified lines
10111046 struct blk_plug {
10121047 unsigned long magic; /* detect uninitialized use-cases */
10131048 struct list_head list; /* requests */
1049+ struct list_head mq_list; /* blk-mq requests */
10141050 struct list_head cb_list; /* md requires an unplug callback */
10151051 };
10161052 #define BLK_MAX_REQUEST_COUNT 16
@@ -1048,7 +1084,10 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) · 31 unmodified lines
10481084 {
10491085 struct blk_plug *plug = tsk->plug;
10501086
1051− return plug && (!list_empty(&plug->list) || !list_empty(&plug->cb_list));
1087+ return plug &&
1088+ (!list_empty(&plug->list) ||
1089+ !list_empty(&plug->mq_list) ||
1090+ !list_empty(&plug->cb_list));
10521091 }
10531092
10541093 /*
@@ -1323,6 +1362,7 @@ static inline void put_dev_sector(Sector p) · 268 unmodified lines
13231362
13241363 struct work_struct;
13251364 int kblockd_schedule_work(struct request_queue *q, struct work_struct *work);
1365+int kblockd_schedule_delayed_work(struct request_queue *q, struct delayed_work *dwork, unsigned long delay);
13261366
13271367 #ifdef CONFIG_BLK_CGROUP
13281368 /*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t