History

nvme: add support for mq_ops->queue_rqs()

This enables the block layer to send us a full plug list of requests that need submitting. The block layer guarantees that they all belong to the same queue, but we do have to check the hardware queue mapping for each request. If errors are encountered, leave them in the passed in list. Then the block layer will handle them individually. This is good for about a 4% improvement in peak performance, taking us from 9.6M to 10M IOPS/core.

Jens Axboe committed 4 years ago

Commit d62cbcf1 parent 62451a21 file · +59 −0Browse files

Unified · Split

drivers/nvme/host/pci.c+59−0
@@ -969,6 +969,64 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, · 968 unmodified lines
969969 return BLK_STS_OK;
970970 }
971971
972+static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist)
973+{
974+ spin_lock(&nvmeq->sq_lock);
975+ while (!rq_list_empty(*rqlist)) {
976+ struct request *req = rq_list_pop(rqlist);
977+ struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
978+
979+ nvme_sq_copy_cmd(nvmeq, &iod->cmd);
980+ }
981+ nvme_write_sq_db(nvmeq, true);
982+ spin_unlock(&nvmeq->sq_lock);
983+}
984+
985+static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req)
986+{
987+ /*
988+ * We should not need to do this, but we're still using this to
989+ * ensure we can drain requests on a dying queue.
990+ */
991+ if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
992+ return false;
993+ if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
994+ return false;
995+
996+ req->mq_hctx->tags->rqs[req->tag] = req;
997+ return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
998+}
999+
1000+static void nvme_queue_rqs(struct request **rqlist)
1001+{
1002+ struct request *req = rq_list_peek(rqlist), *prev = NULL;
1003+ struct request *requeue_list = NULL;
1004+
1005+ do {
1006+ struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1007+
1008+ if (!nvme_prep_rq_batch(nvmeq, req)) {
1009+ /* detach 'req' and add to remainder list */
1010+ if (prev)
1011+ prev->rq_next = req->rq_next;
1012+ rq_list_add(&requeue_list, req);
1013+ } else {
1014+ prev = req;
1015+ }
1016+
1017+ req = rq_list_next(req);
1018+ if (!req || (prev && req->mq_hctx != prev->mq_hctx)) {
1019+ /* detach rest of list, and submit */
1020+ if (prev)
1021+ prev->rq_next = NULL;
1022+ nvme_submit_cmds(nvmeq, rqlist);
1023+ *rqlist = req;
1024+ }
1025+ } while (req);
1026+
1027+ *rqlist = requeue_list;
1028+}
1029+
9721030 static __always_inline void nvme_pci_unmap_rq(struct request *req)
9731031 {
9741032 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
@@ -1670,6 +1728,7 @@ static const struct blk_mq_ops nvme_mq_admin_ops = { · 695 unmodified lines
16701728
16711729 static const struct blk_mq_ops nvme_mq_ops = {
16721730 .queue_rq = nvme_queue_rq,
1731+ .queue_rqs = nvme_queue_rqs,
16731732 .complete = nvme_pci_complete_rq,
16741733 .commit_rqs = nvme_commit_rqs,
16751734 .init_hctx = nvme_init_hctx,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t