History

nvme: add support for mq_ops->queue_rqs()

This enables the block layer to send us a full plug list of requests that need submitting. The block layer guarantees that they all belong to the same queue, but we do have to check the hardware queue mapping for each request. If errors are encountered, leave them in the passed in list. Then the block layer will handle them individually. This is good for about a 4% improvement in peak performance, taking us from 9.6M to 10M IOPS/core.

Jens Axboe committed 4 years ago

Commit d62cbcf1 parent 62451a21 file · +59 −0Browse files

Unified · Split

drivers/nvme/host/pci.c+59−0
@@ -969,6 +969,64 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, · 968 unmodified lines
969 return BLK_STS_OK;969 return BLK_STS_OK;
970}970}
971 971
972static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist)
973{
974 spin_lock(&nvmeq->sq_lock);
975 while (!rq_list_empty(*rqlist)) {
976 struct request *req = rq_list_pop(rqlist);
977 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
978
979 nvme_sq_copy_cmd(nvmeq, &iod->cmd);
980 }
981 nvme_write_sq_db(nvmeq, true);
982 spin_unlock(&nvmeq->sq_lock);
983}
984
985static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req)
986{
987 /*
988 * We should not need to do this, but we're still using this to
989 * ensure we can drain requests on a dying queue.
990 */
991 if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
992 return false;
993 if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
994 return false;
995
996 req->mq_hctx->tags->rqs[req->tag] = req;
997 return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
998}
999
1000static void nvme_queue_rqs(struct request **rqlist)
1001{
1002 struct request *req = rq_list_peek(rqlist), *prev = NULL;
1003 struct request *requeue_list = NULL;
1004
1005 do {
1006 struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1007
1008 if (!nvme_prep_rq_batch(nvmeq, req)) {
1009 /* detach 'req' and add to remainder list */
1010 if (prev)
1011 prev->rq_next = req->rq_next;
1012 rq_list_add(&requeue_list, req);
1013 } else {
1014 prev = req;
1015 }
1016
1017 req = rq_list_next(req);
1018 if (!req || (prev && req->mq_hctx != prev->mq_hctx)) {
1019 /* detach rest of list, and submit */
1020 if (prev)
1021 prev->rq_next = NULL;
1022 nvme_submit_cmds(nvmeq, rqlist);
1023 *rqlist = req;
1024 }
1025 } while (req);
1026
1027 *rqlist = requeue_list;
1028}
1029
972static __always_inline void nvme_pci_unmap_rq(struct request *req)1030static __always_inline void nvme_pci_unmap_rq(struct request *req)
973{1031{
974 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);1032 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
@@ -1670,6 +1728,7 @@ static const struct blk_mq_ops nvme_mq_admin_ops = { · 695 unmodified lines
1670 1728
1671static const struct blk_mq_ops nvme_mq_ops = {1729static const struct blk_mq_ops nvme_mq_ops = {
1672 .queue_rq = nvme_queue_rq,1730 .queue_rq = nvme_queue_rq,
1731 .queue_rqs = nvme_queue_rqs,
1673 .complete = nvme_pci_complete_rq,1732 .complete = nvme_pci_complete_rq,
1674 .commit_rqs = nvme_commit_rqs,1733 .commit_rqs = nvme_commit_rqs,
1675 .init_hctx = nvme_init_hctx,1734 .init_hctx = nvme_init_hctx,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t