blk-mq: when polling for IO, look for any completion

If we want to support async IO polling, then we have to allow finding completions that aren't just for the one we are looking for. Always pass in -1 to the mq_ops->poll() helper, and have that return how many events were found in this poll loop.

Jens Axboe committed 7 years ago

Commit 1052b8a1 parent 1db49095 files · +70 −69Browse files

Unified · Split

5 files

block/blk-core.c +11 −2
block/blk-mq.c +36 −35
drivers/nvme/host/pci.c +7 −7
drivers/nvme/host/rdma.c +15 −24
include/linux/blkdev.h +1 −1

block/blk-core.c+11−2

 }
 EXPORT_SYMBOL(submit_bio);
-bool blk_poll(struct request_queue *q, blk_qc_t cookie)
+/**
+ * blk_poll - poll for IO completions
+ * @q:  the queue
+ * @cookie: cookie passed back at IO submission time
+ *
+ * Description:
+ *    Poll for completions on the passed in queue. Returns number of
+ *    completed entries found.
+ */
+int blk_poll(struct request_queue *q, blk_qc_t cookie)
 {
 	if (!q->poll_fn || !blk_qc_t_valid(cookie))
-		return false;
+		return 0;
 	if (current->plug)
 		blk_flush_plug_list(current->plug, false);

block/blk-mq.c+36−35

 		return false;
 	/*
-	 * poll_nsec can be:
+	 * If we get here, hybrid polling is enabled. Hence poll_nsec can be:
 	 *
-	 * -1:	don't ever hybrid sleep
 	 *  0:	use half of prev avg
 	 * >0:	use this specific value
 	 */
-	if (q->poll_nsec == -1)
+	if (q->poll_nsec > 0)
-		return false;
-	else if (q->poll_nsec > 0)
 		nsecs = q->poll_nsec;
 	else
 		nsecs = blk_mq_poll_nsecs(q, hctx, rq);
 	return true;
 }
-static int __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
+static bool blk_mq_poll_hybrid(struct request_queue *q,
+			       struct blk_mq_hw_ctx *hctx, blk_qc_t cookie)
 {
-	struct request_queue *q = hctx->queue;
+	struct request *rq;
+	if (q->poll_nsec == -1)
+		return false;
+	if (!blk_qc_t_is_internal(cookie))
+		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
+	else {
+		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
+		/*
+		 * With scheduling, if the request has completed, we'll
+		 * get a NULL return here, as we clear the sched tag when
+		 * that happens. The request still remains valid, like always,
+		 * so we should be safe with just the NULL check.
+		 */
+		if (!rq)
+			return false;
+	}
+	return blk_mq_poll_hybrid_sleep(q, hctx, rq);
+}
+static int blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
+{
+	struct blk_mq_hw_ctx *hctx;
 	long state;
+	if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
+		return 0;
+	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
 	/*
 	 * If we sleep, have the caller restart the poll loop to reset
 	 * the state. Like for the other success return cases, the
 	 * the IO isn't complete, we'll get called again and will go
 	 * straight to the busy poll loop.
 	 */
-	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
+	if (blk_mq_poll_hybrid(q, hctx, cookie))
 		return 1;
 	hctx->poll_considered++;
 		hctx->poll_invoked++;
-		ret = q->mq_ops->poll(hctx, rq->tag);
+		ret = q->mq_ops->poll(hctx, -1U);
 		if (ret > 0) {
 			hctx->poll_success++;
 			__set_current_state(TASK_RUNNING);
 	return 0;
 }
-static int blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
-{
-	struct blk_mq_hw_ctx *hctx;
-	struct request *rq;
-	if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
-		return 0;
-	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
-	if (!blk_qc_t_is_internal(cookie))
-		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
-	else {
-		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
-		/*
-		 * With scheduling, if the request has completed, we'll
-		 * get a NULL return here, as we clear the sched tag when
-		 * that happens. The request still remains valid, like always,
-		 * so we should be safe with just the NULL check.
-		 */
-		if (!rq)
-			return 0;
-	}
-	return __blk_mq_poll(hctx, rq);
-}
 unsigned int blk_mq_rq_cpu(struct request *rq)
 {
 	return rq->mq_ctx->cpu;

drivers/nvme/host/pci.c+7−7

@@ -1012,15 +1012,15 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq) · 1011 unmodified lines
1012	}	1012	}
1013	}	1013	}
1014		1014
1015	static inline bool nvme_process_cq(struct nvme_queue nvmeq, u16 start,	1015	static inline int nvme_process_cq(struct nvme_queue nvmeq, u16 start,
1016	u16 *end, int tag)	1016	u16 *end, unsigned int tag)
1017	{	1017	{
1018	bool found = false;	1018	int found = 0;
1019		1019
1020	*start = nvmeq->cq_head;	1020	*start = nvmeq->cq_head;
1021	while (!found && nvme_cqe_pending(nvmeq)) {	1021	while (nvme_cqe_pending(nvmeq)) {
1022	if (nvmeq->cqes[nvmeq->cq_head].command_id == tag)	1022	if (tag == -1U \|\| nvmeq->cqes[nvmeq->cq_head].command_id == tag)
1023	found = true;	1023	found++;
1024	nvme_update_cq_head(nvmeq);	1024	nvme_update_cq_head(nvmeq);
1025	}	1025	}
1026	*end = nvmeq->cq_head;	1026	*end = nvmeq->cq_head;
@@ -1062,7 +1062,7 @@ static irqreturn_t nvme_irq_check(int irq, void *data) · 35 unmodified lines
1062	static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)	1062	static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
1063	{	1063	{
1064	u16 start, end;	1064	u16 start, end;
1065	bool found;	1065	int found;
1066		1066
1067	if (!nvme_cqe_pending(nvmeq))	1067	if (!nvme_cqe_pending(nvmeq))
1068	return 0;	1068	return 0;

drivers/nvme/host/rdma.c+15−24

 	WARN_ON_ONCE(ret);
 }
-static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
+static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
-		struct nvme_completion *cqe, struct ib_wc *wc, int tag)
+		struct nvme_completion *cqe, struct ib_wc *wc)
 {
 	struct request *rq;
 	struct nvme_rdma_request *req;
-	int ret = 0;
 	rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
 	if (!rq) {
 			"tag 0x%x on QP %#x not found\n",
 			cqe->command_id, queue->qp->qp_num);
 		nvme_rdma_error_recovery(queue->ctrl);
-		return ret;
+		return;
 	}
 	req = blk_mq_rq_to_pdu(rq);
 			nvme_rdma_error_recovery(queue->ctrl);
 		}
 	} else if (req->mr) {
+		int ret;
 		ret = nvme_rdma_inv_rkey(queue, req);
 		if (unlikely(ret < 0)) {
 			dev_err(queue->ctrl->ctrl.device,
 			nvme_rdma_error_recovery(queue->ctrl);
 		}
 		/* the local invalidation completion will end the request */
-		return 0;
+		return;
 	}
-	if (refcount_dec_and_test(&req->ref)) {
+	if (refcount_dec_and_test(&req->ref))
-		if (rq->tag == tag)
-			ret = 1;
 		nvme_end_request(rq, req->status, req->result);
-	}
-	return ret;
 }
-static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag)
+static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 {
 	struct nvme_rdma_qe *qe =
 		container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
 	struct ib_device *ibdev = queue->device->dev;
 	struct nvme_completion *cqe = qe->data;
 	const size_t len = sizeof(struct nvme_completion);
-	int ret = 0;
 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
 		nvme_rdma_wr_error(cq, wc, "RECV");
-		return 0;
+		return;
 	}
 	ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
 		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
 				&cqe->result);
 	else
-		ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag);
+		nvme_rdma_process_nvme_rsp(queue, cqe, wc);
 	ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
 	nvme_rdma_post_recv(queue, qe);
-	return ret;
-}
-static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
-{
-	__nvme_rdma_recv_done(cq, wc, -1);
 }
 static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
 		struct ib_cqe *cqe = wc.wr_cqe;
 		if (cqe) {
-			if (cqe->done == nvme_rdma_recv_done)
+			if (cqe->done == nvme_rdma_recv_done) {
-				found |= __nvme_rdma_recv_done(cq, &wc, tag);
+				nvme_rdma_recv_done(cq, &wc);
-			else
+				found++;
+			} else {
 				cqe->done(cq, &wc);
+			}
 		}
 	}

include/linux/blkdev.h+1−1

@@ -867,7 +867,7 @@ extern void blk_execute_rq_nowait(struct request_queue , struct gendisk , · 866 unmodified lines
867	int blk_status_to_errno(blk_status_t status);	867	int blk_status_to_errno(blk_status_t status);
868	blk_status_t errno_to_blk_status(int errno);	868	blk_status_t errno_to_blk_status(int errno);
869		869
870	bool blk_poll(struct request_queue *q, blk_qc_t cookie);	870	int blk_poll(struct request_queue *q, blk_qc_t cookie);
871		871
872	static inline struct request_queue bdev_get_queue(struct block_device bdev)	872	static inline struct request_queue bdev_get_queue(struct block_device bdev)
873	{	873	{

blk-mq: when polling for IO, look for any completion

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code