If we want to support async IO polling, then we have to allow finding completions that aren't just for the one we are looking for. Always pass in -1 to the mq_ops->poll() helper, and have that return how many events were found in this poll loop.
Unified · Split
| @@ -1273,10 +1273,19 @@ blk_qc_t submit_bio(struct bio *bio) · 1272 unmodified lines | ||
| 1273 | 1273 | } |
| 1274 | 1274 | EXPORT_SYMBOL(submit_bio); |
| 1275 | 1275 | |
| 1276 | −bool blk_poll(struct request_queue *q, blk_qc_t cookie) | |
| 1276 | +/** | |
| 1277 | + * blk_poll - poll for IO completions | |
| 1278 | + * @q: the queue | |
| 1279 | + * @cookie: cookie passed back at IO submission time | |
| 1280 | + * | |
| 1281 | + * Description: | |
| 1282 | + * Poll for completions on the passed in queue. Returns number of | |
| 1283 | + * completed entries found. | |
| 1284 | + */ | |
| 1285 | +int blk_poll(struct request_queue *q, blk_qc_t cookie) | |
| 1277 | 1286 | { |
| 1278 | 1287 | if (!q->poll_fn || !blk_qc_t_valid(cookie)) |
| 1279 | − return false; | |
| 1288 | + return 0; | |
| 1280 | 1289 | |
| 1281 | 1290 | if (current->plug) |
| 1282 | 1291 | blk_flush_plug_list(current->plug, false); |
| @@ -3285,15 +3285,12 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, · 3284 unmodified lines | ||
| 3285 | 3285 | return false; |
| 3286 | 3286 | |
| 3287 | 3287 | /* |
| 3288 | − * poll_nsec can be: | |
| 3288 | + * If we get here, hybrid polling is enabled. Hence poll_nsec can be: | |
| 3289 | 3289 | * |
| 3290 | − * -1: don't ever hybrid sleep | |
| 3291 | 3290 | * 0: use half of prev avg |
| 3292 | 3291 | * >0: use this specific value |
| 3293 | 3292 | */ |
| 3294 | − if (q->poll_nsec == -1) | |
| 3295 | − return false; | |
| 3296 | − else if (q->poll_nsec > 0) | |
| 3293 | + if (q->poll_nsec > 0) | |
| 3297 | 3294 | nsecs = q->poll_nsec; |
| 3298 | 3295 | else |
| 3299 | 3296 | nsecs = blk_mq_poll_nsecs(q, hctx, rq); |
| @@ -3330,11 +3327,41 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, · 30 unmodified lines | ||
| 3330 | 3327 | return true; |
| 3331 | 3328 | } |
| 3332 | 3329 | |
| 3333 | −static int __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) | |
| 3330 | +static bool blk_mq_poll_hybrid(struct request_queue *q, | |
| 3331 | + struct blk_mq_hw_ctx *hctx, blk_qc_t cookie) | |
| 3334 | 3332 | { |
| 3335 | − struct request_queue *q = hctx->queue; | |
| 3333 | + struct request *rq; | |
| 3334 | + | |
| 3335 | + if (q->poll_nsec == -1) | |
| 3336 | + return false; | |
| 3337 | + | |
| 3338 | + if (!blk_qc_t_is_internal(cookie)) | |
| 3339 | + rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); | |
| 3340 | + else { | |
| 3341 | + rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); | |
| 3342 | + /* | |
| 3343 | + * With scheduling, if the request has completed, we'll | |
| 3344 | + * get a NULL return here, as we clear the sched tag when | |
| 3345 | + * that happens. The request still remains valid, like always, | |
| 3346 | + * so we should be safe with just the NULL check. | |
| 3347 | + */ | |
| 3348 | + if (!rq) | |
| 3349 | + return false; | |
| 3350 | + } | |
| 3351 | + | |
| 3352 | + return blk_mq_poll_hybrid_sleep(q, hctx, rq); | |
| 3353 | +} | |
| 3354 | + | |
| 3355 | +static int blk_mq_poll(struct request_queue *q, blk_qc_t cookie) | |
| 3356 | +{ | |
| 3357 | + struct blk_mq_hw_ctx *hctx; | |
| 3336 | 3358 | long state; |
| 3337 | 3359 | |
| 3360 | + if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) | |
| 3361 | + return 0; | |
| 3362 | + | |
| 3363 | + hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; | |
| 3364 | + | |
| 3338 | 3365 | /* |
| 3339 | 3366 | * If we sleep, have the caller restart the poll loop to reset |
| 3340 | 3367 | * the state. Like for the other success return cases, the |
| @@ -3342,7 +3369,7 @@ static int __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) · 1 unmodified line | ||
| 3342 | 3369 | * the IO isn't complete, we'll get called again and will go |
| 3343 | 3370 | * straight to the busy poll loop. |
| 3344 | 3371 | */ |
| 3345 | − if (blk_mq_poll_hybrid_sleep(q, hctx, rq)) | |
| 3372 | + if (blk_mq_poll_hybrid(q, hctx, cookie)) | |
| 3346 | 3373 | return 1; |
| 3347 | 3374 | |
| 3348 | 3375 | hctx->poll_considered++; |
| @@ -3353,7 +3380,7 @@ static int __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) · 4 unmodified lines | ||
| 3353 | 3380 | |
| 3354 | 3381 | hctx->poll_invoked++; |
| 3355 | 3382 | |
| 3356 | − ret = q->mq_ops->poll(hctx, rq->tag); | |
| 3383 | + ret = q->mq_ops->poll(hctx, -1U); | |
| 3357 | 3384 | if (ret > 0) { |
| 3358 | 3385 | hctx->poll_success++; |
| 3359 | 3386 | __set_current_state(TASK_RUNNING); |
| @@ -3374,32 +3401,6 @@ static int __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) · 14 unmodified lines | ||
| 3374 | 3401 | return 0; |
| 3375 | 3402 | } |
| 3376 | 3403 | |
| 3377 | −static int blk_mq_poll(struct request_queue *q, blk_qc_t cookie) | |
| 3378 | −{ | |
| 3379 | − struct blk_mq_hw_ctx *hctx; | |
| 3380 | − struct request *rq; | |
| 3381 | − | |
| 3382 | − if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) | |
| 3383 | − return 0; | |
| 3384 | − | |
| 3385 | − hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; | |
| 3386 | − if (!blk_qc_t_is_internal(cookie)) | |
| 3387 | − rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); | |
| 3388 | − else { | |
| 3389 | − rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); | |
| 3390 | − /* | |
| 3391 | − * With scheduling, if the request has completed, we'll | |
| 3392 | − * get a NULL return here, as we clear the sched tag when | |
| 3393 | − * that happens. The request still remains valid, like always, | |
| 3394 | − * so we should be safe with just the NULL check. | |
| 3395 | − */ | |
| 3396 | − if (!rq) | |
| 3397 | − return 0; | |
| 3398 | − } | |
| 3399 | − | |
| 3400 | − return __blk_mq_poll(hctx, rq); | |
| 3401 | −} | |
| 3402 | − | |
| 3403 | 3404 | unsigned int blk_mq_rq_cpu(struct request *rq) |
| 3404 | 3405 | { |
| 3405 | 3406 | return rq->mq_ctx->cpu; |
| @@ -1012,15 +1012,15 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq) · 1011 unmodified lines | ||
| 1012 | 1012 | } |
| 1013 | 1013 | } |
| 1014 | 1014 | |
| 1015 | −static inline bool nvme_process_cq(struct nvme_queue *nvmeq, u16 *start, | |
| 1016 | − u16 *end, int tag) | |
| 1015 | +static inline int nvme_process_cq(struct nvme_queue *nvmeq, u16 *start, | |
| 1016 | + u16 *end, unsigned int tag) | |
| 1017 | 1017 | { |
| 1018 | − bool found = false; | |
| 1018 | + int found = 0; | |
| 1019 | 1019 | |
| 1020 | 1020 | *start = nvmeq->cq_head; |
| 1021 | − while (!found && nvme_cqe_pending(nvmeq)) { | |
| 1022 | − if (nvmeq->cqes[nvmeq->cq_head].command_id == tag) | |
| 1023 | − found = true; | |
| 1021 | + while (nvme_cqe_pending(nvmeq)) { | |
| 1022 | + if (tag == -1U || nvmeq->cqes[nvmeq->cq_head].command_id == tag) | |
| 1023 | + found++; | |
| 1024 | 1024 | nvme_update_cq_head(nvmeq); |
| 1025 | 1025 | } |
| 1026 | 1026 | *end = nvmeq->cq_head; |
| @@ -1062,7 +1062,7 @@ static irqreturn_t nvme_irq_check(int irq, void *data) · 35 unmodified lines | ||
| 1062 | 1062 | static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag) |
| 1063 | 1063 | { |
| 1064 | 1064 | u16 start, end; |
| 1065 | − bool found; | |
| 1065 | + int found; | |
| 1066 | 1066 | |
| 1067 | 1067 | if (!nvme_cqe_pending(nvmeq)) |
| 1068 | 1068 | return 0; |
| @@ -1409,12 +1409,11 @@ static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg) · 1408 unmodified lines | ||
| 1409 | 1409 | WARN_ON_ONCE(ret); |
| 1410 | 1410 | } |
| 1411 | 1411 | |
| 1412 | −static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue, | |
| 1413 | − struct nvme_completion *cqe, struct ib_wc *wc, int tag) | |
| 1412 | +static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue, | |
| 1413 | + struct nvme_completion *cqe, struct ib_wc *wc) | |
| 1414 | 1414 | { |
| 1415 | 1415 | struct request *rq; |
| 1416 | 1416 | struct nvme_rdma_request *req; |
| 1417 | − int ret = 0; | |
| 1418 | 1417 | |
| 1419 | 1418 | rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id); |
| 1420 | 1419 | if (!rq) { |
| @@ -1422,7 +1421,7 @@ static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue, · 1 unmodified line | ||
| 1422 | 1421 | "tag 0x%x on QP %#x not found\n", |
| 1423 | 1422 | cqe->command_id, queue->qp->qp_num); |
| 1424 | 1423 | nvme_rdma_error_recovery(queue->ctrl); |
| 1425 | − return ret; | |
| 1424 | + return; | |
| 1426 | 1425 | } |
| 1427 | 1426 | req = blk_mq_rq_to_pdu(rq); |
| 1428 | 1427 | |
| @@ -1437,6 +1436,8 @@ static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue, · 8 unmodified lines | ||
| 1437 | 1436 | nvme_rdma_error_recovery(queue->ctrl); |
| 1438 | 1437 | } |
| 1439 | 1438 | } else if (req->mr) { |
| 1439 | + int ret; | |
| 1440 | + | |
| 1440 | 1441 | ret = nvme_rdma_inv_rkey(queue, req); |
| 1441 | 1442 | if (unlikely(ret < 0)) { |
| 1442 | 1443 | dev_err(queue->ctrl->ctrl.device, |
| @@ -1445,19 +1446,14 @@ static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue, · 2 unmodified lines | ||
| 1445 | 1446 | nvme_rdma_error_recovery(queue->ctrl); |
| 1446 | 1447 | } |
| 1447 | 1448 | /* the local invalidation completion will end the request */ |
| 1448 | − return 0; | |
| 1449 | + return; | |
| 1449 | 1450 | } |
| 1450 | 1451 | |
| 1451 | − if (refcount_dec_and_test(&req->ref)) { | |
| 1452 | − if (rq->tag == tag) | |
| 1453 | − ret = 1; | |
| 1452 | + if (refcount_dec_and_test(&req->ref)) | |
| 1454 | 1453 | nvme_end_request(rq, req->status, req->result); |
| 1455 | − } | |
| 1456 | − | |
| 1457 | − return ret; | |
| 1458 | 1454 | } |
| 1459 | 1455 | |
| 1460 | −static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag) | |
| 1456 | +static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc) | |
| 1461 | 1457 | { |
| 1462 | 1458 | struct nvme_rdma_qe *qe = |
| 1463 | 1459 | container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe); |
| @@ -1465,11 +1461,10 @@ static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag) · 1 unmodified line | ||
| 1465 | 1461 | struct ib_device *ibdev = queue->device->dev; |
| 1466 | 1462 | struct nvme_completion *cqe = qe->data; |
| 1467 | 1463 | const size_t len = sizeof(struct nvme_completion); |
| 1468 | − int ret = 0; | |
| 1469 | 1464 | |
| 1470 | 1465 | if (unlikely(wc->status != IB_WC_SUCCESS)) { |
| 1471 | 1466 | nvme_rdma_wr_error(cq, wc, "RECV"); |
| 1472 | − return 0; | |
| 1467 | + return; | |
| 1473 | 1468 | } |
| 1474 | 1469 | |
| 1475 | 1470 | ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE); |
| @@ -1484,16 +1479,10 @@ static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag) · 8 unmodified lines | ||
| 1484 | 1479 | nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, |
| 1485 | 1480 | &cqe->result); |
| 1486 | 1481 | else |
| 1487 | − ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag); | |
| 1482 | + nvme_rdma_process_nvme_rsp(queue, cqe, wc); | |
| 1488 | 1483 | ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE); |
| 1489 | 1484 | |
| 1490 | 1485 | nvme_rdma_post_recv(queue, qe); |
| 1491 | − return ret; | |
| 1492 | −} | |
| 1493 | − | |
| 1494 | −static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc) | |
| 1495 | −{ | |
| 1496 | − __nvme_rdma_recv_done(cq, wc, -1); | |
| 1497 | 1486 | } |
| 1498 | 1487 | |
| 1499 | 1488 | static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue) |
| @@ -1758,10 +1747,12 @@ static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag) · 258 unmodified lines | ||
| 1758 | 1747 | struct ib_cqe *cqe = wc.wr_cqe; |
| 1759 | 1748 | |
| 1760 | 1749 | if (cqe) { |
| 1761 | − if (cqe->done == nvme_rdma_recv_done) | |
| 1762 | − found |= __nvme_rdma_recv_done(cq, &wc, tag); | |
| 1763 | − else | |
| 1750 | + if (cqe->done == nvme_rdma_recv_done) { | |
| 1751 | + nvme_rdma_recv_done(cq, &wc); | |
| 1752 | + found++; | |
| 1753 | + } else { | |
| 1764 | 1754 | cqe->done(cq, &wc); |
| 1755 | + } | |
| 1765 | 1756 | } |
| 1766 | 1757 | } |
| 1767 | 1758 | |
| @@ -867,7 +867,7 @@ extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *, · 866 unmodified lines | ||
| 867 | 867 | int blk_status_to_errno(blk_status_t status); |
| 868 | 868 | blk_status_t errno_to_blk_status(int errno); |
| 869 | 869 | |
| 870 | −bool blk_poll(struct request_queue *q, blk_qc_t cookie); | |
| 870 | +int blk_poll(struct request_queue *q, blk_qc_t cookie); | |
| 871 | 871 | |
| 872 | 872 | static inline struct request_queue *bdev_get_queue(struct block_device *bdev) |
| 873 | 873 | { |