Replace the semi-open coded request list helpers with a proper rq_list type that mirrors the bio_list and has head and tail pointers. Besides better type safety this actually allows to insert at the tail of the list, which will be useful soon.
Unified · Split
| @@ -1120,8 +1120,8 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios) · 1119 unmodified lines | |||
| 1120 | return; | 1120 | return; |
| 1121 | 1121 | ||
| 1122 | plug->cur_ktime = 0; | 1122 | plug->cur_ktime = 0; |
| 1123 | plug->mq_list = NULL; | 1123 | rq_list_init(&plug->mq_list); |
| 1124 | plug->cached_rq = NULL; | 1124 | rq_list_init(&plug->cached_rqs); |
| 1125 | plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT); | 1125 | plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT); |
| 1126 | plug->rq_count = 0; | 1126 | plug->rq_count = 0; |
| 1127 | plug->multiple_queues = false; | 1127 | plug->multiple_queues = false; |
| @@ -1217,7 +1217,7 @@ void __blk_flush_plug(struct blk_plug *plug, bool from_schedule) · 89 unmodified lines | |||
| 1217 | * queue for cached requests, we don't want a blocked task holding | 1217 | * queue for cached requests, we don't want a blocked task holding |
| 1218 | * up a queue freeze/quiesce event. | 1218 | * up a queue freeze/quiesce event. |
| 1219 | */ | 1219 | */ |
| 1220 | if (unlikely(!rq_list_empty(plug->cached_rq))) | 1220 | if (unlikely(!rq_list_empty(&plug->cached_rqs))) |
| 1221 | blk_mq_free_plug_rqs(plug); | 1221 | blk_mq_free_plug_rqs(plug); |
| 1222 | 1222 | ||
| 1223 | plug->cur_ktime = 0; | 1223 | plug->cur_ktime = 0; |
| @@ -1179,7 +1179,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, · 1178 unmodified lines | |||
| 1179 | struct blk_plug *plug = current->plug; | 1179 | struct blk_plug *plug = current->plug; |
| 1180 | struct request *rq; | 1180 | struct request *rq; |
| 1181 | 1181 | ||
| 1182 | if (!plug || rq_list_empty(plug->mq_list)) | 1182 | if (!plug || rq_list_empty(&plug->mq_list)) |
| 1183 | return false; | 1183 | return false; |
| 1184 | 1184 | ||
| 1185 | rq_list_for_each(&plug->mq_list, rq) { | 1185 | rq_list_for_each(&plug->mq_list, rq) { |
| @@ -478,7 +478,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) · 477 unmodified lines | |||
| 478 | prefetch(tags->static_rqs[tag]); | 478 | prefetch(tags->static_rqs[tag]); |
| 479 | tag_mask &= ~(1UL << i); | 479 | tag_mask &= ~(1UL << i); |
| 480 | rq = blk_mq_rq_ctx_init(data, tags, tag); | 480 | rq = blk_mq_rq_ctx_init(data, tags, tag); |
| 481 | rq_list_add(data->cached_rq, rq); | 481 | rq_list_add_head(data->cached_rqs, rq); |
| 482 | nr++; | 482 | nr++; |
| 483 | } | 483 | } |
| 484 | if (!(data->rq_flags & RQF_SCHED_TAGS)) | 484 | if (!(data->rq_flags & RQF_SCHED_TAGS)) |
| @@ -487,7 +487,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) · 2 unmodified lines | |||
| 487 | percpu_ref_get_many(&data->q->q_usage_counter, nr - 1); | 487 | percpu_ref_get_many(&data->q->q_usage_counter, nr - 1); |
| 488 | data->nr_tags -= nr; | 488 | data->nr_tags -= nr; |
| 489 | 489 | ||
| 490 | return rq_list_pop(data->cached_rq); | 490 | return rq_list_pop(data->cached_rqs); |
| 491 | } | 491 | } |
| 492 | 492 | ||
| 493 | static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) | 493 | static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) |
| @@ -584,7 +584,7 @@ static struct request *blk_mq_rq_cache_fill(struct request_queue *q, · 90 unmodified lines | |||
| 584 | .flags = flags, | 584 | .flags = flags, |
| 585 | .cmd_flags = opf, | 585 | .cmd_flags = opf, |
| 586 | .nr_tags = plug->nr_ios, | 586 | .nr_tags = plug->nr_ios, |
| 587 | .cached_rq = &plug->cached_rq, | 587 | .cached_rqs = &plug->cached_rqs, |
| 588 | }; | 588 | }; |
| 589 | struct request *rq; | 589 | struct request *rq; |
| 590 | 590 | ||
| @@ -609,14 +609,14 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, · 18 unmodified lines | |||
| 609 | if (!plug) | 609 | if (!plug) |
| 610 | return NULL; | 610 | return NULL; |
| 611 | 611 | ||
| 612 | if (rq_list_empty(plug->cached_rq)) { | 612 | if (rq_list_empty(&plug->cached_rqs)) { |
| 613 | if (plug->nr_ios == 1) | 613 | if (plug->nr_ios == 1) |
| 614 | return NULL; | 614 | return NULL; |
| 615 | rq = blk_mq_rq_cache_fill(q, plug, opf, flags); | 615 | rq = blk_mq_rq_cache_fill(q, plug, opf, flags); |
| 616 | if (!rq) | 616 | if (!rq) |
| 617 | return NULL; | 617 | return NULL; |
| 618 | } else { | 618 | } else { |
| 619 | rq = rq_list_peek(&plug->cached_rq); | 619 | rq = rq_list_peek(&plug->cached_rqs); |
| 620 | if (!rq || rq->q != q) | 620 | if (!rq || rq->q != q) |
| 621 | return NULL; | 621 | return NULL; |
| 622 | 622 | ||
| @@ -625,7 +625,7 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, · 2 unmodified lines | |||
| 625 | if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) | 625 | if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) |
| 626 | return NULL; | 626 | return NULL; |
| 627 | 627 | ||
| 628 | plug->cached_rq = rq_list_next(rq); | 628 | rq_list_pop(&plug->cached_rqs); |
| 629 | blk_mq_rq_time_init(rq, blk_time_get_ns()); | 629 | blk_mq_rq_time_init(rq, blk_time_get_ns()); |
| 630 | } | 630 | } |
| 631 | 631 | ||
| @@ -802,7 +802,7 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug) · 170 unmodified lines | |||
| 802 | { | 802 | { |
| 803 | struct request *rq; | 803 | struct request *rq; |
| 804 | 804 | ||
| 805 | while ((rq = rq_list_pop(&plug->cached_rq)) != NULL) | 805 | while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL) |
| 806 | blk_mq_free_request(rq); | 806 | blk_mq_free_request(rq); |
| 807 | } | 807 | } |
| 808 | 808 | ||
| @@ -1392,8 +1392,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) · 583 unmodified lines | |||
| 1392 | */ | 1392 | */ |
| 1393 | if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS)) | 1393 | if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS)) |
| 1394 | plug->has_elevator = true; | 1394 | plug->has_elevator = true; |
| 1395 | rq->rq_next = NULL; | 1395 | rq_list_add_head(&plug->mq_list, rq); |
| 1396 | rq_list_add(&plug->mq_list, rq); | ||
| 1397 | plug->rq_count++; | 1396 | plug->rq_count++; |
| 1398 | } | 1397 | } |
| 1399 | 1398 | ||
| @@ -2785,7 +2784,7 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug) · 1385 unmodified lines | |||
| 2785 | blk_status_t ret = BLK_STS_OK; | 2784 | blk_status_t ret = BLK_STS_OK; |
| 2786 | 2785 | ||
| 2787 | while ((rq = rq_list_pop(&plug->mq_list))) { | 2786 | while ((rq = rq_list_pop(&plug->mq_list))) { |
| 2788 | bool last = rq_list_empty(plug->mq_list); | 2787 | bool last = rq_list_empty(&plug->mq_list); |
| 2789 | 2788 | ||
| 2790 | if (hctx != rq->mq_hctx) { | 2789 | if (hctx != rq->mq_hctx) { |
| 2791 | if (hctx) { | 2790 | if (hctx) { |
| @@ -2828,8 +2827,7 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) · 36 unmodified lines | |||
| 2828 | { | 2827 | { |
| 2829 | struct blk_mq_hw_ctx *this_hctx = NULL; | 2828 | struct blk_mq_hw_ctx *this_hctx = NULL; |
| 2830 | struct blk_mq_ctx *this_ctx = NULL; | 2829 | struct blk_mq_ctx *this_ctx = NULL; |
| 2831 | struct request *requeue_list = NULL; | 2830 | struct rq_list requeue_list = {}; |
| 2832 | struct request **requeue_lastp = &requeue_list; | ||
| 2833 | unsigned int depth = 0; | 2831 | unsigned int depth = 0; |
| 2834 | bool is_passthrough = false; | 2832 | bool is_passthrough = false; |
| 2835 | LIST_HEAD(list); | 2833 | LIST_HEAD(list); |
| @@ -2843,12 +2841,12 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) · 7 unmodified lines | |||
| 2843 | is_passthrough = blk_rq_is_passthrough(rq); | 2841 | is_passthrough = blk_rq_is_passthrough(rq); |
| 2844 | } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx || | 2842 | } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx || |
| 2845 | is_passthrough != blk_rq_is_passthrough(rq)) { | 2843 | is_passthrough != blk_rq_is_passthrough(rq)) { |
| 2846 | rq_list_add_tail(&requeue_lastp, rq); | 2844 | rq_list_add_tail(&requeue_list, rq); |
| 2847 | continue; | 2845 | continue; |
| 2848 | } | 2846 | } |
| 2849 | list_add(&rq->queuelist, &list); | 2847 | list_add(&rq->queuelist, &list); |
| 2850 | depth++; | 2848 | depth++; |
| 2851 | } while (!rq_list_empty(plug->mq_list)); | 2849 | } while (!rq_list_empty(&plug->mq_list)); |
| 2852 | 2850 | ||
| 2853 | plug->mq_list = requeue_list; | 2851 | plug->mq_list = requeue_list; |
| 2854 | trace_block_unplug(this_hctx->queue, depth, !from_sched); | 2852 | trace_block_unplug(this_hctx->queue, depth, !from_sched); |
| @@ -2903,19 +2901,19 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) · 48 unmodified lines | |||
| 2903 | if (q->mq_ops->queue_rqs) { | 2901 | if (q->mq_ops->queue_rqs) { |
| 2904 | blk_mq_run_dispatch_ops(q, | 2902 | blk_mq_run_dispatch_ops(q, |
| 2905 | __blk_mq_flush_plug_list(q, plug)); | 2903 | __blk_mq_flush_plug_list(q, plug)); |
| 2906 | if (rq_list_empty(plug->mq_list)) | 2904 | if (rq_list_empty(&plug->mq_list)) |
| 2907 | return; | 2905 | return; |
| 2908 | } | 2906 | } |
| 2909 | 2907 | ||
| 2910 | blk_mq_run_dispatch_ops(q, | 2908 | blk_mq_run_dispatch_ops(q, |
| 2911 | blk_mq_plug_issue_direct(plug)); | 2909 | blk_mq_plug_issue_direct(plug)); |
| 2912 | if (rq_list_empty(plug->mq_list)) | 2910 | if (rq_list_empty(&plug->mq_list)) |
| 2913 | return; | 2911 | return; |
| 2914 | } | 2912 | } |
| 2915 | 2913 | ||
| 2916 | do { | 2914 | do { |
| 2917 | blk_mq_dispatch_plug_list(plug, from_schedule); | 2915 | blk_mq_dispatch_plug_list(plug, from_schedule); |
| 2918 | } while (!rq_list_empty(plug->mq_list)); | 2916 | } while (!rq_list_empty(&plug->mq_list)); |
| 2919 | } | 2917 | } |
| 2920 | 2918 | ||
| 2921 | static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, | 2919 | static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, |
| @@ -2980,7 +2978,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, · 58 unmodified lines | |||
| 2980 | if (plug) { | 2978 | if (plug) { |
| 2981 | data.nr_tags = plug->nr_ios; | 2979 | data.nr_tags = plug->nr_ios; |
| 2982 | plug->nr_ios = 1; | 2980 | plug->nr_ios = 1; |
| 2983 | data.cached_rq = &plug->cached_rq; | 2981 | data.cached_rqs = &plug->cached_rqs; |
| 2984 | } | 2982 | } |
| 2985 | 2983 | ||
| 2986 | rq = __blk_mq_alloc_requests(&data); | 2984 | rq = __blk_mq_alloc_requests(&data); |
| @@ -3003,7 +3001,7 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, · 16 unmodified lines | |||
| 3003 | 3001 | ||
| 3004 | if (!plug) | 3002 | if (!plug) |
| 3005 | return NULL; | 3003 | return NULL; |
| 3006 | rq = rq_list_peek(&plug->cached_rq); | 3004 | rq = rq_list_peek(&plug->cached_rqs); |
| 3007 | if (!rq || rq->q != q) | 3005 | if (!rq || rq->q != q) |
| 3008 | return NULL; | 3006 | return NULL; |
| 3009 | if (type != rq->mq_hctx->type && | 3007 | if (type != rq->mq_hctx->type && |
| @@ -3017,14 +3015,14 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, · 7 unmodified lines | |||
| 3017 | static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug, | 3015 | static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug, |
| 3018 | struct bio *bio) | 3016 | struct bio *bio) |
| 3019 | { | 3017 | { |
| 3020 | WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq); | 3018 | if (rq_list_pop(&plug->cached_rqs) != rq) |
| 3019 | WARN_ON_ONCE(1); | ||
| 3021 | 3020 | ||
| 3022 | /* | 3021 | /* |
| 3023 | * If any qos ->throttle() end up blocking, we will have flushed the | 3022 | * If any qos ->throttle() end up blocking, we will have flushed the |
| 3024 | * plug and hence killed the cached_rq list as well. Pop this entry | 3023 | * plug and hence killed the cached_rq list as well. Pop this entry |
| 3025 | * before we throttle. | 3024 | * before we throttle. |
| 3026 | */ | 3025 | */ |
| 3027 | plug->cached_rq = rq_list_next(rq); | ||
| 3028 | rq_qos_throttle(rq->q, bio); | 3026 | rq_qos_throttle(rq->q, bio); |
| 3029 | 3027 | ||
| 3030 | blk_mq_rq_time_init(rq, blk_time_get_ns()); | 3028 | blk_mq_rq_time_init(rq, blk_time_get_ns()); |
| @@ -155,7 +155,7 @@ struct blk_mq_alloc_data { · 154 unmodified lines | |||
| 155 | 155 | ||
| 156 | /* allocate multiple requests/tags in one go */ | 156 | /* allocate multiple requests/tags in one go */ |
| 157 | unsigned int nr_tags; | 157 | unsigned int nr_tags; |
| 158 | struct request **cached_rq; | 158 | struct rq_list *cached_rqs; |
| 159 | 159 | ||
| 160 | /* input & output parameter */ | 160 | /* input & output parameter */ |
| 161 | struct blk_mq_ctx *ctx; | 161 | struct blk_mq_ctx *ctx; |
| @@ -1638,10 +1638,9 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, · 1637 unmodified lines | |||
| 1638 | return BLK_STS_OK; | 1638 | return BLK_STS_OK; |
| 1639 | } | 1639 | } |
| 1640 | 1640 | ||
| 1641 | static void null_queue_rqs(struct request **rqlist) | 1641 | static void null_queue_rqs(struct rq_list *rqlist) |
| 1642 | { | 1642 | { |
| 1643 | struct request *requeue_list = NULL; | 1643 | struct rq_list requeue_list = {}; |
| 1644 | struct request **requeue_lastp = &requeue_list; | ||
| 1645 | struct blk_mq_queue_data bd = { }; | 1644 | struct blk_mq_queue_data bd = { }; |
| 1646 | blk_status_t ret; | 1645 | blk_status_t ret; |
| 1647 | 1646 | ||
| @@ -1651,8 +1650,8 @@ static void null_queue_rqs(struct request **rqlist) · 3 unmodified lines | |||
| 1651 | bd.rq = rq; | 1650 | bd.rq = rq; |
| 1652 | ret = null_queue_rq(rq->mq_hctx, &bd); | 1651 | ret = null_queue_rq(rq->mq_hctx, &bd); |
| 1653 | if (ret != BLK_STS_OK) | 1652 | if (ret != BLK_STS_OK) |
| 1654 | rq_list_add_tail(&requeue_lastp, rq); | 1653 | rq_list_add_tail(&requeue_list, rq); |
| 1655 | } while (!rq_list_empty(*rqlist)); | 1654 | } while (!rq_list_empty(rqlist)); |
| 1656 | 1655 | ||
| 1657 | *rqlist = requeue_list; | 1656 | *rqlist = requeue_list; |
| 1658 | } | 1657 | } |
| @@ -472,7 +472,7 @@ static bool virtblk_prep_rq_batch(struct request *req) · 471 unmodified lines | |||
| 472 | } | 472 | } |
| 473 | 473 | ||
| 474 | static void virtblk_add_req_batch(struct virtio_blk_vq *vq, | 474 | static void virtblk_add_req_batch(struct virtio_blk_vq *vq, |
| 475 | struct request **rqlist) | 475 | struct rq_list *rqlist) |
| 476 | { | 476 | { |
| 477 | struct request *req; | 477 | struct request *req; |
| 478 | unsigned long flags; | 478 | unsigned long flags; |
| @@ -499,11 +499,10 @@ static void virtblk_add_req_batch(struct virtio_blk_vq *vq, · 20 unmodified lines | |||
| 499 | virtqueue_notify(vq->vq); | 499 | virtqueue_notify(vq->vq); |
| 500 | } | 500 | } |
| 501 | 501 | ||
| 502 | static void virtio_queue_rqs(struct request **rqlist) | 502 | static void virtio_queue_rqs(struct rq_list *rqlist) |
| 503 | { | 503 | { |
| 504 | struct request *submit_list = NULL; | 504 | struct rq_list submit_list = { }; |
| 505 | struct request *requeue_list = NULL; | 505 | struct rq_list requeue_list = { }; |
| 506 | struct request **requeue_lastp = &requeue_list; | ||
| 507 | struct virtio_blk_vq *vq = NULL; | 506 | struct virtio_blk_vq *vq = NULL; |
| 508 | struct request *req; | 507 | struct request *req; |
| 509 | 508 | ||
| @@ -515,9 +514,9 @@ static void virtio_queue_rqs(struct request **rqlist) · 5 unmodified lines | |||
| 515 | vq = this_vq; | 514 | vq = this_vq; |
| 516 | 515 | ||
| 517 | if (virtblk_prep_rq_batch(req)) | 516 | if (virtblk_prep_rq_batch(req)) |
| 518 | rq_list_add(&submit_list, req); /* reverse order */ | 517 | rq_list_add_head(&submit_list, req); /* reverse order */ |
| 519 | else | 518 | else |
| 520 | rq_list_add_tail(&requeue_lastp, req); | 519 | rq_list_add_tail(&requeue_list, req); |
| 521 | } | 520 | } |
| 522 | 521 | ||
| 523 | if (vq) | 522 | if (vq) |
| @@ -649,7 +649,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) · 648 unmodified lines | |||
| 649 | 649 | ||
| 650 | found = apple_nvme_poll_cq(q, &iob); | 650 | found = apple_nvme_poll_cq(q, &iob); |
| 651 | 651 | ||
| 652 | if (!rq_list_empty(iob.req_list)) | 652 | if (!rq_list_empty(&iob.req_list)) |
| 653 | apple_nvme_complete_batch(&iob); | 653 | apple_nvme_complete_batch(&iob); |
| 654 | 654 | ||
| 655 | return found; | 655 | return found; |
| @@ -904,7 +904,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, · 903 unmodified lines | |||
| 904 | return BLK_STS_OK; | 904 | return BLK_STS_OK; |
| 905 | } | 905 | } |
| 906 | 906 | ||
| 907 | static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist) | 907 | static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist) |
| 908 | { | 908 | { |
| 909 | struct request *req; | 909 | struct request *req; |
| 910 | 910 | ||
| @@ -932,11 +932,10 @@ static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req) · 21 unmodified lines | |||
| 932 | return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK; | 932 | return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK; |
| 933 | } | 933 | } |
| 934 | 934 | ||
| 935 | static void nvme_queue_rqs(struct request **rqlist) | 935 | static void nvme_queue_rqs(struct rq_list *rqlist) |
| 936 | { | 936 | { |
| 937 | struct request *submit_list = NULL; | 937 | struct rq_list submit_list = { }; |
| 938 | struct request *requeue_list = NULL; | 938 | struct rq_list requeue_list = { }; |
| 939 | struct request **requeue_lastp = &requeue_list; | ||
| 940 | struct nvme_queue *nvmeq = NULL; | 939 | struct nvme_queue *nvmeq = NULL; |
| 941 | struct request *req; | 940 | struct request *req; |
| 942 | 941 | ||
| @@ -946,9 +945,9 @@ static void nvme_queue_rqs(struct request **rqlist) · 3 unmodified lines | |||
| 946 | nvmeq = req->mq_hctx->driver_data; | 945 | nvmeq = req->mq_hctx->driver_data; |
| 947 | 946 | ||
| 948 | if (nvme_prep_rq_batch(nvmeq, req)) | 947 | if (nvme_prep_rq_batch(nvmeq, req)) |
| 949 | rq_list_add(&submit_list, req); /* reverse order */ | 948 | rq_list_add_head(&submit_list, req); /* reverse order */ |
| 950 | else | 949 | else |
| 951 | rq_list_add_tail(&requeue_lastp, req); | 950 | rq_list_add_tail(&requeue_list, req); |
| 952 | } | 951 | } |
| 953 | 952 | ||
| 954 | if (nvmeq) | 953 | if (nvmeq) |
| @@ -1080,7 +1079,7 @@ static irqreturn_t nvme_irq(int irq, void *data) · 125 unmodified lines | |||
| 1080 | DEFINE_IO_COMP_BATCH(iob); | 1079 | DEFINE_IO_COMP_BATCH(iob); |
| 1081 | 1080 | ||
| 1082 | if (nvme_poll_cq(nvmeq, &iob)) { | 1081 | if (nvme_poll_cq(nvmeq, &iob)) { |
| 1083 | if (!rq_list_empty(iob.req_list)) | 1082 | if (!rq_list_empty(&iob.req_list)) |
| 1084 | nvme_pci_complete_batch(&iob); | 1083 | nvme_pci_complete_batch(&iob); |
| 1085 | return IRQ_HANDLED; | 1084 | return IRQ_HANDLED; |
| 1086 | } | 1085 | } |
| @@ -229,44 +229,60 @@ static inline unsigned short req_get_ioprio(struct request *req) · 228 unmodified lines | |||
| 229 | #define rq_dma_dir(rq) \ | 229 | #define rq_dma_dir(rq) \ |
| 230 | (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE) | 230 | (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE) |
| 231 | 231 | ||
| 232 | #define rq_list_add(listptr, rq) do { \ | 232 | static inline int rq_list_empty(const struct rq_list *rl) |
| 233 | (rq)->rq_next = *(listptr); \ | 233 | { |
| 234 | *(listptr) = rq; \ | 234 | return rl->head == NULL; |
| 235 | } while (0) | 235 | } |
| 236 | |||
| 237 | #define rq_list_add_tail(lastpptr, rq) do { \ | ||
| 238 | (rq)->rq_next = NULL; \ | ||
| 239 | **(lastpptr) = rq; \ | ||
| 240 | *(lastpptr) = &rq->rq_next; \ | ||
| 241 | } while (0) | ||
| 242 | |||
| 243 | #define rq_list_pop(listptr) \ | ||
| 244 | ({ \ | ||
| 245 | struct request *__req = NULL; \ | ||
| 246 | if ((listptr) && *(listptr)) { \ | ||
| 247 | __req = *(listptr); \ | ||
| 248 | *(listptr) = __req->rq_next; \ | ||
| 249 | } \ | ||
| 250 | __req; \ | ||
| 251 | }) | ||
| 252 | 236 | ||
| 253 | #define rq_list_peek(listptr) \ | 237 | static inline void rq_list_init(struct rq_list *rl) |
| 254 | ({ \ | 238 | { |
| 255 | struct request *__req = NULL; \ | 239 | rl->head = NULL; |
| 256 | if ((listptr) && *(listptr)) \ | 240 | rl->tail = NULL; |
| 257 | __req = *(listptr); \ | 241 | } |
| 258 | __req; \ | 242 | |
| 259 | }) | 243 | static inline void rq_list_add_tail(struct rq_list *rl, struct request *rq) |
| 244 | { | ||
| 245 | rq->rq_next = NULL; | ||
| 246 | if (rl->tail) | ||
| 247 | rl->tail->rq_next = rq; | ||
| 248 | else | ||
| 249 | rl->head = rq; | ||
| 250 | rl->tail = rq; | ||
| 251 | } | ||
| 252 | |||
| 253 | static inline void rq_list_add_head(struct rq_list *rl, struct request *rq) | ||
| 254 | { | ||
| 255 | rq->rq_next = rl->head; | ||
| 256 | rl->head = rq; | ||
| 257 | if (!rl->tail) | ||
| 258 | rl->tail = rq; | ||
| 259 | } | ||
| 260 | |||
| 261 | static inline struct request *rq_list_pop(struct rq_list *rl) | ||
| 262 | { | ||
| 263 | struct request *rq = rl->head; | ||
| 264 | |||
| 265 | if (rq) { | ||
| 266 | rl->head = rl->head->rq_next; | ||
| 267 | if (!rl->head) | ||
| 268 | rl->tail = NULL; | ||
| 269 | rq->rq_next = NULL; | ||
| 270 | } | ||
| 271 | |||
| 272 | return rq; | ||
| 273 | } | ||
| 260 | 274 | ||
| 261 | #define rq_list_for_each(listptr, pos) \ | 275 | static inline struct request *rq_list_peek(struct rq_list *rl) |
| 262 | for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos)) | 276 | { |
| 277 | return rl->head; | ||
| 278 | } | ||
| 263 | 279 | ||
| 264 | #define rq_list_for_each_safe(listptr, pos, nxt) \ | 280 | #define rq_list_for_each(rl, pos) \ |
| 265 | for (pos = rq_list_peek((listptr)), nxt = rq_list_next(pos); \ | 281 | for (pos = rq_list_peek((rl)); (pos); pos = pos->rq_next) |
| 266 | pos; pos = nxt, nxt = pos ? rq_list_next(pos) : NULL) | ||
| 267 | 282 | ||
| 268 | #define rq_list_next(rq) (rq)->rq_next | 283 | #define rq_list_for_each_safe(rl, pos, nxt) \ |
| 269 | #define rq_list_empty(list) ((list) == (struct request *) NULL) | 284 | for (pos = rq_list_peek((rl)), nxt = pos->rq_next; \ |
| 285 | pos; pos = nxt, nxt = pos ? pos->rq_next : NULL) | ||
| 270 | 286 | ||
| 271 | /** | 287 | /** |
| 272 | * enum blk_eh_timer_return - How the timeout handler should proceed | 288 | * enum blk_eh_timer_return - How the timeout handler should proceed |
| @@ -559,7 +575,7 @@ struct blk_mq_ops { · 286 unmodified lines | |||
| 559 | * empty the @rqlist completely, then the rest will be queued | 575 | * empty the @rqlist completely, then the rest will be queued |
| 560 | * individually by the block layer upon return. | 576 | * individually by the block layer upon return. |
| 561 | */ | 577 | */ |
| 562 | void (*queue_rqs)(struct request **rqlist); | 578 | void (*queue_rqs)(struct rq_list *rqlist); |
| 563 | 579 | ||
| 564 | /** | 580 | /** |
| 565 | * @get_budget: Reserve budget before queue request, once .queue_rq is | 581 | * @get_budget: Reserve budget before queue request, once .queue_rq is |
| @@ -868,7 +884,7 @@ static inline bool blk_mq_add_to_batch(struct request *req, · 302 unmodified lines | |||
| 868 | else if (iob->complete != complete) | 884 | else if (iob->complete != complete) |
| 869 | return false; | 885 | return false; |
| 870 | iob->need_ts |= blk_mq_need_time_stamp(req); | 886 | iob->need_ts |= blk_mq_need_time_stamp(req); |
| 871 | rq_list_add(&iob->req_list, req); | 887 | rq_list_add_head(&iob->req_list, req); |
| 872 | return true; | 888 | return true; |
| 873 | } | 889 | } |
| 874 | 890 | ||
| @@ -1007,6 +1007,11 @@ extern void blk_put_queue(struct request_queue *); · 1006 unmodified lines | |||
| 1007 | void blk_mark_disk_dead(struct gendisk *disk); | 1007 | void blk_mark_disk_dead(struct gendisk *disk); |
| 1008 | 1008 | ||
| 1009 | #ifdef CONFIG_BLOCK | 1009 | #ifdef CONFIG_BLOCK |
| 1010 | struct rq_list { | ||
| 1011 | struct request *head; | ||
| 1012 | struct request *tail; | ||
| 1013 | }; | ||
| 1014 | |||
| 1010 | /* | 1015 | /* |
| 1011 | * blk_plug permits building a queue of related requests by holding the I/O | 1016 | * blk_plug permits building a queue of related requests by holding the I/O |
| 1012 | * fragments for a short period. This allows merging of sequential requests | 1017 | * fragments for a short period. This allows merging of sequential requests |
| @@ -1019,10 +1024,10 @@ void blk_mark_disk_dead(struct gendisk *disk); · 6 unmodified lines | |||
| 1019 | * blk_flush_plug() is called. | 1024 | * blk_flush_plug() is called. |
| 1020 | */ | 1025 | */ |
| 1021 | struct blk_plug { | 1026 | struct blk_plug { |
| 1022 | struct request *mq_list; /* blk-mq requests */ | 1027 | struct rq_list mq_list; /* blk-mq requests */ |
| 1023 | 1028 | ||
| 1024 | /* if ios_left is > 1, we can batch tag/rq allocations */ | 1029 | /* if ios_left is > 1, we can batch tag/rq allocations */ |
| 1025 | struct request *cached_rq; | 1030 | struct rq_list cached_rqs; |
| 1026 | u64 cur_ktime; | 1031 | u64 cur_ktime; |
| 1027 | unsigned short nr_ios; | 1032 | unsigned short nr_ios; |
| 1028 | 1033 | ||
| @@ -1684,7 +1689,7 @@ int bdev_thaw(struct block_device *bdev); · 655 unmodified lines | |||
| 1684 | void bdev_fput(struct file *bdev_file); | 1689 | void bdev_fput(struct file *bdev_file); |
| 1685 | 1690 | ||
| 1686 | struct io_comp_batch { | 1691 | struct io_comp_batch { |
| 1687 | struct request *req_list; | 1692 | struct rq_list req_list; |
| 1688 | bool need_ts; | 1693 | bool need_ts; |
| 1689 | void (*complete)(struct io_comp_batch *); | 1694 | void (*complete)(struct io_comp_batch *); |
| 1690 | }; | 1695 | }; |
| @@ -1160,12 +1160,12 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 1159 unmodified lines | |||
| 1160 | poll_flags |= BLK_POLL_ONESHOT; | 1160 | poll_flags |= BLK_POLL_ONESHOT; |
| 1161 | 1161 | ||
| 1162 | /* iopoll may have completed current req */ | 1162 | /* iopoll may have completed current req */ |
| 1163 | if (!rq_list_empty(iob.req_list) || | 1163 | if (!rq_list_empty(&iob.req_list) || |
| 1164 | READ_ONCE(req->iopoll_completed)) | 1164 | READ_ONCE(req->iopoll_completed)) |
| 1165 | break; | 1165 | break; |
| 1166 | } | 1166 | } |
| 1167 | 1167 | ||
| 1168 | if (!rq_list_empty(iob.req_list)) | 1168 | if (!rq_list_empty(&iob.req_list)) |
| 1169 | iob.complete(&iob); | 1169 | iob.complete(&iob); |
| 1170 | else if (!pos) | 1170 | else if (!pos) |
| 1171 | return 0; | 1171 | return 0; |