Replace the semi-open coded request list helpers with a proper rq_list type that mirrors the bio_list and has head and tail pointers. Besides better type safety this actually allows to insert at the tail of the list, which will be useful soon.
Unified · Split
| @@ -1120,8 +1120,8 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios) · 1119 unmodified lines | ||
| 1120 | 1120 | return; |
| 1121 | 1121 | |
| 1122 | 1122 | plug->cur_ktime = 0; |
| 1123 | − plug->mq_list = NULL; | |
| 1124 | − plug->cached_rq = NULL; | |
| 1123 | + rq_list_init(&plug->mq_list); | |
| 1124 | + rq_list_init(&plug->cached_rqs); | |
| 1125 | 1125 | plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT); |
| 1126 | 1126 | plug->rq_count = 0; |
| 1127 | 1127 | plug->multiple_queues = false; |
| @@ -1217,7 +1217,7 @@ void __blk_flush_plug(struct blk_plug *plug, bool from_schedule) · 89 unmodified lines | ||
| 1217 | 1217 | * queue for cached requests, we don't want a blocked task holding |
| 1218 | 1218 | * up a queue freeze/quiesce event. |
| 1219 | 1219 | */ |
| 1220 | − if (unlikely(!rq_list_empty(plug->cached_rq))) | |
| 1220 | + if (unlikely(!rq_list_empty(&plug->cached_rqs))) | |
| 1221 | 1221 | blk_mq_free_plug_rqs(plug); |
| 1222 | 1222 | |
| 1223 | 1223 | plug->cur_ktime = 0; |
| @@ -1179,7 +1179,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, · 1178 unmodified lines | ||
| 1179 | 1179 | struct blk_plug *plug = current->plug; |
| 1180 | 1180 | struct request *rq; |
| 1181 | 1181 | |
| 1182 | − if (!plug || rq_list_empty(plug->mq_list)) | |
| 1182 | + if (!plug || rq_list_empty(&plug->mq_list)) | |
| 1183 | 1183 | return false; |
| 1184 | 1184 | |
| 1185 | 1185 | rq_list_for_each(&plug->mq_list, rq) { |
| @@ -478,7 +478,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) · 477 unmodified lines | ||
| 478 | 478 | prefetch(tags->static_rqs[tag]); |
| 479 | 479 | tag_mask &= ~(1UL << i); |
| 480 | 480 | rq = blk_mq_rq_ctx_init(data, tags, tag); |
| 481 | − rq_list_add(data->cached_rq, rq); | |
| 481 | + rq_list_add_head(data->cached_rqs, rq); | |
| 482 | 482 | nr++; |
| 483 | 483 | } |
| 484 | 484 | if (!(data->rq_flags & RQF_SCHED_TAGS)) |
| @@ -487,7 +487,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) · 2 unmodified lines | ||
| 487 | 487 | percpu_ref_get_many(&data->q->q_usage_counter, nr - 1); |
| 488 | 488 | data->nr_tags -= nr; |
| 489 | 489 | |
| 490 | − return rq_list_pop(data->cached_rq); | |
| 490 | + return rq_list_pop(data->cached_rqs); | |
| 491 | 491 | } |
| 492 | 492 | |
| 493 | 493 | static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) |
| @@ -584,7 +584,7 @@ static struct request *blk_mq_rq_cache_fill(struct request_queue *q, · 90 unmodified lines | ||
| 584 | 584 | .flags = flags, |
| 585 | 585 | .cmd_flags = opf, |
| 586 | 586 | .nr_tags = plug->nr_ios, |
| 587 | − .cached_rq = &plug->cached_rq, | |
| 587 | + .cached_rqs = &plug->cached_rqs, | |
| 588 | 588 | }; |
| 589 | 589 | struct request *rq; |
| 590 | 590 | |
| @@ -609,14 +609,14 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, · 18 unmodified lines | ||
| 609 | 609 | if (!plug) |
| 610 | 610 | return NULL; |
| 611 | 611 | |
| 612 | − if (rq_list_empty(plug->cached_rq)) { | |
| 612 | + if (rq_list_empty(&plug->cached_rqs)) { | |
| 613 | 613 | if (plug->nr_ios == 1) |
| 614 | 614 | return NULL; |
| 615 | 615 | rq = blk_mq_rq_cache_fill(q, plug, opf, flags); |
| 616 | 616 | if (!rq) |
| 617 | 617 | return NULL; |
| 618 | 618 | } else { |
| 619 | − rq = rq_list_peek(&plug->cached_rq); | |
| 619 | + rq = rq_list_peek(&plug->cached_rqs); | |
| 620 | 620 | if (!rq || rq->q != q) |
| 621 | 621 | return NULL; |
| 622 | 622 | |
| @@ -625,7 +625,7 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, · 2 unmodified lines | ||
| 625 | 625 | if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) |
| 626 | 626 | return NULL; |
| 627 | 627 | |
| 628 | − plug->cached_rq = rq_list_next(rq); | |
| 628 | + rq_list_pop(&plug->cached_rqs); | |
| 629 | 629 | blk_mq_rq_time_init(rq, blk_time_get_ns()); |
| 630 | 630 | } |
| 631 | 631 | |
| @@ -802,7 +802,7 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug) · 170 unmodified lines | ||
| 802 | 802 | { |
| 803 | 803 | struct request *rq; |
| 804 | 804 | |
| 805 | − while ((rq = rq_list_pop(&plug->cached_rq)) != NULL) | |
| 805 | + while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL) | |
| 806 | 806 | blk_mq_free_request(rq); |
| 807 | 807 | } |
| 808 | 808 | |
| @@ -1392,8 +1392,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) · 583 unmodified lines | ||
| 1392 | 1392 | */ |
| 1393 | 1393 | if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS)) |
| 1394 | 1394 | plug->has_elevator = true; |
| 1395 | − rq->rq_next = NULL; | |
| 1396 | − rq_list_add(&plug->mq_list, rq); | |
| 1395 | + rq_list_add_head(&plug->mq_list, rq); | |
| 1397 | 1396 | plug->rq_count++; |
| 1398 | 1397 | } |
| 1399 | 1398 | |
| @@ -2785,7 +2784,7 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug) · 1385 unmodified lines | ||
| 2785 | 2784 | blk_status_t ret = BLK_STS_OK; |
| 2786 | 2785 | |
| 2787 | 2786 | while ((rq = rq_list_pop(&plug->mq_list))) { |
| 2788 | − bool last = rq_list_empty(plug->mq_list); | |
| 2787 | + bool last = rq_list_empty(&plug->mq_list); | |
| 2789 | 2788 | |
| 2790 | 2789 | if (hctx != rq->mq_hctx) { |
| 2791 | 2790 | if (hctx) { |
| @@ -2828,8 +2827,7 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) · 36 unmodified lines | ||
| 2828 | 2827 | { |
| 2829 | 2828 | struct blk_mq_hw_ctx *this_hctx = NULL; |
| 2830 | 2829 | struct blk_mq_ctx *this_ctx = NULL; |
| 2831 | − struct request *requeue_list = NULL; | |
| 2832 | − struct request **requeue_lastp = &requeue_list; | |
| 2830 | + struct rq_list requeue_list = {}; | |
| 2833 | 2831 | unsigned int depth = 0; |
| 2834 | 2832 | bool is_passthrough = false; |
| 2835 | 2833 | LIST_HEAD(list); |
| @@ -2843,12 +2841,12 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) · 7 unmodified lines | ||
| 2843 | 2841 | is_passthrough = blk_rq_is_passthrough(rq); |
| 2844 | 2842 | } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx || |
| 2845 | 2843 | is_passthrough != blk_rq_is_passthrough(rq)) { |
| 2846 | − rq_list_add_tail(&requeue_lastp, rq); | |
| 2844 | + rq_list_add_tail(&requeue_list, rq); | |
| 2847 | 2845 | continue; |
| 2848 | 2846 | } |
| 2849 | 2847 | list_add(&rq->queuelist, &list); |
| 2850 | 2848 | depth++; |
| 2851 | − } while (!rq_list_empty(plug->mq_list)); | |
| 2849 | + } while (!rq_list_empty(&plug->mq_list)); | |
| 2852 | 2850 | |
| 2853 | 2851 | plug->mq_list = requeue_list; |
| 2854 | 2852 | trace_block_unplug(this_hctx->queue, depth, !from_sched); |
| @@ -2903,19 +2901,19 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) · 48 unmodified lines | ||
| 2903 | 2901 | if (q->mq_ops->queue_rqs) { |
| 2904 | 2902 | blk_mq_run_dispatch_ops(q, |
| 2905 | 2903 | __blk_mq_flush_plug_list(q, plug)); |
| 2906 | − if (rq_list_empty(plug->mq_list)) | |
| 2904 | + if (rq_list_empty(&plug->mq_list)) | |
| 2907 | 2905 | return; |
| 2908 | 2906 | } |
| 2909 | 2907 | |
| 2910 | 2908 | blk_mq_run_dispatch_ops(q, |
| 2911 | 2909 | blk_mq_plug_issue_direct(plug)); |
| 2912 | − if (rq_list_empty(plug->mq_list)) | |
| 2910 | + if (rq_list_empty(&plug->mq_list)) | |
| 2913 | 2911 | return; |
| 2914 | 2912 | } |
| 2915 | 2913 | |
| 2916 | 2914 | do { |
| 2917 | 2915 | blk_mq_dispatch_plug_list(plug, from_schedule); |
| 2918 | − } while (!rq_list_empty(plug->mq_list)); | |
| 2916 | + } while (!rq_list_empty(&plug->mq_list)); | |
| 2919 | 2917 | } |
| 2920 | 2918 | |
| 2921 | 2919 | static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, |
| @@ -2980,7 +2978,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, · 58 unmodified lines | ||
| 2980 | 2978 | if (plug) { |
| 2981 | 2979 | data.nr_tags = plug->nr_ios; |
| 2982 | 2980 | plug->nr_ios = 1; |
| 2983 | − data.cached_rq = &plug->cached_rq; | |
| 2981 | + data.cached_rqs = &plug->cached_rqs; | |
| 2984 | 2982 | } |
| 2985 | 2983 | |
| 2986 | 2984 | rq = __blk_mq_alloc_requests(&data); |
| @@ -3003,7 +3001,7 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, · 16 unmodified lines | ||
| 3003 | 3001 | |
| 3004 | 3002 | if (!plug) |
| 3005 | 3003 | return NULL; |
| 3006 | − rq = rq_list_peek(&plug->cached_rq); | |
| 3004 | + rq = rq_list_peek(&plug->cached_rqs); | |
| 3007 | 3005 | if (!rq || rq->q != q) |
| 3008 | 3006 | return NULL; |
| 3009 | 3007 | if (type != rq->mq_hctx->type && |
| @@ -3017,14 +3015,14 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, · 7 unmodified lines | ||
| 3017 | 3015 | static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug, |
| 3018 | 3016 | struct bio *bio) |
| 3019 | 3017 | { |
| 3020 | − WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq); | |
| 3018 | + if (rq_list_pop(&plug->cached_rqs) != rq) | |
| 3019 | + WARN_ON_ONCE(1); | |
| 3021 | 3020 | |
| 3022 | 3021 | /* |
| 3023 | 3022 | * If any qos ->throttle() end up blocking, we will have flushed the |
| 3024 | 3023 | * plug and hence killed the cached_rq list as well. Pop this entry |
| 3025 | 3024 | * before we throttle. |
| 3026 | 3025 | */ |
| 3027 | − plug->cached_rq = rq_list_next(rq); | |
| 3028 | 3026 | rq_qos_throttle(rq->q, bio); |
| 3029 | 3027 | |
| 3030 | 3028 | blk_mq_rq_time_init(rq, blk_time_get_ns()); |
| @@ -155,7 +155,7 @@ struct blk_mq_alloc_data { · 154 unmodified lines | ||
| 155 | 155 | |
| 156 | 156 | /* allocate multiple requests/tags in one go */ |
| 157 | 157 | unsigned int nr_tags; |
| 158 | − struct request **cached_rq; | |
| 158 | + struct rq_list *cached_rqs; | |
| 159 | 159 | |
| 160 | 160 | /* input & output parameter */ |
| 161 | 161 | struct blk_mq_ctx *ctx; |
| @@ -1638,10 +1638,9 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, · 1637 unmodified lines | ||
| 1638 | 1638 | return BLK_STS_OK; |
| 1639 | 1639 | } |
| 1640 | 1640 | |
| 1641 | −static void null_queue_rqs(struct request **rqlist) | |
| 1641 | +static void null_queue_rqs(struct rq_list *rqlist) | |
| 1642 | 1642 | { |
| 1643 | − struct request *requeue_list = NULL; | |
| 1644 | − struct request **requeue_lastp = &requeue_list; | |
| 1643 | + struct rq_list requeue_list = {}; | |
| 1645 | 1644 | struct blk_mq_queue_data bd = { }; |
| 1646 | 1645 | blk_status_t ret; |
| 1647 | 1646 | |
| @@ -1651,8 +1650,8 @@ static void null_queue_rqs(struct request **rqlist) · 3 unmodified lines | ||
| 1651 | 1650 | bd.rq = rq; |
| 1652 | 1651 | ret = null_queue_rq(rq->mq_hctx, &bd); |
| 1653 | 1652 | if (ret != BLK_STS_OK) |
| 1654 | − rq_list_add_tail(&requeue_lastp, rq); | |
| 1655 | − } while (!rq_list_empty(*rqlist)); | |
| 1653 | + rq_list_add_tail(&requeue_list, rq); | |
| 1654 | + } while (!rq_list_empty(rqlist)); | |
| 1656 | 1655 | |
| 1657 | 1656 | *rqlist = requeue_list; |
| 1658 | 1657 | } |
| @@ -472,7 +472,7 @@ static bool virtblk_prep_rq_batch(struct request *req) · 471 unmodified lines | ||
| 472 | 472 | } |
| 473 | 473 | |
| 474 | 474 | static void virtblk_add_req_batch(struct virtio_blk_vq *vq, |
| 475 | − struct request **rqlist) | |
| 475 | + struct rq_list *rqlist) | |
| 476 | 476 | { |
| 477 | 477 | struct request *req; |
| 478 | 478 | unsigned long flags; |
| @@ -499,11 +499,10 @@ static void virtblk_add_req_batch(struct virtio_blk_vq *vq, · 20 unmodified lines | ||
| 499 | 499 | virtqueue_notify(vq->vq); |
| 500 | 500 | } |
| 501 | 501 | |
| 502 | −static void virtio_queue_rqs(struct request **rqlist) | |
| 502 | +static void virtio_queue_rqs(struct rq_list *rqlist) | |
| 503 | 503 | { |
| 504 | − struct request *submit_list = NULL; | |
| 505 | − struct request *requeue_list = NULL; | |
| 506 | − struct request **requeue_lastp = &requeue_list; | |
| 504 | + struct rq_list submit_list = { }; | |
| 505 | + struct rq_list requeue_list = { }; | |
| 507 | 506 | struct virtio_blk_vq *vq = NULL; |
| 508 | 507 | struct request *req; |
| 509 | 508 | |
| @@ -515,9 +514,9 @@ static void virtio_queue_rqs(struct request **rqlist) · 5 unmodified lines | ||
| 515 | 514 | vq = this_vq; |
| 516 | 515 | |
| 517 | 516 | if (virtblk_prep_rq_batch(req)) |
| 518 | − rq_list_add(&submit_list, req); /* reverse order */ | |
| 517 | + rq_list_add_head(&submit_list, req); /* reverse order */ | |
| 519 | 518 | else |
| 520 | − rq_list_add_tail(&requeue_lastp, req); | |
| 519 | + rq_list_add_tail(&requeue_list, req); | |
| 521 | 520 | } |
| 522 | 521 | |
| 523 | 522 | if (vq) |
| @@ -649,7 +649,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) · 648 unmodified lines | ||
| 649 | 649 | |
| 650 | 650 | found = apple_nvme_poll_cq(q, &iob); |
| 651 | 651 | |
| 652 | − if (!rq_list_empty(iob.req_list)) | |
| 652 | + if (!rq_list_empty(&iob.req_list)) | |
| 653 | 653 | apple_nvme_complete_batch(&iob); |
| 654 | 654 | |
| 655 | 655 | return found; |
| @@ -904,7 +904,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, · 903 unmodified lines | ||
| 904 | 904 | return BLK_STS_OK; |
| 905 | 905 | } |
| 906 | 906 | |
| 907 | −static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist) | |
| 907 | +static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist) | |
| 908 | 908 | { |
| 909 | 909 | struct request *req; |
| 910 | 910 | |
| @@ -932,11 +932,10 @@ static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req) · 21 unmodified lines | ||
| 932 | 932 | return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK; |
| 933 | 933 | } |
| 934 | 934 | |
| 935 | −static void nvme_queue_rqs(struct request **rqlist) | |
| 935 | +static void nvme_queue_rqs(struct rq_list *rqlist) | |
| 936 | 936 | { |
| 937 | − struct request *submit_list = NULL; | |
| 938 | − struct request *requeue_list = NULL; | |
| 939 | − struct request **requeue_lastp = &requeue_list; | |
| 937 | + struct rq_list submit_list = { }; | |
| 938 | + struct rq_list requeue_list = { }; | |
| 940 | 939 | struct nvme_queue *nvmeq = NULL; |
| 941 | 940 | struct request *req; |
| 942 | 941 | |
| @@ -946,9 +945,9 @@ static void nvme_queue_rqs(struct request **rqlist) · 3 unmodified lines | ||
| 946 | 945 | nvmeq = req->mq_hctx->driver_data; |
| 947 | 946 | |
| 948 | 947 | if (nvme_prep_rq_batch(nvmeq, req)) |
| 949 | − rq_list_add(&submit_list, req); /* reverse order */ | |
| 948 | + rq_list_add_head(&submit_list, req); /* reverse order */ | |
| 950 | 949 | else |
| 951 | − rq_list_add_tail(&requeue_lastp, req); | |
| 950 | + rq_list_add_tail(&requeue_list, req); | |
| 952 | 951 | } |
| 953 | 952 | |
| 954 | 953 | if (nvmeq) |
| @@ -1080,7 +1079,7 @@ static irqreturn_t nvme_irq(int irq, void *data) · 125 unmodified lines | ||
| 1080 | 1079 | DEFINE_IO_COMP_BATCH(iob); |
| 1081 | 1080 | |
| 1082 | 1081 | if (nvme_poll_cq(nvmeq, &iob)) { |
| 1083 | − if (!rq_list_empty(iob.req_list)) | |
| 1082 | + if (!rq_list_empty(&iob.req_list)) | |
| 1084 | 1083 | nvme_pci_complete_batch(&iob); |
| 1085 | 1084 | return IRQ_HANDLED; |
| 1086 | 1085 | } |
| @@ -229,44 +229,60 @@ static inline unsigned short req_get_ioprio(struct request *req) · 228 unmodified lines | ||
| 229 | 229 | #define rq_dma_dir(rq) \ |
| 230 | 230 | (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE) |
| 231 | 231 | |
| 232 | −#define rq_list_add(listptr, rq) do { \ | |
| 233 | − (rq)->rq_next = *(listptr); \ | |
| 234 | − *(listptr) = rq; \ | |
| 235 | −} while (0) | |
| 236 | − | |
| 237 | −#define rq_list_add_tail(lastpptr, rq) do { \ | |
| 238 | − (rq)->rq_next = NULL; \ | |
| 239 | − **(lastpptr) = rq; \ | |
| 240 | − *(lastpptr) = &rq->rq_next; \ | |
| 241 | −} while (0) | |
| 242 | − | |
| 243 | −#define rq_list_pop(listptr) \ | |
| 244 | −({ \ | |
| 245 | − struct request *__req = NULL; \ | |
| 246 | − if ((listptr) && *(listptr)) { \ | |
| 247 | − __req = *(listptr); \ | |
| 248 | − *(listptr) = __req->rq_next; \ | |
| 249 | − } \ | |
| 250 | − __req; \ | |
| 251 | −}) | |
| 232 | +static inline int rq_list_empty(const struct rq_list *rl) | |
| 233 | +{ | |
| 234 | + return rl->head == NULL; | |
| 235 | +} | |
| 252 | 236 | |
| 253 | −#define rq_list_peek(listptr) \ | |
| 254 | −({ \ | |
| 255 | − struct request *__req = NULL; \ | |
| 256 | − if ((listptr) && *(listptr)) \ | |
| 257 | − __req = *(listptr); \ | |
| 258 | − __req; \ | |
| 259 | −}) | |
| 237 | +static inline void rq_list_init(struct rq_list *rl) | |
| 238 | +{ | |
| 239 | + rl->head = NULL; | |
| 240 | + rl->tail = NULL; | |
| 241 | +} | |
| 242 | + | |
| 243 | +static inline void rq_list_add_tail(struct rq_list *rl, struct request *rq) | |
| 244 | +{ | |
| 245 | + rq->rq_next = NULL; | |
| 246 | + if (rl->tail) | |
| 247 | + rl->tail->rq_next = rq; | |
| 248 | + else | |
| 249 | + rl->head = rq; | |
| 250 | + rl->tail = rq; | |
| 251 | +} | |
| 252 | + | |
| 253 | +static inline void rq_list_add_head(struct rq_list *rl, struct request *rq) | |
| 254 | +{ | |
| 255 | + rq->rq_next = rl->head; | |
| 256 | + rl->head = rq; | |
| 257 | + if (!rl->tail) | |
| 258 | + rl->tail = rq; | |
| 259 | +} | |
| 260 | + | |
| 261 | +static inline struct request *rq_list_pop(struct rq_list *rl) | |
| 262 | +{ | |
| 263 | + struct request *rq = rl->head; | |
| 264 | + | |
| 265 | + if (rq) { | |
| 266 | + rl->head = rl->head->rq_next; | |
| 267 | + if (!rl->head) | |
| 268 | + rl->tail = NULL; | |
| 269 | + rq->rq_next = NULL; | |
| 270 | + } | |
| 271 | + | |
| 272 | + return rq; | |
| 273 | +} | |
| 260 | 274 | |
| 261 | −#define rq_list_for_each(listptr, pos) \ | |
| 262 | − for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos)) | |
| 275 | +static inline struct request *rq_list_peek(struct rq_list *rl) | |
| 276 | +{ | |
| 277 | + return rl->head; | |
| 278 | +} | |
| 263 | 279 | |
| 264 | −#define rq_list_for_each_safe(listptr, pos, nxt) \ | |
| 265 | − for (pos = rq_list_peek((listptr)), nxt = rq_list_next(pos); \ | |
| 266 | − pos; pos = nxt, nxt = pos ? rq_list_next(pos) : NULL) | |
| 280 | +#define rq_list_for_each(rl, pos) \ | |
| 281 | + for (pos = rq_list_peek((rl)); (pos); pos = pos->rq_next) | |
| 267 | 282 | |
| 268 | −#define rq_list_next(rq) (rq)->rq_next | |
| 269 | −#define rq_list_empty(list) ((list) == (struct request *) NULL) | |
| 283 | +#define rq_list_for_each_safe(rl, pos, nxt) \ | |
| 284 | + for (pos = rq_list_peek((rl)), nxt = pos->rq_next; \ | |
| 285 | + pos; pos = nxt, nxt = pos ? pos->rq_next : NULL) | |
| 270 | 286 | |
| 271 | 287 | /** |
| 272 | 288 | * enum blk_eh_timer_return - How the timeout handler should proceed |
| @@ -559,7 +575,7 @@ struct blk_mq_ops { · 286 unmodified lines | ||
| 559 | 575 | * empty the @rqlist completely, then the rest will be queued |
| 560 | 576 | * individually by the block layer upon return. |
| 561 | 577 | */ |
| 562 | − void (*queue_rqs)(struct request **rqlist); | |
| 578 | + void (*queue_rqs)(struct rq_list *rqlist); | |
| 563 | 579 | |
| 564 | 580 | /** |
| 565 | 581 | * @get_budget: Reserve budget before queue request, once .queue_rq is |
| @@ -868,7 +884,7 @@ static inline bool blk_mq_add_to_batch(struct request *req, · 302 unmodified lines | ||
| 868 | 884 | else if (iob->complete != complete) |
| 869 | 885 | return false; |
| 870 | 886 | iob->need_ts |= blk_mq_need_time_stamp(req); |
| 871 | − rq_list_add(&iob->req_list, req); | |
| 887 | + rq_list_add_head(&iob->req_list, req); | |
| 872 | 888 | return true; |
| 873 | 889 | } |
| 874 | 890 | |
| @@ -1007,6 +1007,11 @@ extern void blk_put_queue(struct request_queue *); · 1006 unmodified lines | ||
| 1007 | 1007 | void blk_mark_disk_dead(struct gendisk *disk); |
| 1008 | 1008 | |
| 1009 | 1009 | #ifdef CONFIG_BLOCK |
| 1010 | +struct rq_list { | |
| 1011 | + struct request *head; | |
| 1012 | + struct request *tail; | |
| 1013 | +}; | |
| 1014 | + | |
| 1010 | 1015 | /* |
| 1011 | 1016 | * blk_plug permits building a queue of related requests by holding the I/O |
| 1012 | 1017 | * fragments for a short period. This allows merging of sequential requests |
| @@ -1019,10 +1024,10 @@ void blk_mark_disk_dead(struct gendisk *disk); · 6 unmodified lines | ||
| 1019 | 1024 | * blk_flush_plug() is called. |
| 1020 | 1025 | */ |
| 1021 | 1026 | struct blk_plug { |
| 1022 | − struct request *mq_list; /* blk-mq requests */ | |
| 1027 | + struct rq_list mq_list; /* blk-mq requests */ | |
| 1023 | 1028 | |
| 1024 | 1029 | /* if ios_left is > 1, we can batch tag/rq allocations */ |
| 1025 | − struct request *cached_rq; | |
| 1030 | + struct rq_list cached_rqs; | |
| 1026 | 1031 | u64 cur_ktime; |
| 1027 | 1032 | unsigned short nr_ios; |
| 1028 | 1033 | |
| @@ -1684,7 +1689,7 @@ int bdev_thaw(struct block_device *bdev); · 655 unmodified lines | ||
| 1684 | 1689 | void bdev_fput(struct file *bdev_file); |
| 1685 | 1690 | |
| 1686 | 1691 | struct io_comp_batch { |
| 1687 | − struct request *req_list; | |
| 1692 | + struct rq_list req_list; | |
| 1688 | 1693 | bool need_ts; |
| 1689 | 1694 | void (*complete)(struct io_comp_batch *); |
| 1690 | 1695 | }; |
| @@ -1160,12 +1160,12 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 1159 unmodified lines | ||
| 1160 | 1160 | poll_flags |= BLK_POLL_ONESHOT; |
| 1161 | 1161 | |
| 1162 | 1162 | /* iopoll may have completed current req */ |
| 1163 | − if (!rq_list_empty(iob.req_list) || | |
| 1163 | + if (!rq_list_empty(&iob.req_list) || | |
| 1164 | 1164 | READ_ONCE(req->iopoll_completed)) |
| 1165 | 1165 | break; |
| 1166 | 1166 | } |
| 1167 | 1167 | |
| 1168 | − if (!rq_list_empty(iob.req_list)) | |
| 1168 | + if (!rq_list_empty(&iob.req_list)) | |
| 1169 | 1169 | iob.complete(&iob); |
| 1170 | 1170 | else if (!pos) |
| 1171 | 1171 | return 0; |