History

block: add a rq_list type

Replace the semi-open coded request list helpers with a proper rq_list type that mirrors the bio_list and has head and tail pointers. Besides better type safety this actually allows to insert at the tail of the list, which will be useful soon.

Christoph Hellwig committed 2 years ago

Commit a3396b91 parent e8225ab11 files · +104 −88Browse files

Unified · Split

11 files
  1. block/blk-core.c +3 −3
  2. block/blk-merge.c +1 −1
  3. block/blk-mq.c +19 −21
  4. block/blk-mq.h +1 −1
  5. drivers/block/null_blk/main.c +4 −5
  6. drivers/block/virtio_blk.c +6 −7
  7. drivers/nvme/host/apple.c +1 −1
  8. drivers/nvme/host/pci.c +7 −8
  9. include/linux/blk-mq.h +52 −36
  10. include/linux/blkdev.h +8 −3
  11. io_uring/rw.c +2 −2
block/blk-core.c+3−3
@@ -1120,8 +1120,8 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios) · 1119 unmodified lines
1120 return;1120 return;
1121 1121
1122 plug->cur_ktime = 0;1122 plug->cur_ktime = 0;
1123 plug->mq_list = NULL;1123 rq_list_init(&plug->mq_list);
1124 plug->cached_rq = NULL;1124 rq_list_init(&plug->cached_rqs);
1125 plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);1125 plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);
1126 plug->rq_count = 0;1126 plug->rq_count = 0;
1127 plug->multiple_queues = false;1127 plug->multiple_queues = false;
@@ -1217,7 +1217,7 @@ void __blk_flush_plug(struct blk_plug *plug, bool from_schedule) · 89 unmodified lines
1217 * queue for cached requests, we don't want a blocked task holding1217 * queue for cached requests, we don't want a blocked task holding
1218 * up a queue freeze/quiesce event.1218 * up a queue freeze/quiesce event.
1219 */1219 */
1220 if (unlikely(!rq_list_empty(plug->cached_rq)))1220 if (unlikely(!rq_list_empty(&plug->cached_rqs)))
1221 blk_mq_free_plug_rqs(plug);1221 blk_mq_free_plug_rqs(plug);
1222 1222
1223 plug->cur_ktime = 0;1223 plug->cur_ktime = 0;
block/blk-merge.c+1−1
@@ -1179,7 +1179,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, · 1178 unmodified lines
1179 struct blk_plug *plug = current->plug;1179 struct blk_plug *plug = current->plug;
1180 struct request *rq;1180 struct request *rq;
1181 1181
1182 if (!plug || rq_list_empty(plug->mq_list))1182 if (!plug || rq_list_empty(&plug->mq_list))
1183 return false;1183 return false;
1184 1184
1185 rq_list_for_each(&plug->mq_list, rq) {1185 rq_list_for_each(&plug->mq_list, rq) {
block/blk-mq.c+19−21
@@ -478,7 +478,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) · 477 unmodified lines
478 prefetch(tags->static_rqs[tag]);478 prefetch(tags->static_rqs[tag]);
479 tag_mask &= ~(1UL << i);479 tag_mask &= ~(1UL << i);
480 rq = blk_mq_rq_ctx_init(data, tags, tag);480 rq = blk_mq_rq_ctx_init(data, tags, tag);
481 rq_list_add(data->cached_rq, rq);481 rq_list_add_head(data->cached_rqs, rq);
482 nr++;482 nr++;
483 }483 }
484 if (!(data->rq_flags & RQF_SCHED_TAGS))484 if (!(data->rq_flags & RQF_SCHED_TAGS))
@@ -487,7 +487,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) · 2 unmodified lines
487 percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);487 percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
488 data->nr_tags -= nr;488 data->nr_tags -= nr;
489 489
490 return rq_list_pop(data->cached_rq);490 return rq_list_pop(data->cached_rqs);
491}491}
492 492
493static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)493static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
@@ -584,7 +584,7 @@ static struct request *blk_mq_rq_cache_fill(struct request_queue *q, · 90 unmodified lines
584 .flags = flags,584 .flags = flags,
585 .cmd_flags = opf,585 .cmd_flags = opf,
586 .nr_tags = plug->nr_ios,586 .nr_tags = plug->nr_ios,
587 .cached_rq = &plug->cached_rq,587 .cached_rqs = &plug->cached_rqs,
588 };588 };
589 struct request *rq;589 struct request *rq;
590 590
@@ -609,14 +609,14 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, · 18 unmodified lines
609 if (!plug)609 if (!plug)
610 return NULL;610 return NULL;
611 611
612 if (rq_list_empty(plug->cached_rq)) {612 if (rq_list_empty(&plug->cached_rqs)) {
613 if (plug->nr_ios == 1)613 if (plug->nr_ios == 1)
614 return NULL;614 return NULL;
615 rq = blk_mq_rq_cache_fill(q, plug, opf, flags);615 rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
616 if (!rq)616 if (!rq)
617 return NULL;617 return NULL;
618 } else {618 } else {
619 rq = rq_list_peek(&plug->cached_rq);619 rq = rq_list_peek(&plug->cached_rqs);
620 if (!rq || rq->q != q)620 if (!rq || rq->q != q)
621 return NULL;621 return NULL;
622 622
@@ -625,7 +625,7 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, · 2 unmodified lines
625 if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))625 if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
626 return NULL;626 return NULL;
627 627
628 plug->cached_rq = rq_list_next(rq);628 rq_list_pop(&plug->cached_rqs);
629 blk_mq_rq_time_init(rq, blk_time_get_ns());629 blk_mq_rq_time_init(rq, blk_time_get_ns());
630 }630 }
631 631
@@ -802,7 +802,7 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug) · 170 unmodified lines
802{802{
803 struct request *rq;803 struct request *rq;
804 804
805 while ((rq = rq_list_pop(&plug->cached_rq)) != NULL)805 while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL)
806 blk_mq_free_request(rq);806 blk_mq_free_request(rq);
807}807}
808 808
@@ -1392,8 +1392,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) · 583 unmodified lines
1392 */1392 */
1393 if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))1393 if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
1394 plug->has_elevator = true;1394 plug->has_elevator = true;
1395 rq->rq_next = NULL;1395 rq_list_add_head(&plug->mq_list, rq);
1396 rq_list_add(&plug->mq_list, rq);
1397 plug->rq_count++;1396 plug->rq_count++;
1398}1397}
1399 1398
@@ -2785,7 +2784,7 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug) · 1385 unmodified lines
2785 blk_status_t ret = BLK_STS_OK;2784 blk_status_t ret = BLK_STS_OK;
2786 2785
2787 while ((rq = rq_list_pop(&plug->mq_list))) {2786 while ((rq = rq_list_pop(&plug->mq_list))) {
2788 bool last = rq_list_empty(plug->mq_list);2787 bool last = rq_list_empty(&plug->mq_list);
2789 2788
2790 if (hctx != rq->mq_hctx) {2789 if (hctx != rq->mq_hctx) {
2791 if (hctx) {2790 if (hctx) {
@@ -2828,8 +2827,7 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) · 36 unmodified lines
2828{2827{
2829 struct blk_mq_hw_ctx *this_hctx = NULL;2828 struct blk_mq_hw_ctx *this_hctx = NULL;
2830 struct blk_mq_ctx *this_ctx = NULL;2829 struct blk_mq_ctx *this_ctx = NULL;
2831 struct request *requeue_list = NULL;2830 struct rq_list requeue_list = {};
2832 struct request **requeue_lastp = &requeue_list;
2833 unsigned int depth = 0;2831 unsigned int depth = 0;
2834 bool is_passthrough = false;2832 bool is_passthrough = false;
2835 LIST_HEAD(list);2833 LIST_HEAD(list);
@@ -2843,12 +2841,12 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) · 7 unmodified lines
2843 is_passthrough = blk_rq_is_passthrough(rq);2841 is_passthrough = blk_rq_is_passthrough(rq);
2844 } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||2842 } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||
2845 is_passthrough != blk_rq_is_passthrough(rq)) {2843 is_passthrough != blk_rq_is_passthrough(rq)) {
2846 rq_list_add_tail(&requeue_lastp, rq);2844 rq_list_add_tail(&requeue_list, rq);
2847 continue;2845 continue;
2848 }2846 }
2849 list_add(&rq->queuelist, &list);2847 list_add(&rq->queuelist, &list);
2850 depth++;2848 depth++;
2851 } while (!rq_list_empty(plug->mq_list));2849 } while (!rq_list_empty(&plug->mq_list));
2852 2850
2853 plug->mq_list = requeue_list;2851 plug->mq_list = requeue_list;
2854 trace_block_unplug(this_hctx->queue, depth, !from_sched);2852 trace_block_unplug(this_hctx->queue, depth, !from_sched);
@@ -2903,19 +2901,19 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) · 48 unmodified lines
2903 if (q->mq_ops->queue_rqs) {2901 if (q->mq_ops->queue_rqs) {
2904 blk_mq_run_dispatch_ops(q,2902 blk_mq_run_dispatch_ops(q,
2905 __blk_mq_flush_plug_list(q, plug));2903 __blk_mq_flush_plug_list(q, plug));
2906 if (rq_list_empty(plug->mq_list))2904 if (rq_list_empty(&plug->mq_list))
2907 return;2905 return;
2908 }2906 }
2909 2907
2910 blk_mq_run_dispatch_ops(q,2908 blk_mq_run_dispatch_ops(q,
2911 blk_mq_plug_issue_direct(plug));2909 blk_mq_plug_issue_direct(plug));
2912 if (rq_list_empty(plug->mq_list))2910 if (rq_list_empty(&plug->mq_list))
2913 return;2911 return;
2914 }2912 }
2915 2913
2916 do {2914 do {
2917 blk_mq_dispatch_plug_list(plug, from_schedule);2915 blk_mq_dispatch_plug_list(plug, from_schedule);
2918 } while (!rq_list_empty(plug->mq_list));2916 } while (!rq_list_empty(&plug->mq_list));
2919}2917}
2920 2918
2921static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,2919static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
@@ -2980,7 +2978,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, · 58 unmodified lines
2980 if (plug) {2978 if (plug) {
2981 data.nr_tags = plug->nr_ios;2979 data.nr_tags = plug->nr_ios;
2982 plug->nr_ios = 1;2980 plug->nr_ios = 1;
2983 data.cached_rq = &plug->cached_rq;2981 data.cached_rqs = &plug->cached_rqs;
2984 }2982 }
2985 2983
2986 rq = __blk_mq_alloc_requests(&data);2984 rq = __blk_mq_alloc_requests(&data);
@@ -3003,7 +3001,7 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, · 16 unmodified lines
3003 3001
3004 if (!plug)3002 if (!plug)
3005 return NULL;3003 return NULL;
3006 rq = rq_list_peek(&plug->cached_rq);3004 rq = rq_list_peek(&plug->cached_rqs);
3007 if (!rq || rq->q != q)3005 if (!rq || rq->q != q)
3008 return NULL;3006 return NULL;
3009 if (type != rq->mq_hctx->type &&3007 if (type != rq->mq_hctx->type &&
@@ -3017,14 +3015,14 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, · 7 unmodified lines
3017static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,3015static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,
3018 struct bio *bio)3016 struct bio *bio)
3019{3017{
3020 WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq);3018 if (rq_list_pop(&plug->cached_rqs) != rq)
3019 WARN_ON_ONCE(1);
3021 3020
3022 /*3021 /*
3023 * If any qos ->throttle() end up blocking, we will have flushed the3022 * If any qos ->throttle() end up blocking, we will have flushed the
3024 * plug and hence killed the cached_rq list as well. Pop this entry3023 * plug and hence killed the cached_rq list as well. Pop this entry
3025 * before we throttle.3024 * before we throttle.
3026 */3025 */
3027 plug->cached_rq = rq_list_next(rq);
3028 rq_qos_throttle(rq->q, bio);3026 rq_qos_throttle(rq->q, bio);
3029 3027
3030 blk_mq_rq_time_init(rq, blk_time_get_ns());3028 blk_mq_rq_time_init(rq, blk_time_get_ns());
block/blk-mq.h+1−1
@@ -155,7 +155,7 @@ struct blk_mq_alloc_data { · 154 unmodified lines
155 155
156 /* allocate multiple requests/tags in one go */156 /* allocate multiple requests/tags in one go */
157 unsigned int nr_tags;157 unsigned int nr_tags;
158 struct request **cached_rq;158 struct rq_list *cached_rqs;
159 159
160 /* input & output parameter */160 /* input & output parameter */
161 struct blk_mq_ctx *ctx;161 struct blk_mq_ctx *ctx;
drivers/block/null_blk/main.c+4−5
@@ -1638,10 +1638,9 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, · 1637 unmodified lines
1638 return BLK_STS_OK;1638 return BLK_STS_OK;
1639}1639}
1640 1640
1641static void null_queue_rqs(struct request **rqlist)1641static void null_queue_rqs(struct rq_list *rqlist)
1642{1642{
1643 struct request *requeue_list = NULL;1643 struct rq_list requeue_list = {};
1644 struct request **requeue_lastp = &requeue_list;
1645 struct blk_mq_queue_data bd = { };1644 struct blk_mq_queue_data bd = { };
1646 blk_status_t ret;1645 blk_status_t ret;
1647 1646
@@ -1651,8 +1650,8 @@ static void null_queue_rqs(struct request **rqlist) · 3 unmodified lines
1651 bd.rq = rq;1650 bd.rq = rq;
1652 ret = null_queue_rq(rq->mq_hctx, &bd);1651 ret = null_queue_rq(rq->mq_hctx, &bd);
1653 if (ret != BLK_STS_OK)1652 if (ret != BLK_STS_OK)
1654 rq_list_add_tail(&requeue_lastp, rq);1653 rq_list_add_tail(&requeue_list, rq);
1655 } while (!rq_list_empty(*rqlist));1654 } while (!rq_list_empty(rqlist));
1656 1655
1657 *rqlist = requeue_list;1656 *rqlist = requeue_list;
1658}1657}
drivers/block/virtio_blk.c+6−7
@@ -472,7 +472,7 @@ static bool virtblk_prep_rq_batch(struct request *req) · 471 unmodified lines
472}472}
473 473
474static void virtblk_add_req_batch(struct virtio_blk_vq *vq,474static void virtblk_add_req_batch(struct virtio_blk_vq *vq,
475 struct request **rqlist)475 struct rq_list *rqlist)
476{476{
477 struct request *req;477 struct request *req;
478 unsigned long flags;478 unsigned long flags;
@@ -499,11 +499,10 @@ static void virtblk_add_req_batch(struct virtio_blk_vq *vq, · 20 unmodified lines
499 virtqueue_notify(vq->vq);499 virtqueue_notify(vq->vq);
500}500}
501 501
502static void virtio_queue_rqs(struct request **rqlist)502static void virtio_queue_rqs(struct rq_list *rqlist)
503{503{
504 struct request *submit_list = NULL;504 struct rq_list submit_list = { };
505 struct request *requeue_list = NULL;505 struct rq_list requeue_list = { };
506 struct request **requeue_lastp = &requeue_list;
507 struct virtio_blk_vq *vq = NULL;506 struct virtio_blk_vq *vq = NULL;
508 struct request *req;507 struct request *req;
509 508
@@ -515,9 +514,9 @@ static void virtio_queue_rqs(struct request **rqlist) · 5 unmodified lines
515 vq = this_vq;514 vq = this_vq;
516 515
517 if (virtblk_prep_rq_batch(req))516 if (virtblk_prep_rq_batch(req))
518 rq_list_add(&submit_list, req); /* reverse order */517 rq_list_add_head(&submit_list, req); /* reverse order */
519 else518 else
520 rq_list_add_tail(&requeue_lastp, req);519 rq_list_add_tail(&requeue_list, req);
521 }520 }
522 521
523 if (vq)522 if (vq)
drivers/nvme/host/apple.c+1−1
@@ -649,7 +649,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) · 648 unmodified lines
649 649
650 found = apple_nvme_poll_cq(q, &iob);650 found = apple_nvme_poll_cq(q, &iob);
651 651
652 if (!rq_list_empty(iob.req_list))652 if (!rq_list_empty(&iob.req_list))
653 apple_nvme_complete_batch(&iob);653 apple_nvme_complete_batch(&iob);
654 654
655 return found;655 return found;
drivers/nvme/host/pci.c+7−8
@@ -904,7 +904,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, · 903 unmodified lines
904 return BLK_STS_OK;904 return BLK_STS_OK;
905}905}
906 906
907static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist)907static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
908{908{
909 struct request *req;909 struct request *req;
910 910
@@ -932,11 +932,10 @@ static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req) · 21 unmodified lines
932 return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;932 return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
933}933}
934 934
935static void nvme_queue_rqs(struct request **rqlist)935static void nvme_queue_rqs(struct rq_list *rqlist)
936{936{
937 struct request *submit_list = NULL;937 struct rq_list submit_list = { };
938 struct request *requeue_list = NULL;938 struct rq_list requeue_list = { };
939 struct request **requeue_lastp = &requeue_list;
940 struct nvme_queue *nvmeq = NULL;939 struct nvme_queue *nvmeq = NULL;
941 struct request *req;940 struct request *req;
942 941
@@ -946,9 +945,9 @@ static void nvme_queue_rqs(struct request **rqlist) · 3 unmodified lines
946 nvmeq = req->mq_hctx->driver_data;945 nvmeq = req->mq_hctx->driver_data;
947 946
948 if (nvme_prep_rq_batch(nvmeq, req))947 if (nvme_prep_rq_batch(nvmeq, req))
949 rq_list_add(&submit_list, req); /* reverse order */948 rq_list_add_head(&submit_list, req); /* reverse order */
950 else949 else
951 rq_list_add_tail(&requeue_lastp, req);950 rq_list_add_tail(&requeue_list, req);
952 }951 }
953 952
954 if (nvmeq)953 if (nvmeq)
@@ -1080,7 +1079,7 @@ static irqreturn_t nvme_irq(int irq, void *data) · 125 unmodified lines
1080 DEFINE_IO_COMP_BATCH(iob);1079 DEFINE_IO_COMP_BATCH(iob);
1081 1080
1082 if (nvme_poll_cq(nvmeq, &iob)) {1081 if (nvme_poll_cq(nvmeq, &iob)) {
1083 if (!rq_list_empty(iob.req_list))1082 if (!rq_list_empty(&iob.req_list))
1084 nvme_pci_complete_batch(&iob);1083 nvme_pci_complete_batch(&iob);
1085 return IRQ_HANDLED;1084 return IRQ_HANDLED;
1086 }1085 }
include/linux/blk-mq.h+52−36
@@ -229,44 +229,60 @@ static inline unsigned short req_get_ioprio(struct request *req) · 228 unmodified lines
229#define rq_dma_dir(rq) \229#define rq_dma_dir(rq) \
230 (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)230 (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
231 231
232#define rq_list_add(listptr, rq) do { \232static inline int rq_list_empty(const struct rq_list *rl)
233 (rq)->rq_next = *(listptr); \233{
234 *(listptr) = rq; \234 return rl->head == NULL;
235} while (0)235}
236
237#define rq_list_add_tail(lastpptr, rq) do { \
238 (rq)->rq_next = NULL; \
239 **(lastpptr) = rq; \
240 *(lastpptr) = &rq->rq_next; \
241} while (0)
242
243#define rq_list_pop(listptr) \
244({ \
245 struct request *__req = NULL; \
246 if ((listptr) && *(listptr)) { \
247 __req = *(listptr); \
248 *(listptr) = __req->rq_next; \
249 } \
250 __req; \
251})
252 236
253#define rq_list_peek(listptr) \237static inline void rq_list_init(struct rq_list *rl)
254({ \238{
255 struct request *__req = NULL; \239 rl->head = NULL;
256 if ((listptr) && *(listptr)) \240 rl->tail = NULL;
257 __req = *(listptr); \241}
258 __req; \242
259})243static inline void rq_list_add_tail(struct rq_list *rl, struct request *rq)
244{
245 rq->rq_next = NULL;
246 if (rl->tail)
247 rl->tail->rq_next = rq;
248 else
249 rl->head = rq;
250 rl->tail = rq;
251}
252
253static inline void rq_list_add_head(struct rq_list *rl, struct request *rq)
254{
255 rq->rq_next = rl->head;
256 rl->head = rq;
257 if (!rl->tail)
258 rl->tail = rq;
259}
260
261static inline struct request *rq_list_pop(struct rq_list *rl)
262{
263 struct request *rq = rl->head;
264
265 if (rq) {
266 rl->head = rl->head->rq_next;
267 if (!rl->head)
268 rl->tail = NULL;
269 rq->rq_next = NULL;
270 }
271
272 return rq;
273}
260 274
261#define rq_list_for_each(listptr, pos) \275static inline struct request *rq_list_peek(struct rq_list *rl)
262 for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos))276{
277 return rl->head;
278}
263 279
264#define rq_list_for_each_safe(listptr, pos, nxt) \280#define rq_list_for_each(rl, pos) \
265 for (pos = rq_list_peek((listptr)), nxt = rq_list_next(pos); \281 for (pos = rq_list_peek((rl)); (pos); pos = pos->rq_next)
266 pos; pos = nxt, nxt = pos ? rq_list_next(pos) : NULL)
267 282
268#define rq_list_next(rq) (rq)->rq_next283#define rq_list_for_each_safe(rl, pos, nxt) \
269#define rq_list_empty(list) ((list) == (struct request *) NULL)284 for (pos = rq_list_peek((rl)), nxt = pos->rq_next; \
285 pos; pos = nxt, nxt = pos ? pos->rq_next : NULL)
270 286
271/**287/**
272 * enum blk_eh_timer_return - How the timeout handler should proceed288 * enum blk_eh_timer_return - How the timeout handler should proceed
@@ -559,7 +575,7 @@ struct blk_mq_ops { · 286 unmodified lines
559 * empty the @rqlist completely, then the rest will be queued575 * empty the @rqlist completely, then the rest will be queued
560 * individually by the block layer upon return.576 * individually by the block layer upon return.
561 */577 */
562 void (*queue_rqs)(struct request **rqlist);578 void (*queue_rqs)(struct rq_list *rqlist);
563 579
564 /**580 /**
565 * @get_budget: Reserve budget before queue request, once .queue_rq is581 * @get_budget: Reserve budget before queue request, once .queue_rq is
@@ -868,7 +884,7 @@ static inline bool blk_mq_add_to_batch(struct request *req, · 302 unmodified lines
868 else if (iob->complete != complete)884 else if (iob->complete != complete)
869 return false;885 return false;
870 iob->need_ts |= blk_mq_need_time_stamp(req);886 iob->need_ts |= blk_mq_need_time_stamp(req);
871 rq_list_add(&iob->req_list, req);887 rq_list_add_head(&iob->req_list, req);
872 return true;888 return true;
873}889}
874 890
include/linux/blkdev.h+8−3
@@ -1007,6 +1007,11 @@ extern void blk_put_queue(struct request_queue *); · 1006 unmodified lines
1007void blk_mark_disk_dead(struct gendisk *disk);1007void blk_mark_disk_dead(struct gendisk *disk);
1008 1008
1009#ifdef CONFIG_BLOCK1009#ifdef CONFIG_BLOCK
1010struct rq_list {
1011 struct request *head;
1012 struct request *tail;
1013};
1014
1010/*1015/*
1011 * blk_plug permits building a queue of related requests by holding the I/O1016 * blk_plug permits building a queue of related requests by holding the I/O
1012 * fragments for a short period. This allows merging of sequential requests1017 * fragments for a short period. This allows merging of sequential requests
@@ -1019,10 +1024,10 @@ void blk_mark_disk_dead(struct gendisk *disk); · 6 unmodified lines
1019 * blk_flush_plug() is called.1024 * blk_flush_plug() is called.
1020 */1025 */
1021struct blk_plug {1026struct blk_plug {
1022 struct request *mq_list; /* blk-mq requests */1027 struct rq_list mq_list; /* blk-mq requests */
1023 1028
1024 /* if ios_left is > 1, we can batch tag/rq allocations */1029 /* if ios_left is > 1, we can batch tag/rq allocations */
1025 struct request *cached_rq;1030 struct rq_list cached_rqs;
1026 u64 cur_ktime;1031 u64 cur_ktime;
1027 unsigned short nr_ios;1032 unsigned short nr_ios;
1028 1033
@@ -1684,7 +1689,7 @@ int bdev_thaw(struct block_device *bdev); · 655 unmodified lines
1684void bdev_fput(struct file *bdev_file);1689void bdev_fput(struct file *bdev_file);
1685 1690
1686struct io_comp_batch {1691struct io_comp_batch {
1687 struct request *req_list;1692 struct rq_list req_list;
1688 bool need_ts;1693 bool need_ts;
1689 void (*complete)(struct io_comp_batch *);1694 void (*complete)(struct io_comp_batch *);
1690};1695};
io_uring/rw.c+2−2
@@ -1160,12 +1160,12 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 1159 unmodified lines
1160 poll_flags |= BLK_POLL_ONESHOT;1160 poll_flags |= BLK_POLL_ONESHOT;
1161 1161
1162 /* iopoll may have completed current req */1162 /* iopoll may have completed current req */
1163 if (!rq_list_empty(iob.req_list) ||1163 if (!rq_list_empty(&iob.req_list) ||
1164 READ_ONCE(req->iopoll_completed))1164 READ_ONCE(req->iopoll_completed))
1165 break;1165 break;
1166 }1166 }
1167 1167
1168 if (!rq_list_empty(iob.req_list))1168 if (!rq_list_empty(&iob.req_list))
1169 iob.complete(&iob);1169 iob.complete(&iob);
1170 else if (!pos)1170 else if (!pos)
1171 return 0;1171 return 0;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t