History

block: add a rq_list type

Replace the semi-open coded request list helpers with a proper rq_list type that mirrors the bio_list and has head and tail pointers. Besides better type safety this actually allows to insert at the tail of the list, which will be useful soon.

Christoph Hellwig committed 2 years ago

Commit a3396b91 parent e8225ab11 files · +104 −88Browse files

Unified · Split

11 files
  1. block/blk-core.c +3 −3
  2. block/blk-merge.c +1 −1
  3. block/blk-mq.c +19 −21
  4. block/blk-mq.h +1 −1
  5. drivers/block/null_blk/main.c +4 −5
  6. drivers/block/virtio_blk.c +6 −7
  7. drivers/nvme/host/apple.c +1 −1
  8. drivers/nvme/host/pci.c +7 −8
  9. include/linux/blk-mq.h +52 −36
  10. include/linux/blkdev.h +8 −3
  11. io_uring/rw.c +2 −2
block/blk-core.c+3−3
@@ -1120,8 +1120,8 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios) · 1119 unmodified lines
11201120 return;
11211121
11221122 plug->cur_ktime = 0;
1123− plug->mq_list = NULL;
1124− plug->cached_rq = NULL;
1123+ rq_list_init(&plug->mq_list);
1124+ rq_list_init(&plug->cached_rqs);
11251125 plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);
11261126 plug->rq_count = 0;
11271127 plug->multiple_queues = false;
@@ -1217,7 +1217,7 @@ void __blk_flush_plug(struct blk_plug *plug, bool from_schedule) · 89 unmodified lines
12171217 * queue for cached requests, we don't want a blocked task holding
12181218 * up a queue freeze/quiesce event.
12191219 */
1220− if (unlikely(!rq_list_empty(plug->cached_rq)))
1220+ if (unlikely(!rq_list_empty(&plug->cached_rqs)))
12211221 blk_mq_free_plug_rqs(plug);
12221222
12231223 plug->cur_ktime = 0;
block/blk-merge.c+1−1
@@ -1179,7 +1179,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, · 1178 unmodified lines
11791179 struct blk_plug *plug = current->plug;
11801180 struct request *rq;
11811181
1182− if (!plug || rq_list_empty(plug->mq_list))
1182+ if (!plug || rq_list_empty(&plug->mq_list))
11831183 return false;
11841184
11851185 rq_list_for_each(&plug->mq_list, rq) {
block/blk-mq.c+19−21
@@ -478,7 +478,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) · 477 unmodified lines
478478 prefetch(tags->static_rqs[tag]);
479479 tag_mask &= ~(1UL << i);
480480 rq = blk_mq_rq_ctx_init(data, tags, tag);
481− rq_list_add(data->cached_rq, rq);
481+ rq_list_add_head(data->cached_rqs, rq);
482482 nr++;
483483 }
484484 if (!(data->rq_flags & RQF_SCHED_TAGS))
@@ -487,7 +487,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) · 2 unmodified lines
487487 percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
488488 data->nr_tags -= nr;
489489
490− return rq_list_pop(data->cached_rq);
490+ return rq_list_pop(data->cached_rqs);
491491 }
492492
493493 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
@@ -584,7 +584,7 @@ static struct request *blk_mq_rq_cache_fill(struct request_queue *q, · 90 unmodified lines
584584 .flags = flags,
585585 .cmd_flags = opf,
586586 .nr_tags = plug->nr_ios,
587− .cached_rq = &plug->cached_rq,
587+ .cached_rqs = &plug->cached_rqs,
588588 };
589589 struct request *rq;
590590
@@ -609,14 +609,14 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, · 18 unmodified lines
609609 if (!plug)
610610 return NULL;
611611
612− if (rq_list_empty(plug->cached_rq)) {
612+ if (rq_list_empty(&plug->cached_rqs)) {
613613 if (plug->nr_ios == 1)
614614 return NULL;
615615 rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
616616 if (!rq)
617617 return NULL;
618618 } else {
619− rq = rq_list_peek(&plug->cached_rq);
619+ rq = rq_list_peek(&plug->cached_rqs);
620620 if (!rq || rq->q != q)
621621 return NULL;
622622
@@ -625,7 +625,7 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, · 2 unmodified lines
625625 if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
626626 return NULL;
627627
628− plug->cached_rq = rq_list_next(rq);
628+ rq_list_pop(&plug->cached_rqs);
629629 blk_mq_rq_time_init(rq, blk_time_get_ns());
630630 }
631631
@@ -802,7 +802,7 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug) · 170 unmodified lines
802802 {
803803 struct request *rq;
804804
805− while ((rq = rq_list_pop(&plug->cached_rq)) != NULL)
805+ while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL)
806806 blk_mq_free_request(rq);
807807 }
808808
@@ -1392,8 +1392,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) · 583 unmodified lines
13921392 */
13931393 if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
13941394 plug->has_elevator = true;
1395− rq->rq_next = NULL;
1396− rq_list_add(&plug->mq_list, rq);
1395+ rq_list_add_head(&plug->mq_list, rq);
13971396 plug->rq_count++;
13981397 }
13991398
@@ -2785,7 +2784,7 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug) · 1385 unmodified lines
27852784 blk_status_t ret = BLK_STS_OK;
27862785
27872786 while ((rq = rq_list_pop(&plug->mq_list))) {
2788− bool last = rq_list_empty(plug->mq_list);
2787+ bool last = rq_list_empty(&plug->mq_list);
27892788
27902789 if (hctx != rq->mq_hctx) {
27912790 if (hctx) {
@@ -2828,8 +2827,7 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) · 36 unmodified lines
28282827 {
28292828 struct blk_mq_hw_ctx *this_hctx = NULL;
28302829 struct blk_mq_ctx *this_ctx = NULL;
2831− struct request *requeue_list = NULL;
2832− struct request **requeue_lastp = &requeue_list;
2830+ struct rq_list requeue_list = {};
28332831 unsigned int depth = 0;
28342832 bool is_passthrough = false;
28352833 LIST_HEAD(list);
@@ -2843,12 +2841,12 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) · 7 unmodified lines
28432841 is_passthrough = blk_rq_is_passthrough(rq);
28442842 } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||
28452843 is_passthrough != blk_rq_is_passthrough(rq)) {
2846− rq_list_add_tail(&requeue_lastp, rq);
2844+ rq_list_add_tail(&requeue_list, rq);
28472845 continue;
28482846 }
28492847 list_add(&rq->queuelist, &list);
28502848 depth++;
2851− } while (!rq_list_empty(plug->mq_list));
2849+ } while (!rq_list_empty(&plug->mq_list));
28522850
28532851 plug->mq_list = requeue_list;
28542852 trace_block_unplug(this_hctx->queue, depth, !from_sched);
@@ -2903,19 +2901,19 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) · 48 unmodified lines
29032901 if (q->mq_ops->queue_rqs) {
29042902 blk_mq_run_dispatch_ops(q,
29052903 __blk_mq_flush_plug_list(q, plug));
2906− if (rq_list_empty(plug->mq_list))
2904+ if (rq_list_empty(&plug->mq_list))
29072905 return;
29082906 }
29092907
29102908 blk_mq_run_dispatch_ops(q,
29112909 blk_mq_plug_issue_direct(plug));
2912− if (rq_list_empty(plug->mq_list))
2910+ if (rq_list_empty(&plug->mq_list))
29132911 return;
29142912 }
29152913
29162914 do {
29172915 blk_mq_dispatch_plug_list(plug, from_schedule);
2918− } while (!rq_list_empty(plug->mq_list));
2916+ } while (!rq_list_empty(&plug->mq_list));
29192917 }
29202918
29212919 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
@@ -2980,7 +2978,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, · 58 unmodified lines
29802978 if (plug) {
29812979 data.nr_tags = plug->nr_ios;
29822980 plug->nr_ios = 1;
2983− data.cached_rq = &plug->cached_rq;
2981+ data.cached_rqs = &plug->cached_rqs;
29842982 }
29852983
29862984 rq = __blk_mq_alloc_requests(&data);
@@ -3003,7 +3001,7 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, · 16 unmodified lines
30033001
30043002 if (!plug)
30053003 return NULL;
3006− rq = rq_list_peek(&plug->cached_rq);
3004+ rq = rq_list_peek(&plug->cached_rqs);
30073005 if (!rq || rq->q != q)
30083006 return NULL;
30093007 if (type != rq->mq_hctx->type &&
@@ -3017,14 +3015,14 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, · 7 unmodified lines
30173015 static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,
30183016 struct bio *bio)
30193017 {
3020− WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq);
3018+ if (rq_list_pop(&plug->cached_rqs) != rq)
3019+ WARN_ON_ONCE(1);
30213020
30223021 /*
30233022 * If any qos ->throttle() end up blocking, we will have flushed the
30243023 * plug and hence killed the cached_rq list as well. Pop this entry
30253024 * before we throttle.
30263025 */
3027− plug->cached_rq = rq_list_next(rq);
30283026 rq_qos_throttle(rq->q, bio);
30293027
30303028 blk_mq_rq_time_init(rq, blk_time_get_ns());
block/blk-mq.h+1−1
@@ -155,7 +155,7 @@ struct blk_mq_alloc_data { · 154 unmodified lines
155155
156156 /* allocate multiple requests/tags in one go */
157157 unsigned int nr_tags;
158− struct request **cached_rq;
158+ struct rq_list *cached_rqs;
159159
160160 /* input & output parameter */
161161 struct blk_mq_ctx *ctx;
drivers/block/null_blk/main.c+4−5
@@ -1638,10 +1638,9 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, · 1637 unmodified lines
16381638 return BLK_STS_OK;
16391639 }
16401640
1641−static void null_queue_rqs(struct request **rqlist)
1641+static void null_queue_rqs(struct rq_list *rqlist)
16421642 {
1643− struct request *requeue_list = NULL;
1644− struct request **requeue_lastp = &requeue_list;
1643+ struct rq_list requeue_list = {};
16451644 struct blk_mq_queue_data bd = { };
16461645 blk_status_t ret;
16471646
@@ -1651,8 +1650,8 @@ static void null_queue_rqs(struct request **rqlist) · 3 unmodified lines
16511650 bd.rq = rq;
16521651 ret = null_queue_rq(rq->mq_hctx, &bd);
16531652 if (ret != BLK_STS_OK)
1654− rq_list_add_tail(&requeue_lastp, rq);
1655− } while (!rq_list_empty(*rqlist));
1653+ rq_list_add_tail(&requeue_list, rq);
1654+ } while (!rq_list_empty(rqlist));
16561655
16571656 *rqlist = requeue_list;
16581657 }
drivers/block/virtio_blk.c+6−7
@@ -472,7 +472,7 @@ static bool virtblk_prep_rq_batch(struct request *req) · 471 unmodified lines
472472 }
473473
474474 static void virtblk_add_req_batch(struct virtio_blk_vq *vq,
475− struct request **rqlist)
475+ struct rq_list *rqlist)
476476 {
477477 struct request *req;
478478 unsigned long flags;
@@ -499,11 +499,10 @@ static void virtblk_add_req_batch(struct virtio_blk_vq *vq, · 20 unmodified lines
499499 virtqueue_notify(vq->vq);
500500 }
501501
502−static void virtio_queue_rqs(struct request **rqlist)
502+static void virtio_queue_rqs(struct rq_list *rqlist)
503503 {
504− struct request *submit_list = NULL;
505− struct request *requeue_list = NULL;
506− struct request **requeue_lastp = &requeue_list;
504+ struct rq_list submit_list = { };
505+ struct rq_list requeue_list = { };
507506 struct virtio_blk_vq *vq = NULL;
508507 struct request *req;
509508
@@ -515,9 +514,9 @@ static void virtio_queue_rqs(struct request **rqlist) · 5 unmodified lines
515514 vq = this_vq;
516515
517516 if (virtblk_prep_rq_batch(req))
518− rq_list_add(&submit_list, req); /* reverse order */
517+ rq_list_add_head(&submit_list, req); /* reverse order */
519518 else
520− rq_list_add_tail(&requeue_lastp, req);
519+ rq_list_add_tail(&requeue_list, req);
521520 }
522521
523522 if (vq)
drivers/nvme/host/apple.c+1−1
@@ -649,7 +649,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) · 648 unmodified lines
649649
650650 found = apple_nvme_poll_cq(q, &iob);
651651
652− if (!rq_list_empty(iob.req_list))
652+ if (!rq_list_empty(&iob.req_list))
653653 apple_nvme_complete_batch(&iob);
654654
655655 return found;
drivers/nvme/host/pci.c+7−8
@@ -904,7 +904,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, · 903 unmodified lines
904904 return BLK_STS_OK;
905905 }
906906
907−static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist)
907+static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
908908 {
909909 struct request *req;
910910
@@ -932,11 +932,10 @@ static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req) · 21 unmodified lines
932932 return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
933933 }
934934
935−static void nvme_queue_rqs(struct request **rqlist)
935+static void nvme_queue_rqs(struct rq_list *rqlist)
936936 {
937− struct request *submit_list = NULL;
938− struct request *requeue_list = NULL;
939− struct request **requeue_lastp = &requeue_list;
937+ struct rq_list submit_list = { };
938+ struct rq_list requeue_list = { };
940939 struct nvme_queue *nvmeq = NULL;
941940 struct request *req;
942941
@@ -946,9 +945,9 @@ static void nvme_queue_rqs(struct request **rqlist) · 3 unmodified lines
946945 nvmeq = req->mq_hctx->driver_data;
947946
948947 if (nvme_prep_rq_batch(nvmeq, req))
949− rq_list_add(&submit_list, req); /* reverse order */
948+ rq_list_add_head(&submit_list, req); /* reverse order */
950949 else
951− rq_list_add_tail(&requeue_lastp, req);
950+ rq_list_add_tail(&requeue_list, req);
952951 }
953952
954953 if (nvmeq)
@@ -1080,7 +1079,7 @@ static irqreturn_t nvme_irq(int irq, void *data) · 125 unmodified lines
10801079 DEFINE_IO_COMP_BATCH(iob);
10811080
10821081 if (nvme_poll_cq(nvmeq, &iob)) {
1083− if (!rq_list_empty(iob.req_list))
1082+ if (!rq_list_empty(&iob.req_list))
10841083 nvme_pci_complete_batch(&iob);
10851084 return IRQ_HANDLED;
10861085 }
include/linux/blk-mq.h+52−36
@@ -229,44 +229,60 @@ static inline unsigned short req_get_ioprio(struct request *req) · 228 unmodified lines
229229 #define rq_dma_dir(rq) \
230230 (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
231231
232−#define rq_list_add(listptr, rq) do { \
233− (rq)->rq_next = *(listptr); \
234− *(listptr) = rq; \
235−} while (0)
236−
237−#define rq_list_add_tail(lastpptr, rq) do { \
238− (rq)->rq_next = NULL; \
239− **(lastpptr) = rq; \
240− *(lastpptr) = &rq->rq_next; \
241−} while (0)
242−
243−#define rq_list_pop(listptr) \
244−({ \
245− struct request *__req = NULL; \
246− if ((listptr) && *(listptr)) { \
247− __req = *(listptr); \
248− *(listptr) = __req->rq_next; \
249− } \
250− __req; \
251−})
232+static inline int rq_list_empty(const struct rq_list *rl)
233+{
234+ return rl->head == NULL;
235+}
252236
253−#define rq_list_peek(listptr) \
254−({ \
255− struct request *__req = NULL; \
256− if ((listptr) && *(listptr)) \
257− __req = *(listptr); \
258− __req; \
259−})
237+static inline void rq_list_init(struct rq_list *rl)
238+{
239+ rl->head = NULL;
240+ rl->tail = NULL;
241+}
242+
243+static inline void rq_list_add_tail(struct rq_list *rl, struct request *rq)
244+{
245+ rq->rq_next = NULL;
246+ if (rl->tail)
247+ rl->tail->rq_next = rq;
248+ else
249+ rl->head = rq;
250+ rl->tail = rq;
251+}
252+
253+static inline void rq_list_add_head(struct rq_list *rl, struct request *rq)
254+{
255+ rq->rq_next = rl->head;
256+ rl->head = rq;
257+ if (!rl->tail)
258+ rl->tail = rq;
259+}
260+
261+static inline struct request *rq_list_pop(struct rq_list *rl)
262+{
263+ struct request *rq = rl->head;
264+
265+ if (rq) {
266+ rl->head = rl->head->rq_next;
267+ if (!rl->head)
268+ rl->tail = NULL;
269+ rq->rq_next = NULL;
270+ }
271+
272+ return rq;
273+}
260274
261−#define rq_list_for_each(listptr, pos) \
262− for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos))
275+static inline struct request *rq_list_peek(struct rq_list *rl)
276+{
277+ return rl->head;
278+}
263279
264−#define rq_list_for_each_safe(listptr, pos, nxt) \
265− for (pos = rq_list_peek((listptr)), nxt = rq_list_next(pos); \
266− pos; pos = nxt, nxt = pos ? rq_list_next(pos) : NULL)
280+#define rq_list_for_each(rl, pos) \
281+ for (pos = rq_list_peek((rl)); (pos); pos = pos->rq_next)
267282
268−#define rq_list_next(rq) (rq)->rq_next
269−#define rq_list_empty(list) ((list) == (struct request *) NULL)
283+#define rq_list_for_each_safe(rl, pos, nxt) \
284+ for (pos = rq_list_peek((rl)), nxt = pos->rq_next; \
285+ pos; pos = nxt, nxt = pos ? pos->rq_next : NULL)
270286
271287 /**
272288 * enum blk_eh_timer_return - How the timeout handler should proceed
@@ -559,7 +575,7 @@ struct blk_mq_ops { · 286 unmodified lines
559575 * empty the @rqlist completely, then the rest will be queued
560576 * individually by the block layer upon return.
561577 */
562− void (*queue_rqs)(struct request **rqlist);
578+ void (*queue_rqs)(struct rq_list *rqlist);
563579
564580 /**
565581 * @get_budget: Reserve budget before queue request, once .queue_rq is
@@ -868,7 +884,7 @@ static inline bool blk_mq_add_to_batch(struct request *req, · 302 unmodified lines
868884 else if (iob->complete != complete)
869885 return false;
870886 iob->need_ts |= blk_mq_need_time_stamp(req);
871− rq_list_add(&iob->req_list, req);
887+ rq_list_add_head(&iob->req_list, req);
872888 return true;
873889 }
874890
include/linux/blkdev.h+8−3
@@ -1007,6 +1007,11 @@ extern void blk_put_queue(struct request_queue *); · 1006 unmodified lines
10071007 void blk_mark_disk_dead(struct gendisk *disk);
10081008
10091009 #ifdef CONFIG_BLOCK
1010+struct rq_list {
1011+ struct request *head;
1012+ struct request *tail;
1013+};
1014+
10101015 /*
10111016 * blk_plug permits building a queue of related requests by holding the I/O
10121017 * fragments for a short period. This allows merging of sequential requests
@@ -1019,10 +1024,10 @@ void blk_mark_disk_dead(struct gendisk *disk); · 6 unmodified lines
10191024 * blk_flush_plug() is called.
10201025 */
10211026 struct blk_plug {
1022− struct request *mq_list; /* blk-mq requests */
1027+ struct rq_list mq_list; /* blk-mq requests */
10231028
10241029 /* if ios_left is > 1, we can batch tag/rq allocations */
1025− struct request *cached_rq;
1030+ struct rq_list cached_rqs;
10261031 u64 cur_ktime;
10271032 unsigned short nr_ios;
10281033
@@ -1684,7 +1689,7 @@ int bdev_thaw(struct block_device *bdev); · 655 unmodified lines
16841689 void bdev_fput(struct file *bdev_file);
16851690
16861691 struct io_comp_batch {
1687− struct request *req_list;
1692+ struct rq_list req_list;
16881693 bool need_ts;
16891694 void (*complete)(struct io_comp_batch *);
16901695 };
io_uring/rw.c+2−2
@@ -1160,12 +1160,12 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 1159 unmodified lines
11601160 poll_flags |= BLK_POLL_ONESHOT;
11611161
11621162 /* iopoll may have completed current req */
1163− if (!rq_list_empty(iob.req_list) ||
1163+ if (!rq_list_empty(&iob.req_list) ||
11641164 READ_ONCE(req->iopoll_completed))
11651165 break;
11661166 }
11671167
1168− if (!rq_list_empty(iob.req_list))
1168+ if (!rq_list_empty(&iob.req_list))
11691169 iob.complete(&iob);
11701170 else if (!pos)
11711171 return 0;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t