We call blk_mq_map_queue() a lot, at least two times for each request per IO, sometimes more. Since we now have an indirect call as well in that function. cache the mapping so we don't have to re-call blk_mq_map_queue() for the same request multiple times.
Unified · Split
| @@ -215,7 +215,7 @@ static void flush_end_io(struct request *flush_rq, blk_status_t error) · 214 unmodified lines | ||
| 215 | 215 | |
| 216 | 216 | /* release the tag's ownership to the req cloned from */ |
| 217 | 217 | spin_lock_irqsave(&fq->mq_flush_lock, flags); |
| 218 | − hctx = blk_mq_map_queue(q, flush_rq->cmd_flags, flush_rq->mq_ctx->cpu); | |
| 218 | + hctx = flush_rq->mq_hctx; | |
| 219 | 219 | if (!q->elevator) { |
| 220 | 220 | blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq); |
| 221 | 221 | flush_rq->tag = -1; |
| @@ -262,7 +262,6 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, · 40 unmodified lines | ||
| 262 | 262 | struct request *first_rq = |
| 263 | 263 | list_first_entry(pending, struct request, flush.list); |
| 264 | 264 | struct request *flush_rq = fq->flush_rq; |
| 265 | − struct blk_mq_hw_ctx *hctx; | |
| 266 | 265 | |
| 267 | 266 | /* C1 described at the top of this file */ |
| 268 | 267 | if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending)) |
| @@ -297,13 +296,12 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, · 28 unmodified lines | ||
| 297 | 296 | * just for cheating put/get driver tag. |
| 298 | 297 | */ |
| 299 | 298 | flush_rq->mq_ctx = first_rq->mq_ctx; |
| 299 | + flush_rq->mq_hctx = first_rq->mq_hctx; | |
| 300 | 300 | |
| 301 | 301 | if (!q->elevator) { |
| 302 | 302 | fq->orig_rq = first_rq; |
| 303 | 303 | flush_rq->tag = first_rq->tag; |
| 304 | − hctx = blk_mq_map_queue(q, first_rq->cmd_flags, | |
| 305 | − first_rq->mq_ctx->cpu); | |
| 306 | − blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq); | |
| 304 | + blk_mq_tag_set_rq(flush_rq->mq_hctx, first_rq->tag, flush_rq); | |
| 307 | 305 | } else { |
| 308 | 306 | flush_rq->internal_tag = first_rq->internal_tag; |
| 309 | 307 | } |
| @@ -320,13 +318,11 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, · 10 unmodified lines | ||
| 320 | 318 | static void mq_flush_data_end_io(struct request *rq, blk_status_t error) |
| 321 | 319 | { |
| 322 | 320 | struct request_queue *q = rq->q; |
| 323 | − struct blk_mq_hw_ctx *hctx; | |
| 321 | + struct blk_mq_hw_ctx *hctx = rq->mq_hctx; | |
| 324 | 322 | struct blk_mq_ctx *ctx = rq->mq_ctx; |
| 325 | 323 | unsigned long flags; |
| 326 | 324 | struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx); |
| 327 | 325 | |
| 328 | − hctx = blk_mq_map_queue(q, rq->cmd_flags, ctx->cpu); | |
| 329 | − | |
| 330 | 326 | if (q->elevator) { |
| 331 | 327 | WARN_ON(rq->tag < 0); |
| 332 | 328 | blk_mq_put_driver_tag_hctx(hctx, rq); |
| @@ -427,10 +427,8 @@ struct show_busy_params { · 426 unmodified lines | ||
| 427 | 427 | static void hctx_show_busy_rq(struct request *rq, void *data, bool reserved) |
| 428 | 428 | { |
| 429 | 429 | const struct show_busy_params *params = data; |
| 430 | − struct blk_mq_hw_ctx *hctx; | |
| 431 | 430 | |
| 432 | − hctx = blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); | |
| 433 | − if (hctx == params->hctx) | |
| 431 | + if (rq->mq_hctx == params->hctx) | |
| 434 | 432 | __blk_mq_debugfs_rq_show(params->m, |
| 435 | 433 | list_entry_rq(&rq->queuelist)); |
| 436 | 434 | } |
| @@ -366,9 +366,7 @@ void blk_mq_sched_insert_request(struct request *rq, bool at_head, · 365 unmodified lines | ||
| 366 | 366 | struct request_queue *q = rq->q; |
| 367 | 367 | struct elevator_queue *e = q->elevator; |
| 368 | 368 | struct blk_mq_ctx *ctx = rq->mq_ctx; |
| 369 | − struct blk_mq_hw_ctx *hctx; | |
| 370 | − | |
| 371 | − hctx = blk_mq_map_queue(q, rq->cmd_flags, ctx->cpu); | |
| 369 | + struct blk_mq_hw_ctx *hctx = rq->mq_hctx; | |
| 372 | 370 | |
| 373 | 371 | /* flush rq in flush machinery need to be dispatched directly */ |
| 374 | 372 | if (!(rq->rq_flags & RQF_FLUSH_SEQ) && op_is_flush(rq->cmd_flags)) { |
| @@ -407,7 +405,7 @@ void blk_mq_sched_insert_requests(struct request_queue *q, · 32 unmodified lines | ||
| 407 | 405 | |
| 408 | 406 | /* For list inserts, requests better be on the same hw queue */ |
| 409 | 407 | rq = list_first_entry(list, struct request, queuelist); |
| 410 | − hctx = blk_mq_map_queue(q, rq->cmd_flags, ctx->cpu); | |
| 408 | + hctx = rq->mq_hctx; | |
| 411 | 409 | |
| 412 | 410 | e = hctx->queue->elevator; |
| 413 | 411 | if (e && e->type->ops.insert_requests) |
| @@ -527,14 +527,7 @@ int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, · 526 unmodified lines | ||
| 527 | 527 | */ |
| 528 | 528 | u32 blk_mq_unique_tag(struct request *rq) |
| 529 | 529 | { |
| 530 | − struct request_queue *q = rq->q; | |
| 531 | − struct blk_mq_hw_ctx *hctx; | |
| 532 | − int hwq = 0; | |
| 533 | − | |
| 534 | − hctx = blk_mq_map_queue(q, rq->cmd_flags, rq->mq_ctx->cpu); | |
| 535 | − hwq = hctx->queue_num; | |
| 536 | − | |
| 537 | − return (hwq << BLK_MQ_UNIQUE_TAG_BITS) | | |
| 530 | + return (rq->mq_hctx->queue_num << BLK_MQ_UNIQUE_TAG_BITS) | | |
| 538 | 531 | (rq->tag & BLK_MQ_UNIQUE_TAG_MASK); |
| 539 | 532 | } |
| 540 | 533 | EXPORT_SYMBOL(blk_mq_unique_tag); |
| @@ -300,6 +300,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, · 299 unmodified lines | ||
| 300 | 300 | /* csd/requeue_work/fifo_time is initialized before use */ |
| 301 | 301 | rq->q = data->q; |
| 302 | 302 | rq->mq_ctx = data->ctx; |
| 303 | + rq->mq_hctx = data->hctx; | |
| 303 | 304 | rq->rq_flags = rq_flags; |
| 304 | 305 | rq->cmd_flags = op; |
| 305 | 306 | if (data->flags & BLK_MQ_REQ_PREEMPT) |
| @@ -472,10 +473,11 @@ static void __blk_mq_free_request(struct request *rq) · 166 unmodified lines | ||
| 472 | 473 | { |
| 473 | 474 | struct request_queue *q = rq->q; |
| 474 | 475 | struct blk_mq_ctx *ctx = rq->mq_ctx; |
| 475 | − struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, rq->cmd_flags, ctx->cpu); | |
| 476 | + struct blk_mq_hw_ctx *hctx = rq->mq_hctx; | |
| 476 | 477 | const int sched_tag = rq->internal_tag; |
| 477 | 478 | |
| 478 | 479 | blk_pm_mark_last_busy(rq); |
| 480 | + rq->mq_hctx = NULL; | |
| 479 | 481 | if (rq->tag != -1) |
| 480 | 482 | blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); |
| 481 | 483 | if (sched_tag != -1) |
| @@ -489,7 +491,7 @@ void blk_mq_free_request(struct request *rq) · 7 unmodified lines | ||
| 489 | 491 | struct request_queue *q = rq->q; |
| 490 | 492 | struct elevator_queue *e = q->elevator; |
| 491 | 493 | struct blk_mq_ctx *ctx = rq->mq_ctx; |
| 492 | − struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, rq->cmd_flags, ctx->cpu); | |
| 494 | + struct blk_mq_hw_ctx *hctx = rq->mq_hctx; | |
| 493 | 495 | |
| 494 | 496 | if (rq->rq_flags & RQF_ELVPRIV) { |
| 495 | 497 | if (e && e->type->ops.finish_request) |
| @@ -983,7 +985,7 @@ bool blk_mq_get_driver_tag(struct request *rq) · 487 unmodified lines | ||
| 983 | 985 | { |
| 984 | 986 | struct blk_mq_alloc_data data = { |
| 985 | 987 | .q = rq->q, |
| 986 | − .hctx = blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu), | |
| 988 | + .hctx = rq->mq_hctx, | |
| 987 | 989 | .flags = BLK_MQ_REQ_NOWAIT, |
| 988 | 990 | .cmd_flags = rq->cmd_flags, |
| 989 | 991 | }; |
| @@ -1149,7 +1151,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, · 159 unmodified lines | ||
| 1149 | 1151 | |
| 1150 | 1152 | rq = list_first_entry(list, struct request, queuelist); |
| 1151 | 1153 | |
| 1152 | − hctx = blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); | |
| 1154 | + hctx = rq->mq_hctx; | |
| 1153 | 1155 | if (!got_budget && !blk_mq_get_dispatch_budget(hctx)) |
| 1154 | 1156 | break; |
| 1155 | 1157 | |
| @@ -1579,9 +1581,7 @@ void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, · 423 unmodified lines | ||
| 1579 | 1581 | */ |
| 1580 | 1582 | void blk_mq_request_bypass_insert(struct request *rq, bool run_queue) |
| 1581 | 1583 | { |
| 1582 | − struct blk_mq_ctx *ctx = rq->mq_ctx; | |
| 1583 | − struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, rq->cmd_flags, | |
| 1584 | − ctx->cpu); | |
| 1584 | + struct blk_mq_hw_ctx *hctx = rq->mq_hctx; | |
| 1585 | 1585 | |
| 1586 | 1586 | spin_lock(&hctx->lock); |
| 1587 | 1587 | list_add_tail(&rq->queuelist, &hctx->dispatch); |
| @@ -1790,9 +1790,7 @@ blk_status_t blk_mq_request_issue_directly(struct request *rq) · 202 unmodified lines | ||
| 1790 | 1790 | blk_status_t ret; |
| 1791 | 1791 | int srcu_idx; |
| 1792 | 1792 | blk_qc_t unused_cookie; |
| 1793 | − struct blk_mq_ctx *ctx = rq->mq_ctx; | |
| 1794 | − struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, rq->cmd_flags, | |
| 1795 | − ctx->cpu); | |
| 1793 | + struct blk_mq_hw_ctx *hctx = rq->mq_hctx; | |
| 1796 | 1794 | |
| 1797 | 1795 | hctx_lock(hctx, &srcu_idx); |
| 1798 | 1796 | ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true); |
| @@ -1917,9 +1915,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) · 118 unmodified lines | ||
| 1917 | 1915 | blk_mq_put_ctx(data.ctx); |
| 1918 | 1916 | |
| 1919 | 1917 | if (same_queue_rq) { |
| 1920 | − data.hctx = blk_mq_map_queue(q, | |
| 1921 | − same_queue_rq->cmd_flags, | |
| 1922 | − same_queue_rq->mq_ctx->cpu); | |
| 1918 | + data.hctx = same_queue_rq->mq_hctx; | |
| 1923 | 1919 | blk_mq_try_issue_directly(data.hctx, same_queue_rq, |
| 1924 | 1920 | &cookie); |
| 1925 | 1921 | } |
| @@ -223,13 +223,10 @@ static inline void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx, · 222 unmodified lines | ||
| 223 | 223 | |
| 224 | 224 | static inline void blk_mq_put_driver_tag(struct request *rq) |
| 225 | 225 | { |
| 226 | − struct blk_mq_hw_ctx *hctx; | |
| 227 | − | |
| 228 | 226 | if (rq->tag == -1 || rq->internal_tag == -1) |
| 229 | 227 | return; |
| 230 | 228 | |
| 231 | − hctx = blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); | |
| 232 | − __blk_mq_put_driver_tag(hctx, rq); | |
| 229 | + __blk_mq_put_driver_tag(rq->mq_hctx, rq); | |
| 233 | 230 | } |
| 234 | 231 | |
| 235 | 232 | static inline void blk_mq_clear_mq_map(struct blk_mq_queue_map *qmap) |
| @@ -129,6 +129,7 @@ enum mq_rq_state { · 128 unmodified lines | ||
| 129 | 129 | struct request { |
| 130 | 130 | struct request_queue *q; |
| 131 | 131 | struct blk_mq_ctx *mq_ctx; |
| 132 | + struct blk_mq_hw_ctx *mq_hctx; | |
| 132 | 133 | |
| 133 | 134 | unsigned int cmd_flags; /* op and common flags */ |
| 134 | 135 | req_flags_t rq_flags; |