That we we can also poll non blk-mq queues. Mostly needed for the NVMe multipath code, but could also be useful elsewhere.
Unified · Split
| @@ -2321,6 +2321,17 @@ blk_qc_t submit_bio(struct bio *bio) · 2320 unmodified lines | ||
| 2321 | 2321 | } |
| 2322 | 2322 | EXPORT_SYMBOL(submit_bio); |
| 2323 | 2323 | |
| 2324 | +bool blk_poll(struct request_queue *q, blk_qc_t cookie) | |
| 2325 | +{ | |
| 2326 | + if (!q->poll_fn || !blk_qc_t_valid(cookie)) | |
| 2327 | + return false; | |
| 2328 | + | |
| 2329 | + if (current->plug) | |
| 2330 | + blk_flush_plug_list(current->plug, false); | |
| 2331 | + return q->poll_fn(q, cookie); | |
| 2332 | +} | |
| 2333 | +EXPORT_SYMBOL_GPL(blk_poll); | |
| 2334 | + | |
| 2324 | 2335 | /** |
| 2325 | 2336 | * blk_cloned_rq_check_limits - Helper function to check a cloned request |
| 2326 | 2337 | * for new the queue limits |
| @@ -37,6 +37,7 @@ · 36 unmodified lines | ||
| 37 | 37 | #include "blk-wbt.h" |
| 38 | 38 | #include "blk-mq-sched.h" |
| 39 | 39 | |
| 40 | +static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie); | |
| 40 | 41 | static void blk_mq_poll_stats_start(struct request_queue *q); |
| 41 | 42 | static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); |
| 42 | 43 | |
| @@ -2499,6 +2500,8 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, · 2456 unmodified lines | ||
| 2499 | 2500 | spin_lock_init(&q->requeue_lock); |
| 2500 | 2501 | |
| 2501 | 2502 | blk_queue_make_request(q, blk_mq_make_request); |
| 2503 | + if (q->mq_ops->poll) | |
| 2504 | + q->poll_fn = blk_mq_poll; | |
| 2502 | 2505 | |
| 2503 | 2506 | /* |
| 2504 | 2507 | * Do this after blk_queue_make_request() overrides it... |
| @@ -2961,20 +2964,14 @@ static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) · 456 unmodified lines | ||
| 2961 | 2964 | return false; |
| 2962 | 2965 | } |
| 2963 | 2966 | |
| 2964 | −bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) | |
| 2967 | +static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) | |
| 2965 | 2968 | { |
| 2966 | 2969 | struct blk_mq_hw_ctx *hctx; |
| 2967 | − struct blk_plug *plug; | |
| 2968 | 2970 | struct request *rq; |
| 2969 | 2971 | |
| 2970 | − if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) || | |
| 2971 | − !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) | |
| 2972 | + if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) | |
| 2972 | 2973 | return false; |
| 2973 | 2974 | |
| 2974 | − plug = current->plug; | |
| 2975 | − if (plug) | |
| 2976 | − blk_flush_plug_list(plug, false); | |
| 2977 | − | |
| 2978 | 2975 | hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; |
| 2979 | 2976 | if (!blk_qc_t_is_internal(cookie)) |
| 2980 | 2977 | rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); |
| @@ -2992,7 +2989,6 @@ bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) · 11 unmodified lines | ||
| 2992 | 2989 | |
| 2993 | 2990 | return __blk_mq_poll(hctx, rq); |
| 2994 | 2991 | } |
| 2995 | −EXPORT_SYMBOL_GPL(blk_mq_poll); | |
| 2996 | 2992 | |
| 2997 | 2993 | static int __init blk_mq_init(void) |
| 2998 | 2994 | { |
| @@ -94,7 +94,7 @@ static void nvmet_execute_rw(struct nvmet_req *req) · 93 unmodified lines | ||
| 94 | 94 | |
| 95 | 95 | cookie = submit_bio(bio); |
| 96 | 96 | |
| 97 | − blk_mq_poll(bdev_get_queue(req->ns->bdev), cookie); | |
| 97 | + blk_poll(bdev_get_queue(req->ns->bdev), cookie); | |
| 98 | 98 | } |
| 99 | 99 | |
| 100 | 100 | static void nvmet_execute_flush(struct nvmet_req *req) |
| @@ -237,7 +237,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter, · 236 unmodified lines | ||
| 237 | 237 | if (!READ_ONCE(bio.bi_private)) |
| 238 | 238 | break; |
| 239 | 239 | if (!(iocb->ki_flags & IOCB_HIPRI) || |
| 240 | − !blk_mq_poll(bdev_get_queue(bdev), qc)) | |
| 240 | + !blk_poll(bdev_get_queue(bdev), qc)) | |
| 241 | 241 | io_schedule(); |
| 242 | 242 | } |
| 243 | 243 | __set_current_state(TASK_RUNNING); |
| @@ -402,7 +402,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) · 158 unmodified lines | ||
| 402 | 402 | break; |
| 403 | 403 | |
| 404 | 404 | if (!(iocb->ki_flags & IOCB_HIPRI) || |
| 405 | − !blk_mq_poll(bdev_get_queue(bdev), qc)) | |
| 405 | + !blk_poll(bdev_get_queue(bdev), qc)) | |
| 406 | 406 | io_schedule(); |
| 407 | 407 | } |
| 408 | 408 | __set_current_state(TASK_RUNNING); |
| @@ -486,7 +486,7 @@ static struct bio *dio_await_one(struct dio *dio) · 485 unmodified lines | ||
| 486 | 486 | dio->waiter = current; |
| 487 | 487 | spin_unlock_irqrestore(&dio->bio_lock, flags); |
| 488 | 488 | if (!(dio->iocb->ki_flags & IOCB_HIPRI) || |
| 489 | − !blk_mq_poll(dio->bio_disk->queue, dio->bio_cookie)) | |
| 489 | + !blk_poll(dio->bio_disk->queue, dio->bio_cookie)) | |
| 490 | 490 | io_schedule(); |
| 491 | 491 | /* wake up sets us TASK_RUNNING */ |
| 492 | 492 | spin_lock_irqsave(&dio->bio_lock, flags); |
| @@ -1049,7 +1049,7 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, · 1048 unmodified lines | ||
| 1049 | 1049 | |
| 1050 | 1050 | if (!(iocb->ki_flags & IOCB_HIPRI) || |
| 1051 | 1051 | !dio->submit.last_queue || |
| 1052 | − !blk_mq_poll(dio->submit.last_queue, | |
| 1052 | + !blk_poll(dio->submit.last_queue, | |
| 1053 | 1053 | dio->submit.cookie)) |
| 1054 | 1054 | io_schedule(); |
| 1055 | 1055 | } |
| @@ -266,6 +266,7 @@ struct blk_queue_ctx; · 265 unmodified lines | ||
| 266 | 266 | |
| 267 | 267 | typedef void (request_fn_proc) (struct request_queue *q); |
| 268 | 268 | typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio); |
| 269 | +typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t); | |
| 269 | 270 | typedef int (prep_rq_fn) (struct request_queue *, struct request *); |
| 270 | 271 | typedef void (unprep_rq_fn) (struct request_queue *, struct request *); |
| 271 | 272 | |
| @@ -408,6 +409,7 @@ struct request_queue { · 136 unmodified lines | ||
| 408 | 409 | |
| 409 | 410 | request_fn_proc *request_fn; |
| 410 | 411 | make_request_fn *make_request_fn; |
| 412 | + poll_q_fn *poll_fn; | |
| 411 | 413 | prep_rq_fn *prep_rq_fn; |
| 412 | 414 | unprep_rq_fn *unprep_rq_fn; |
| 413 | 415 | softirq_done_fn *softirq_done_fn; |
| @@ -975,7 +977,7 @@ extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *, · 561 unmodified lines | ||
| 975 | 977 | int blk_status_to_errno(blk_status_t status); |
| 976 | 978 | blk_status_t errno_to_blk_status(int errno); |
| 977 | 979 | |
| 978 | −bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie); | |
| 980 | +bool blk_poll(struct request_queue *q, blk_qc_t cookie); | |
| 979 | 981 | |
| 980 | 982 | static inline struct request_queue *bdev_get_queue(struct block_device *bdev) |
| 981 | 983 | { |
| @@ -407,7 +407,7 @@ int swap_readpage(struct page *page, bool do_poll) · 406 unmodified lines | ||
| 407 | 407 | if (!READ_ONCE(bio->bi_private)) |
| 408 | 408 | break; |
| 409 | 409 | |
| 410 | − if (!blk_mq_poll(disk->queue, qc)) | |
| 410 | + if (!blk_poll(disk->queue, qc)) | |
| 411 | 411 | break; |
| 412 | 412 | } |
| 413 | 413 | __set_current_state(TASK_RUNNING); |