History

block: add block polling support

Add basic support for polling for specific IO to complete. This uses the cookie that blk-mq passes back, which enables the block layer to pass this cookie to the driver to spin for a specific request. This will be combined with request latency tracking, so we can make qualified decisions about when to poll and when not to. For now, for benchmark purposes, we add a sysfs file that controls whether polling is enabled or not.

Jens Axboe committed 11 years ago

Commit 05229be1 parent 7b371635 files · +99 −0Browse files

Unified · Split

5 files
  1. block/blk-core.c +41 −0
  2. block/blk-mq-sysfs.c +10 −0
  3. block/blk-sysfs.c +35 −0
  4. include/linux/blk-mq.h +10 −0
  5. include/linux/blkdev.h +3 −0
block/blk-core.c+41−0
@@ -3312,6 +3312,47 @@ void blk_finish_plug(struct blk_plug *plug) · 3311 unmodified lines
3312}3312}
3313EXPORT_SYMBOL(blk_finish_plug);3313EXPORT_SYMBOL(blk_finish_plug);
3314 3314
3315bool blk_poll(struct request_queue *q, blk_qc_t cookie)
3316{
3317 struct blk_plug *plug;
3318 long state;
3319
3320 if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
3321 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
3322 return false;
3323
3324 plug = current->plug;
3325 if (plug)
3326 blk_flush_plug_list(plug, false);
3327
3328 state = current->state;
3329 while (!need_resched()) {
3330 unsigned int queue_num = blk_qc_t_to_queue_num(cookie);
3331 struct blk_mq_hw_ctx *hctx = q->queue_hw_ctx[queue_num];
3332 int ret;
3333
3334 hctx->poll_invoked++;
3335
3336 ret = q->mq_ops->poll(hctx, blk_qc_t_to_tag(cookie));
3337 if (ret > 0) {
3338 hctx->poll_success++;
3339 set_current_state(TASK_RUNNING);
3340 return true;
3341 }
3342
3343 if (signal_pending_state(state, current))
3344 set_current_state(TASK_RUNNING);
3345
3346 if (current->state == TASK_RUNNING)
3347 return true;
3348 if (ret < 0)
3349 break;
3350 cpu_relax();
3351 }
3352
3353 return false;
3354}
3355
3315#ifdef CONFIG_PM3356#ifdef CONFIG_PM
3316/**3357/**
3317 * blk_pm_runtime_init - Block layer runtime PM initialization routine3358 * blk_pm_runtime_init - Block layer runtime PM initialization routine
block/blk-mq-sysfs.c+10−0
@@ -174,6 +174,11 @@ static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page) · 173 unmodified lines
174 return ret;174 return ret;
175}175}
176 176
177static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx *hctx, char *page)
178{
179 return sprintf(page, "invoked=%lu, success=%lu\n", hctx->poll_invoked, hctx->poll_success);
180}
181
177static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,182static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
178 char *page)183 char *page)
179{184{
@@ -295,6 +300,10 @@ static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { · 115 unmodified lines
295 .attr = {.name = "cpu_list", .mode = S_IRUGO },300 .attr = {.name = "cpu_list", .mode = S_IRUGO },
296 .show = blk_mq_hw_sysfs_cpus_show,301 .show = blk_mq_hw_sysfs_cpus_show,
297};302};
303static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
304 .attr = {.name = "io_poll", .mode = S_IRUGO },
305 .show = blk_mq_hw_sysfs_poll_show,
306};
298 307
299static struct attribute *default_hw_ctx_attrs[] = {308static struct attribute *default_hw_ctx_attrs[] = {
300 &blk_mq_hw_sysfs_queued.attr,309 &blk_mq_hw_sysfs_queued.attr,
@@ -304,6 +313,7 @@ static struct attribute *default_hw_ctx_attrs[] = { · 3 unmodified lines
304 &blk_mq_hw_sysfs_tags.attr,313 &blk_mq_hw_sysfs_tags.attr,
305 &blk_mq_hw_sysfs_cpus.attr,314 &blk_mq_hw_sysfs_cpus.attr,
306 &blk_mq_hw_sysfs_active.attr,315 &blk_mq_hw_sysfs_active.attr,
316 &blk_mq_hw_sysfs_poll.attr,
307 NULL,317 NULL,
308};318};
309 319
block/blk-sysfs.c+35−0
@@ -317,6 +317,34 @@ queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count) · 316 unmodified lines
317 return ret;317 return ret;
318}318}
319 319
320static ssize_t queue_poll_show(struct request_queue *q, char *page)
321{
322 return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
323}
324
325static ssize_t queue_poll_store(struct request_queue *q, const char *page,
326 size_t count)
327{
328 unsigned long poll_on;
329 ssize_t ret;
330
331 if (!q->mq_ops || !q->mq_ops->poll)
332 return -EINVAL;
333
334 ret = queue_var_store(&poll_on, page, count);
335 if (ret < 0)
336 return ret;
337
338 spin_lock_irq(q->queue_lock);
339 if (poll_on)
340 queue_flag_set(QUEUE_FLAG_POLL, q);
341 else
342 queue_flag_clear(QUEUE_FLAG_POLL, q);
343 spin_unlock_irq(q->queue_lock);
344
345 return ret;
346}
347
320static struct queue_sysfs_entry queue_requests_entry = {348static struct queue_sysfs_entry queue_requests_entry = {
321 .attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },349 .attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
322 .show = queue_requests_show,350 .show = queue_requests_show,
@@ -442,6 +470,12 @@ static struct queue_sysfs_entry queue_random_entry = { · 119 unmodified lines
442 .store = queue_store_random,470 .store = queue_store_random,
443};471};
444 472
473static struct queue_sysfs_entry queue_poll_entry = {
474 .attr = {.name = "io_poll", .mode = S_IRUGO | S_IWUSR },
475 .show = queue_poll_show,
476 .store = queue_poll_store,
477};
478
445static struct attribute *default_attrs[] = {479static struct attribute *default_attrs[] = {
446 &queue_requests_entry.attr,480 &queue_requests_entry.attr,
447 &queue_ra_entry.attr,481 &queue_ra_entry.attr,
@@ -466,6 +500,7 @@ static struct attribute *default_attrs[] = { · 18 unmodified lines
466 &queue_rq_affinity_entry.attr,500 &queue_rq_affinity_entry.attr,
467 &queue_iostats_entry.attr,501 &queue_iostats_entry.attr,
468 &queue_random_entry.attr,502 &queue_random_entry.attr,
503 &queue_poll_entry.attr,
469 NULL,504 NULL,
470};505};
471 506
include/linux/blk-mq.h+10−0
@@ -59,6 +59,9 @@ struct blk_mq_hw_ctx { · 58 unmodified lines
59 59
60 struct blk_mq_cpu_notifier cpu_notifier;60 struct blk_mq_cpu_notifier cpu_notifier;
61 struct kobject kobj;61 struct kobject kobj;
62
63 unsigned long poll_invoked;
64 unsigned long poll_success;
62};65};
63 66
64struct blk_mq_tag_set {67struct blk_mq_tag_set {
@@ -97,6 +100,8 @@ typedef void (exit_request_fn)(void *, struct request *, unsigned int, · 32 unmodified lines
97typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,100typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,
98 bool);101 bool);
99typedef void (busy_tag_iter_fn)(struct request *, void *, bool);102typedef void (busy_tag_iter_fn)(struct request *, void *, bool);
103typedef int (poll_fn)(struct blk_mq_hw_ctx *, unsigned int);
104
100 105
101struct blk_mq_ops {106struct blk_mq_ops {
102 /*107 /*
@@ -114,6 +119,11 @@ struct blk_mq_ops { · 11 unmodified lines
114 */119 */
115 timeout_fn *timeout;120 timeout_fn *timeout;
116 121
122 /*
123 * Called to poll for completion of a specific tag.
124 */
125 poll_fn *poll;
126
117 softirq_done_fn *complete;127 softirq_done_fn *complete;
118 128
119 /*129 /*
include/linux/blkdev.h+3−0
@@ -487,6 +487,7 @@ struct request_queue { · 486 unmodified lines
487#define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */487#define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */
488#define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */488#define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */
489#define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/489#define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/
490#define QUEUE_FLAG_POLL 22 /* IO polling enabled if set */
490 491
491#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \492#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
492 (1 << QUEUE_FLAG_STACKABLE) | \493 (1 << QUEUE_FLAG_STACKABLE) | \
@@ -814,6 +815,8 @@ extern int blk_execute_rq(struct request_queue *, struct gendisk *, · 321 unmodified lines
814extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,815extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
815 struct request *, int, rq_end_io_fn *);816 struct request *, int, rq_end_io_fn *);
816 817
818bool blk_poll(struct request_queue *q, blk_qc_t cookie);
819
817static inline struct request_queue *bdev_get_queue(struct block_device *bdev)820static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
818{821{
819 return bdev->bd_disk->queue; /* this is never NULL */822 return bdev->bd_disk->queue; /* this is never NULL */