History

block: add block polling support

Add basic support for polling for specific IO to complete. This uses the cookie that blk-mq passes back, which enables the block layer to pass this cookie to the driver to spin for a specific request. This will be combined with request latency tracking, so we can make qualified decisions about when to poll and when not to. For now, for benchmark purposes, we add a sysfs file that controls whether polling is enabled or not.

Jens Axboe committed 11 years ago

Commit 05229be1 parent 7b371635 files · +99 −0Browse files

Unified · Split

5 files
  1. block/blk-core.c +41 −0
  2. block/blk-mq-sysfs.c +10 −0
  3. block/blk-sysfs.c +35 −0
  4. include/linux/blk-mq.h +10 −0
  5. include/linux/blkdev.h +3 −0
block/blk-core.c+41−0
@@ -3312,6 +3312,47 @@ void blk_finish_plug(struct blk_plug *plug) · 3311 unmodified lines
33123312 }
33133313 EXPORT_SYMBOL(blk_finish_plug);
33143314
3315+bool blk_poll(struct request_queue *q, blk_qc_t cookie)
3316+{
3317+ struct blk_plug *plug;
3318+ long state;
3319+
3320+ if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
3321+ !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
3322+ return false;
3323+
3324+ plug = current->plug;
3325+ if (plug)
3326+ blk_flush_plug_list(plug, false);
3327+
3328+ state = current->state;
3329+ while (!need_resched()) {
3330+ unsigned int queue_num = blk_qc_t_to_queue_num(cookie);
3331+ struct blk_mq_hw_ctx *hctx = q->queue_hw_ctx[queue_num];
3332+ int ret;
3333+
3334+ hctx->poll_invoked++;
3335+
3336+ ret = q->mq_ops->poll(hctx, blk_qc_t_to_tag(cookie));
3337+ if (ret > 0) {
3338+ hctx->poll_success++;
3339+ set_current_state(TASK_RUNNING);
3340+ return true;
3341+ }
3342+
3343+ if (signal_pending_state(state, current))
3344+ set_current_state(TASK_RUNNING);
3345+
3346+ if (current->state == TASK_RUNNING)
3347+ return true;
3348+ if (ret < 0)
3349+ break;
3350+ cpu_relax();
3351+ }
3352+
3353+ return false;
3354+}
3355+
33153356 #ifdef CONFIG_PM
33163357 /**
33173358 * blk_pm_runtime_init - Block layer runtime PM initialization routine
block/blk-mq-sysfs.c+10−0
@@ -174,6 +174,11 @@ static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page) · 173 unmodified lines
174174 return ret;
175175 }
176176
177+static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx *hctx, char *page)
178+{
179+ return sprintf(page, "invoked=%lu, success=%lu\n", hctx->poll_invoked, hctx->poll_success);
180+}
181+
177182 static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
178183 char *page)
179184 {
@@ -295,6 +300,10 @@ static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { · 115 unmodified lines
295300 .attr = {.name = "cpu_list", .mode = S_IRUGO },
296301 .show = blk_mq_hw_sysfs_cpus_show,
297302 };
303+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
304+ .attr = {.name = "io_poll", .mode = S_IRUGO },
305+ .show = blk_mq_hw_sysfs_poll_show,
306+};
298307
299308 static struct attribute *default_hw_ctx_attrs[] = {
300309 &blk_mq_hw_sysfs_queued.attr,
@@ -304,6 +313,7 @@ static struct attribute *default_hw_ctx_attrs[] = { · 3 unmodified lines
304313 &blk_mq_hw_sysfs_tags.attr,
305314 &blk_mq_hw_sysfs_cpus.attr,
306315 &blk_mq_hw_sysfs_active.attr,
316+ &blk_mq_hw_sysfs_poll.attr,
307317 NULL,
308318 };
309319
block/blk-sysfs.c+35−0
@@ -317,6 +317,34 @@ queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count) · 316 unmodified lines
317317 return ret;
318318 }
319319
320+static ssize_t queue_poll_show(struct request_queue *q, char *page)
321+{
322+ return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
323+}
324+
325+static ssize_t queue_poll_store(struct request_queue *q, const char *page,
326+ size_t count)
327+{
328+ unsigned long poll_on;
329+ ssize_t ret;
330+
331+ if (!q->mq_ops || !q->mq_ops->poll)
332+ return -EINVAL;
333+
334+ ret = queue_var_store(&poll_on, page, count);
335+ if (ret < 0)
336+ return ret;
337+
338+ spin_lock_irq(q->queue_lock);
339+ if (poll_on)
340+ queue_flag_set(QUEUE_FLAG_POLL, q);
341+ else
342+ queue_flag_clear(QUEUE_FLAG_POLL, q);
343+ spin_unlock_irq(q->queue_lock);
344+
345+ return ret;
346+}
347+
320348 static struct queue_sysfs_entry queue_requests_entry = {
321349 .attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
322350 .show = queue_requests_show,
@@ -442,6 +470,12 @@ static struct queue_sysfs_entry queue_random_entry = { · 119 unmodified lines
442470 .store = queue_store_random,
443471 };
444472
473+static struct queue_sysfs_entry queue_poll_entry = {
474+ .attr = {.name = "io_poll", .mode = S_IRUGO | S_IWUSR },
475+ .show = queue_poll_show,
476+ .store = queue_poll_store,
477+};
478+
445479 static struct attribute *default_attrs[] = {
446480 &queue_requests_entry.attr,
447481 &queue_ra_entry.attr,
@@ -466,6 +500,7 @@ static struct attribute *default_attrs[] = { · 18 unmodified lines
466500 &queue_rq_affinity_entry.attr,
467501 &queue_iostats_entry.attr,
468502 &queue_random_entry.attr,
503+ &queue_poll_entry.attr,
469504 NULL,
470505 };
471506
include/linux/blk-mq.h+10−0
@@ -59,6 +59,9 @@ struct blk_mq_hw_ctx { · 58 unmodified lines
5959
6060 struct blk_mq_cpu_notifier cpu_notifier;
6161 struct kobject kobj;
62+
63+ unsigned long poll_invoked;
64+ unsigned long poll_success;
6265 };
6366
6467 struct blk_mq_tag_set {
@@ -97,6 +100,8 @@ typedef void (exit_request_fn)(void *, struct request *, unsigned int, · 32 unmodified lines
97100 typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,
98101 bool);
99102 typedef void (busy_tag_iter_fn)(struct request *, void *, bool);
103+typedef int (poll_fn)(struct blk_mq_hw_ctx *, unsigned int);
104+
100105
101106 struct blk_mq_ops {
102107 /*
@@ -114,6 +119,11 @@ struct blk_mq_ops { · 11 unmodified lines
114119 */
115120 timeout_fn *timeout;
116121
122+ /*
123+ * Called to poll for completion of a specific tag.
124+ */
125+ poll_fn *poll;
126+
117127 softirq_done_fn *complete;
118128
119129 /*
include/linux/blkdev.h+3−0
@@ -487,6 +487,7 @@ struct request_queue { · 486 unmodified lines
487487 #define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */
488488 #define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */
489489 #define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/
490+#define QUEUE_FLAG_POLL 22 /* IO polling enabled if set */
490491
491492 #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
492493 (1 << QUEUE_FLAG_STACKABLE) | \
@@ -814,6 +815,8 @@ extern int blk_execute_rq(struct request_queue *, struct gendisk *, · 321 unmodified lines
814815 extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
815816 struct request *, int, rq_end_io_fn *);
816817
818+bool blk_poll(struct request_queue *q, blk_qc_t cookie);
819+
817820 static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
818821 {
819822 return bdev->bd_disk->queue; /* this is never NULL */