History

[BLOCK] ll_rw_blk: Enable out-of-order request completions through softirq

Request completion can be a quite heavy process, since it needs to iterate through the entire request and complete the bio's it holds. This patch adds blk_complete_request() which moves this processing into a dedicated block softirq.

Jens Axboe committed 20 years ago

Commit ff856ba1 parent 5367f2d3 files · +124 −4Browse files

Showing the changes to block (1 of 3 files). Show all

Unified · Split

block/ll_rw_blk.c+105−1
@@ -27,6 +27,8 @@ · 26 unmodified lines
2727 #include <linux/swap.h>
2828 #include <linux/writeback.h>
2929 #include <linux/blkdev.h>
30+#include <linux/interrupt.h>
31+#include <linux/cpu.h>
3032
3133 /*
3234 * for max sense size
@@ -62,13 +64,15 @@ static wait_queue_head_t congestion_wqh[2] = { · 29 unmodified lines
6264 /*
6365 * Controlling structure to kblockd
6466 */
65−static struct workqueue_struct *kblockd_workqueue;
67+static struct workqueue_struct *kblockd_workqueue;
6668
6769 unsigned long blk_max_low_pfn, blk_max_pfn;
6870
6971 EXPORT_SYMBOL(blk_max_low_pfn);
7072 EXPORT_SYMBOL(blk_max_pfn);
7173
74+static DEFINE_PER_CPU(struct list_head, blk_cpu_done);
75+
7276 /* Amount of time in which a process may batch requests */
7377 #define BLK_BATCH_TIME (HZ/50UL)
7478
@@ -207,6 +211,13 @@ void blk_queue_merge_bvec(request_queue_t *q, merge_bvec_fn *mbfn) · 132 unmodified lines
207211
208212 EXPORT_SYMBOL(blk_queue_merge_bvec);
209213
214+void blk_queue_softirq_done(request_queue_t *q, softirq_done_fn *fn)
215+{
216+ q->softirq_done_fn = fn;
217+}
218+
219+EXPORT_SYMBOL(blk_queue_softirq_done);
220+
210221 /**
211222 * blk_queue_make_request - define an alternate make_request function for a device
212223 * @q: the request queue for the device to be affected
@@ -270,6 +281,7 @@ EXPORT_SYMBOL(blk_queue_make_request); · 57 unmodified lines
270281 static inline void rq_init(request_queue_t *q, struct request *rq)
271282 {
272283 INIT_LIST_HEAD(&rq->queuelist);
284+ INIT_LIST_HEAD(&rq->donelist);
273285
274286 rq->errors = 0;
275287 rq->rq_status = RQ_ACTIVE;
@@ -286,6 +298,7 @@ static inline void rq_init(request_queue_t *q, struct request *rq) · 10 unmodified lines
286298 rq->sense = NULL;
287299 rq->end_io = NULL;
288300 rq->end_io_data = NULL;
301+ rq->completion_data = NULL;
289302 }
290303
291304 /**
@@ -3286,6 +3299,87 @@ int end_that_request_chunk(struct request *req, int uptodate, int nr_bytes) · 2994 unmodified lines
32863299
32873300 EXPORT_SYMBOL(end_that_request_chunk);
32883301
3302+/*
3303+ * splice the completion data to a local structure and hand off to
3304+ * process_completion_queue() to complete the requests
3305+ */
3306+static void blk_done_softirq(struct softirq_action *h)
3307+{
3308+ struct list_head *cpu_list;
3309+ LIST_HEAD(local_list);
3310+
3311+ local_irq_disable();
3312+ cpu_list = &__get_cpu_var(blk_cpu_done);
3313+ list_splice_init(cpu_list, &local_list);
3314+ local_irq_enable();
3315+
3316+ while (!list_empty(&local_list)) {
3317+ struct request *rq = list_entry(local_list.next, struct request, donelist);
3318+
3319+ list_del_init(&rq->donelist);
3320+ rq->q->softirq_done_fn(rq);
3321+ }
3322+}
3323+
3324+#ifdef CONFIG_HOTPLUG_CPU
3325+
3326+static int blk_cpu_notify(struct notifier_block *self, unsigned long action,
3327+ void *hcpu)
3328+{
3329+ /*
3330+ * If a CPU goes away, splice its entries to the current CPU
3331+ * and trigger a run of the softirq
3332+ */
3333+ if (action == CPU_DEAD) {
3334+ int cpu = (unsigned long) hcpu;
3335+
3336+ local_irq_disable();
3337+ list_splice_init(&per_cpu(blk_cpu_done, cpu),
3338+ &__get_cpu_var(blk_cpu_done));
3339+ raise_softirq_irqoff(BLOCK_SOFTIRQ);
3340+ local_irq_enable();
3341+ }
3342+
3343+ return NOTIFY_OK;
3344+}
3345+
3346+
3347+static struct notifier_block __devinitdata blk_cpu_notifier = {
3348+ .notifier_call = blk_cpu_notify,
3349+};
3350+
3351+#endif /* CONFIG_HOTPLUG_CPU */
3352+
3353+/**
3354+ * blk_complete_request - end I/O on a request
3355+ * @req: the request being processed
3356+ *
3357+ * Description:
3358+ * Ends all I/O on a request. It does not handle partial completions,
3359+ * unless the driver actually implements this in its completionc callback
3360+ * through requeueing. Theh actual completion happens out-of-order,
3361+ * through a softirq handler. The user must have registered a completion
3362+ * callback through blk_queue_softirq_done().
3363+ **/
3364+
3365+void blk_complete_request(struct request *req)
3366+{
3367+ struct list_head *cpu_list;
3368+ unsigned long flags;
3369+
3370+ BUG_ON(!req->q->softirq_done_fn);
3371+
3372+ local_irq_save(flags);
3373+
3374+ cpu_list = &__get_cpu_var(blk_cpu_done);
3375+ list_add_tail(&req->donelist, cpu_list);
3376+ raise_softirq_irqoff(BLOCK_SOFTIRQ);
3377+
3378+ local_irq_restore(flags);
3379+}
3380+
3381+EXPORT_SYMBOL(blk_complete_request);
3382+
32893383 /*
32903384 * queue lock must be held
32913385 */
@@ -3364,6 +3458,8 @@ EXPORT_SYMBOL(kblockd_flush); · 72 unmodified lines
33643458
33653459 int __init blk_dev_init(void)
33663460 {
3461+ int i;
3462+
33673463 kblockd_workqueue = create_workqueue("kblockd");
33683464 if (!kblockd_workqueue)
33693465 panic("Failed to create kblockd\n");
@@ -3377,6 +3473,14 @@ int __init blk_dev_init(void) · 7 unmodified lines
33773473 iocontext_cachep = kmem_cache_create("blkdev_ioc",
33783474 sizeof(struct io_context), 0, SLAB_PANIC, NULL, NULL);
33793475
3476+ for (i = 0; i < NR_CPUS; i++)
3477+ INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i));
3478+
3479+ open_softirq(BLOCK_SOFTIRQ, blk_done_softirq, NULL);
3480+#ifdef CONFIG_HOTPLUG_CPU
3481+ register_cpu_notifier(&blk_cpu_notifier);
3482+#endif
3483+
33803484 blk_max_low_pfn = max_low_pfn;
33813485 blk_max_pfn = max_pfn;
33823486