History

[BLOCK] ll_rw_blk: Enable out-of-order request completions through softirq

Request completion can be a quite heavy process, since it needs to iterate through the entire request and complete the bio's it holds. This patch adds blk_complete_request() which moves this processing into a dedicated block softirq.

Jens Axboe committed 20 years ago

Commit ff856ba1 parent 5367f2d3 files · +124 −4Browse files

Unified · Split

3 files
  1. block/ll_rw_blk.c +105 −1
  2. include/linux/blkdev.h +18 −3
  3. include/linux/interrupt.h +1 −0
block/ll_rw_blk.c+105−1
@@ -27,6 +27,8 @@ · 26 unmodified lines
2727 #include <linux/swap.h>
2828 #include <linux/writeback.h>
2929 #include <linux/blkdev.h>
30+#include <linux/interrupt.h>
31+#include <linux/cpu.h>
3032
3133 /*
3234 * for max sense size
@@ -62,13 +64,15 @@ static wait_queue_head_t congestion_wqh[2] = { · 29 unmodified lines
6264 /*
6365 * Controlling structure to kblockd
6466 */
65−static struct workqueue_struct *kblockd_workqueue;
67+static struct workqueue_struct *kblockd_workqueue;
6668
6769 unsigned long blk_max_low_pfn, blk_max_pfn;
6870
6971 EXPORT_SYMBOL(blk_max_low_pfn);
7072 EXPORT_SYMBOL(blk_max_pfn);
7173
74+static DEFINE_PER_CPU(struct list_head, blk_cpu_done);
75+
7276 /* Amount of time in which a process may batch requests */
7377 #define BLK_BATCH_TIME (HZ/50UL)
7478
@@ -207,6 +211,13 @@ void blk_queue_merge_bvec(request_queue_t *q, merge_bvec_fn *mbfn) · 132 unmodified lines
207211
208212 EXPORT_SYMBOL(blk_queue_merge_bvec);
209213
214+void blk_queue_softirq_done(request_queue_t *q, softirq_done_fn *fn)
215+{
216+ q->softirq_done_fn = fn;
217+}
218+
219+EXPORT_SYMBOL(blk_queue_softirq_done);
220+
210221 /**
211222 * blk_queue_make_request - define an alternate make_request function for a device
212223 * @q: the request queue for the device to be affected
@@ -270,6 +281,7 @@ EXPORT_SYMBOL(blk_queue_make_request); · 57 unmodified lines
270281 static inline void rq_init(request_queue_t *q, struct request *rq)
271282 {
272283 INIT_LIST_HEAD(&rq->queuelist);
284+ INIT_LIST_HEAD(&rq->donelist);
273285
274286 rq->errors = 0;
275287 rq->rq_status = RQ_ACTIVE;
@@ -286,6 +298,7 @@ static inline void rq_init(request_queue_t *q, struct request *rq) · 10 unmodified lines
286298 rq->sense = NULL;
287299 rq->end_io = NULL;
288300 rq->end_io_data = NULL;
301+ rq->completion_data = NULL;
289302 }
290303
291304 /**
@@ -3286,6 +3299,87 @@ int end_that_request_chunk(struct request *req, int uptodate, int nr_bytes) · 2994 unmodified lines
32863299
32873300 EXPORT_SYMBOL(end_that_request_chunk);
32883301
3302+/*
3303+ * splice the completion data to a local structure and hand off to
3304+ * process_completion_queue() to complete the requests
3305+ */
3306+static void blk_done_softirq(struct softirq_action *h)
3307+{
3308+ struct list_head *cpu_list;
3309+ LIST_HEAD(local_list);
3310+
3311+ local_irq_disable();
3312+ cpu_list = &__get_cpu_var(blk_cpu_done);
3313+ list_splice_init(cpu_list, &local_list);
3314+ local_irq_enable();
3315+
3316+ while (!list_empty(&local_list)) {
3317+ struct request *rq = list_entry(local_list.next, struct request, donelist);
3318+
3319+ list_del_init(&rq->donelist);
3320+ rq->q->softirq_done_fn(rq);
3321+ }
3322+}
3323+
3324+#ifdef CONFIG_HOTPLUG_CPU
3325+
3326+static int blk_cpu_notify(struct notifier_block *self, unsigned long action,
3327+ void *hcpu)
3328+{
3329+ /*
3330+ * If a CPU goes away, splice its entries to the current CPU
3331+ * and trigger a run of the softirq
3332+ */
3333+ if (action == CPU_DEAD) {
3334+ int cpu = (unsigned long) hcpu;
3335+
3336+ local_irq_disable();
3337+ list_splice_init(&per_cpu(blk_cpu_done, cpu),
3338+ &__get_cpu_var(blk_cpu_done));
3339+ raise_softirq_irqoff(BLOCK_SOFTIRQ);
3340+ local_irq_enable();
3341+ }
3342+
3343+ return NOTIFY_OK;
3344+}
3345+
3346+
3347+static struct notifier_block __devinitdata blk_cpu_notifier = {
3348+ .notifier_call = blk_cpu_notify,
3349+};
3350+
3351+#endif /* CONFIG_HOTPLUG_CPU */
3352+
3353+/**
3354+ * blk_complete_request - end I/O on a request
3355+ * @req: the request being processed
3356+ *
3357+ * Description:
3358+ * Ends all I/O on a request. It does not handle partial completions,
3359+ * unless the driver actually implements this in its completionc callback
3360+ * through requeueing. Theh actual completion happens out-of-order,
3361+ * through a softirq handler. The user must have registered a completion
3362+ * callback through blk_queue_softirq_done().
3363+ **/
3364+
3365+void blk_complete_request(struct request *req)
3366+{
3367+ struct list_head *cpu_list;
3368+ unsigned long flags;
3369+
3370+ BUG_ON(!req->q->softirq_done_fn);
3371+
3372+ local_irq_save(flags);
3373+
3374+ cpu_list = &__get_cpu_var(blk_cpu_done);
3375+ list_add_tail(&req->donelist, cpu_list);
3376+ raise_softirq_irqoff(BLOCK_SOFTIRQ);
3377+
3378+ local_irq_restore(flags);
3379+}
3380+
3381+EXPORT_SYMBOL(blk_complete_request);
3382+
32893383 /*
32903384 * queue lock must be held
32913385 */
@@ -3364,6 +3458,8 @@ EXPORT_SYMBOL(kblockd_flush); · 72 unmodified lines
33643458
33653459 int __init blk_dev_init(void)
33663460 {
3461+ int i;
3462+
33673463 kblockd_workqueue = create_workqueue("kblockd");
33683464 if (!kblockd_workqueue)
33693465 panic("Failed to create kblockd\n");
@@ -3377,6 +3473,14 @@ int __init blk_dev_init(void) · 7 unmodified lines
33773473 iocontext_cachep = kmem_cache_create("blkdev_ioc",
33783474 sizeof(struct io_context), 0, SLAB_PANIC, NULL, NULL);
33793475
3476+ for (i = 0; i < NR_CPUS; i++)
3477+ INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i));
3478+
3479+ open_softirq(BLOCK_SOFTIRQ, blk_done_softirq, NULL);
3480+#ifdef CONFIG_HOTPLUG_CPU
3481+ register_cpu_notifier(&blk_cpu_notifier);
3482+#endif
3483+
33803484 blk_max_low_pfn = max_low_pfn;
33813485 blk_max_pfn = max_pfn;
33823486
include/linux/blkdev.h+18−3
@@ -118,9 +118,9 @@ struct request_list { · 117 unmodified lines
118118 * try to put the fields that are referenced together in the same cacheline
119119 */
120120 struct request {
121− struct list_head queuelist; /* looking for ->queue? you must _not_
122− * access it directly, use
123− * blkdev_dequeue_request! */
121+ struct list_head queuelist;
122+ struct list_head donelist;
123+
124124 unsigned long flags; /* see REQ_ bits below */
125125
126126 /* Maintain bio traversal state for part by part I/O submission.
@@ -141,6 +141,7 @@ struct request { · 14 unmodified lines
141141 struct bio *biotail;
142142
143143 void *elevator_private;
144+ void *completion_data;
144145
145146 unsigned short ioprio;
146147
@@ -291,6 +292,7 @@ typedef int (merge_bvec_fn) (request_queue_t *, struct bio *, struct bio_vec *); · 144 unmodified lines
291292 typedef void (activity_fn) (void *data, int rw);
292293 typedef int (issue_flush_fn) (request_queue_t *, struct gendisk *, sector_t *);
293294 typedef void (prepare_flush_fn) (request_queue_t *, struct request *);
295+typedef void (softirq_done_fn)(struct request *);
294296
295297 enum blk_queue_state {
296298 Queue_down,
@@ -332,6 +334,7 @@ struct request_queue · 35 unmodified lines
332334 activity_fn *activity_fn;
333335 issue_flush_fn *issue_flush_fn;
334336 prepare_flush_fn *prepare_flush_fn;
337+ softirq_done_fn *softirq_done_fn;
335338
336339 /*
337340 * Dispatch queue sorting
@@ -646,6 +649,17 @@ extern int end_that_request_first(struct request *, int, int); · 308 unmodified lines
646649 extern int end_that_request_chunk(struct request *, int, int);
647650 extern void end_that_request_last(struct request *, int);
648651 extern void end_request(struct request *req, int uptodate);
652+extern void blk_complete_request(struct request *);
653+
654+static inline int rq_all_done(struct request *rq, unsigned int nr_bytes)
655+{
656+ if (blk_fs_request(rq))
657+ return (nr_bytes >= (rq->hard_nr_sectors << 9));
658+ else if (blk_pc_request(rq))
659+ return nr_bytes >= rq->data_len;
660+
661+ return 0;
662+}
649663
650664 /*
651665 * end_that_request_first/chunk() takes an uptodate argument. we account
@@ -694,6 +708,7 @@ extern void blk_queue_segment_boundary(request_queue_t *, unsigned long); · 42 unmodified lines
694708 extern void blk_queue_prep_rq(request_queue_t *, prep_rq_fn *pfn);
695709 extern void blk_queue_merge_bvec(request_queue_t *, merge_bvec_fn *);
696710 extern void blk_queue_dma_alignment(request_queue_t *, int);
711+extern void blk_queue_softirq_done(request_queue_t *, softirq_done_fn *);
697712 extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
698713 extern int blk_queue_ordered(request_queue_t *, unsigned, prepare_flush_fn *);
699714 extern void blk_queue_issue_flush_fn(request_queue_t *, issue_flush_fn *);
include/linux/interrupt.h+1−0
@@ -112,6 +112,7 @@ enum · 111 unmodified lines
112112 TIMER_SOFTIRQ,
113113 NET_TX_SOFTIRQ,
114114 NET_RX_SOFTIRQ,
115+ BLOCK_SOFTIRQ,
115116 SCSI_SOFTIRQ,
116117 TASKLET_SOFTIRQ
117118 };