Request completion can be a quite heavy process, since it needs to iterate through the entire request and complete the bio's it holds. This patch adds blk_complete_request() which moves this processing into a dedicated block softirq.
Showing the changes to block (1 of 3 files). Show all
Unified · Split
| @@ -27,6 +27,8 @@ · 26 unmodified lines | |||
| 27 | #include <linux/swap.h> | 27 | #include <linux/swap.h> |
| 28 | #include <linux/writeback.h> | 28 | #include <linux/writeback.h> |
| 29 | #include <linux/blkdev.h> | 29 | #include <linux/blkdev.h> |
| 30 | #include <linux/interrupt.h> | ||
| 31 | #include <linux/cpu.h> | ||
| 30 | 32 | ||
| 31 | /* | 33 | /* |
| 32 | * for max sense size | 34 | * for max sense size |
| @@ -62,13 +64,15 @@ static wait_queue_head_t congestion_wqh[2] = { · 29 unmodified lines | |||
| 62 | /* | 64 | /* |
| 63 | * Controlling structure to kblockd | 65 | * Controlling structure to kblockd |
| 64 | */ | 66 | */ |
| 65 | static struct workqueue_struct *kblockd_workqueue; | 67 | static struct workqueue_struct *kblockd_workqueue; |
| 66 | 68 | ||
| 67 | unsigned long blk_max_low_pfn, blk_max_pfn; | 69 | unsigned long blk_max_low_pfn, blk_max_pfn; |
| 68 | 70 | ||
| 69 | EXPORT_SYMBOL(blk_max_low_pfn); | 71 | EXPORT_SYMBOL(blk_max_low_pfn); |
| 70 | EXPORT_SYMBOL(blk_max_pfn); | 72 | EXPORT_SYMBOL(blk_max_pfn); |
| 71 | 73 | ||
| 74 | static DEFINE_PER_CPU(struct list_head, blk_cpu_done); | ||
| 75 | |||
| 72 | /* Amount of time in which a process may batch requests */ | 76 | /* Amount of time in which a process may batch requests */ |
| 73 | #define BLK_BATCH_TIME (HZ/50UL) | 77 | #define BLK_BATCH_TIME (HZ/50UL) |
| 74 | 78 | ||
| @@ -207,6 +211,13 @@ void blk_queue_merge_bvec(request_queue_t *q, merge_bvec_fn *mbfn) · 132 unmodified lines | |||
| 207 | 211 | ||
| 208 | EXPORT_SYMBOL(blk_queue_merge_bvec); | 212 | EXPORT_SYMBOL(blk_queue_merge_bvec); |
| 209 | 213 | ||
| 214 | void blk_queue_softirq_done(request_queue_t *q, softirq_done_fn *fn) | ||
| 215 | { | ||
| 216 | q->softirq_done_fn = fn; | ||
| 217 | } | ||
| 218 | |||
| 219 | EXPORT_SYMBOL(blk_queue_softirq_done); | ||
| 220 | |||
| 210 | /** | 221 | /** |
| 211 | * blk_queue_make_request - define an alternate make_request function for a device | 222 | * blk_queue_make_request - define an alternate make_request function for a device |
| 212 | * @q: the request queue for the device to be affected | 223 | * @q: the request queue for the device to be affected |
| @@ -270,6 +281,7 @@ EXPORT_SYMBOL(blk_queue_make_request); · 57 unmodified lines | |||
| 270 | static inline void rq_init(request_queue_t *q, struct request *rq) | 281 | static inline void rq_init(request_queue_t *q, struct request *rq) |
| 271 | { | 282 | { |
| 272 | INIT_LIST_HEAD(&rq->queuelist); | 283 | INIT_LIST_HEAD(&rq->queuelist); |
| 284 | INIT_LIST_HEAD(&rq->donelist); | ||
| 273 | 285 | ||
| 274 | rq->errors = 0; | 286 | rq->errors = 0; |
| 275 | rq->rq_status = RQ_ACTIVE; | 287 | rq->rq_status = RQ_ACTIVE; |
| @@ -286,6 +298,7 @@ static inline void rq_init(request_queue_t *q, struct request *rq) · 10 unmodified lines | |||
| 286 | rq->sense = NULL; | 298 | rq->sense = NULL; |
| 287 | rq->end_io = NULL; | 299 | rq->end_io = NULL; |
| 288 | rq->end_io_data = NULL; | 300 | rq->end_io_data = NULL; |
| 301 | rq->completion_data = NULL; | ||
| 289 | } | 302 | } |
| 290 | 303 | ||
| 291 | /** | 304 | /** |
| @@ -3286,6 +3299,87 @@ int end_that_request_chunk(struct request *req, int uptodate, int nr_bytes) · 2994 unmodified lines | |||
| 3286 | 3299 | ||
| 3287 | EXPORT_SYMBOL(end_that_request_chunk); | 3300 | EXPORT_SYMBOL(end_that_request_chunk); |
| 3288 | 3301 | ||
| 3302 | /* | ||
| 3303 | * splice the completion data to a local structure and hand off to | ||
| 3304 | * process_completion_queue() to complete the requests | ||
| 3305 | */ | ||
| 3306 | static void blk_done_softirq(struct softirq_action *h) | ||
| 3307 | { | ||
| 3308 | struct list_head *cpu_list; | ||
| 3309 | LIST_HEAD(local_list); | ||
| 3310 | |||
| 3311 | local_irq_disable(); | ||
| 3312 | cpu_list = &__get_cpu_var(blk_cpu_done); | ||
| 3313 | list_splice_init(cpu_list, &local_list); | ||
| 3314 | local_irq_enable(); | ||
| 3315 | |||
| 3316 | while (!list_empty(&local_list)) { | ||
| 3317 | struct request *rq = list_entry(local_list.next, struct request, donelist); | ||
| 3318 | |||
| 3319 | list_del_init(&rq->donelist); | ||
| 3320 | rq->q->softirq_done_fn(rq); | ||
| 3321 | } | ||
| 3322 | } | ||
| 3323 | |||
| 3324 | #ifdef CONFIG_HOTPLUG_CPU | ||
| 3325 | |||
| 3326 | static int blk_cpu_notify(struct notifier_block *self, unsigned long action, | ||
| 3327 | void *hcpu) | ||
| 3328 | { | ||
| 3329 | /* | ||
| 3330 | * If a CPU goes away, splice its entries to the current CPU | ||
| 3331 | * and trigger a run of the softirq | ||
| 3332 | */ | ||
| 3333 | if (action == CPU_DEAD) { | ||
| 3334 | int cpu = (unsigned long) hcpu; | ||
| 3335 | |||
| 3336 | local_irq_disable(); | ||
| 3337 | list_splice_init(&per_cpu(blk_cpu_done, cpu), | ||
| 3338 | &__get_cpu_var(blk_cpu_done)); | ||
| 3339 | raise_softirq_irqoff(BLOCK_SOFTIRQ); | ||
| 3340 | local_irq_enable(); | ||
| 3341 | } | ||
| 3342 | |||
| 3343 | return NOTIFY_OK; | ||
| 3344 | } | ||
| 3345 | |||
| 3346 | |||
| 3347 | static struct notifier_block __devinitdata blk_cpu_notifier = { | ||
| 3348 | .notifier_call = blk_cpu_notify, | ||
| 3349 | }; | ||
| 3350 | |||
| 3351 | #endif /* CONFIG_HOTPLUG_CPU */ | ||
| 3352 | |||
| 3353 | /** | ||
| 3354 | * blk_complete_request - end I/O on a request | ||
| 3355 | * @req: the request being processed | ||
| 3356 | * | ||
| 3357 | * Description: | ||
| 3358 | * Ends all I/O on a request. It does not handle partial completions, | ||
| 3359 | * unless the driver actually implements this in its completionc callback | ||
| 3360 | * through requeueing. Theh actual completion happens out-of-order, | ||
| 3361 | * through a softirq handler. The user must have registered a completion | ||
| 3362 | * callback through blk_queue_softirq_done(). | ||
| 3363 | **/ | ||
| 3364 | |||
| 3365 | void blk_complete_request(struct request *req) | ||
| 3366 | { | ||
| 3367 | struct list_head *cpu_list; | ||
| 3368 | unsigned long flags; | ||
| 3369 | |||
| 3370 | BUG_ON(!req->q->softirq_done_fn); | ||
| 3371 | |||
| 3372 | local_irq_save(flags); | ||
| 3373 | |||
| 3374 | cpu_list = &__get_cpu_var(blk_cpu_done); | ||
| 3375 | list_add_tail(&req->donelist, cpu_list); | ||
| 3376 | raise_softirq_irqoff(BLOCK_SOFTIRQ); | ||
| 3377 | |||
| 3378 | local_irq_restore(flags); | ||
| 3379 | } | ||
| 3380 | |||
| 3381 | EXPORT_SYMBOL(blk_complete_request); | ||
| 3382 | |||
| 3289 | /* | 3383 | /* |
| 3290 | * queue lock must be held | 3384 | * queue lock must be held |
| 3291 | */ | 3385 | */ |
| @@ -3364,6 +3458,8 @@ EXPORT_SYMBOL(kblockd_flush); · 72 unmodified lines | |||
| 3364 | 3458 | ||
| 3365 | int __init blk_dev_init(void) | 3459 | int __init blk_dev_init(void) |
| 3366 | { | 3460 | { |
| 3461 | int i; | ||
| 3462 | |||
| 3367 | kblockd_workqueue = create_workqueue("kblockd"); | 3463 | kblockd_workqueue = create_workqueue("kblockd"); |
| 3368 | if (!kblockd_workqueue) | 3464 | if (!kblockd_workqueue) |
| 3369 | panic("Failed to create kblockd\n"); | 3465 | panic("Failed to create kblockd\n"); |
| @@ -3377,6 +3473,14 @@ int __init blk_dev_init(void) · 7 unmodified lines | |||
| 3377 | iocontext_cachep = kmem_cache_create("blkdev_ioc", | 3473 | iocontext_cachep = kmem_cache_create("blkdev_ioc", |
| 3378 | sizeof(struct io_context), 0, SLAB_PANIC, NULL, NULL); | 3474 | sizeof(struct io_context), 0, SLAB_PANIC, NULL, NULL); |
| 3379 | 3475 | ||
| 3476 | for (i = 0; i < NR_CPUS; i++) | ||
| 3477 | INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i)); | ||
| 3478 | |||
| 3479 | open_softirq(BLOCK_SOFTIRQ, blk_done_softirq, NULL); | ||
| 3480 | #ifdef CONFIG_HOTPLUG_CPU | ||
| 3481 | register_cpu_notifier(&blk_cpu_notifier); | ||
| 3482 | #endif | ||
| 3483 | |||
| 3380 | blk_max_low_pfn = max_low_pfn; | 3484 | blk_max_low_pfn = max_low_pfn; |
| 3381 | blk_max_pfn = max_pfn; | 3485 | blk_max_pfn = max_pfn; |
| 3382 | 3486 | ||