History

blk-mq: drain I/O when all CPUs in a hctx are offline

Most of blk-mq drivers depend on managed IRQ's auto-affinity to setup up queue mapping. Thomas mentioned the following point[1]: "That was the constraint of managed interrupts from the very beginning: The driver/subsystem has to quiesce the interrupt line and the associated queue _before_ it gets shutdown in CPU unplug and not fiddle with it until it's restarted by the core when the CPU is plugged in again." However, current blk-mq implementation doesn't quiesce hw queue before the last CPU in the hctx is shutdown. Even worse, CPUHP_BLK_MQ_DEAD is a cpuhp state handled after the CPU is down, so there isn't any chance to quiesce the hctx before shutting down the CPU. Add new CPUHP_AP_BLK_MQ_ONLINE state to stop allocating from blk-mq hctxs where the last CPU goes away, and wait for completion of in-flight requests. This guarantees that there is no inflight I/O before shutting down the managed IRQ. Add a BLK_MQ_F_STACKING and set it for dm-rq and loop, so we don't need to wait for completion of in-flight requests from these drivers to avoid a potential dead-lock. It is safe to do this for stacking drivers as those do not use interrupts at all and their I/O completions are triggered by underlying devices I/O completion. [1] https://lore.kernel.org/linux-block/alpine.DEB.2.21.1904051331270.1802@nanos.tec.linutronix.de/ [hch: different retry mechanism, merged two patches, minor cleanups]

Ming Lei committed 6 years ago

Commit bf0beec1 parent 602380d7 files · +133 −4Browse files

Unified · Split

7 files
  1. block/blk-mq-debugfs.c +2 −0
  2. block/blk-mq-tag.c +8 −0
  3. block/blk-mq.c +110 −2
  4. drivers/block/loop.c +1 −1
  5. drivers/md/dm-rq.c +1 −1
  6. include/linux/blk-mq.h +10 −0
  7. include/linux/cpuhotplug.h +1 −0
block/blk-mq-debugfs.c+2−0
@@ -213,6 +213,7 @@ static const char *const hctx_state_name[] = { · 212 unmodified lines
213 HCTX_STATE_NAME(STOPPED),213 HCTX_STATE_NAME(STOPPED),
214 HCTX_STATE_NAME(TAG_ACTIVE),214 HCTX_STATE_NAME(TAG_ACTIVE),
215 HCTX_STATE_NAME(SCHED_RESTART),215 HCTX_STATE_NAME(SCHED_RESTART),
216 HCTX_STATE_NAME(INACTIVE),
216};217};
217#undef HCTX_STATE_NAME218#undef HCTX_STATE_NAME
218 219
@@ -239,6 +240,7 @@ static const char *const hctx_flag_name[] = { · 20 unmodified lines
239 HCTX_FLAG_NAME(TAG_SHARED),240 HCTX_FLAG_NAME(TAG_SHARED),
240 HCTX_FLAG_NAME(BLOCKING),241 HCTX_FLAG_NAME(BLOCKING),
241 HCTX_FLAG_NAME(NO_SCHED),242 HCTX_FLAG_NAME(NO_SCHED),
243 HCTX_FLAG_NAME(STACKING),
242};244};
243#undef HCTX_FLAG_NAME245#undef HCTX_FLAG_NAME
244 246
block/blk-mq-tag.c+8−0
@@ -180,6 +180,14 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) · 179 unmodified lines
180 sbitmap_finish_wait(bt, ws, &wait);180 sbitmap_finish_wait(bt, ws, &wait);
181 181
182found_tag:182found_tag:
183 /*
184 * Give up this allocation if the hctx is inactive. The caller will
185 * retry on an active hctx.
186 */
187 if (unlikely(test_bit(BLK_MQ_S_INACTIVE, &data->hctx->state))) {
188 blk_mq_put_tag(tags, data->ctx, tag + tag_offset);
189 return BLK_MQ_NO_TAG;
190 }
183 return tag + tag_offset;191 return tag + tag_offset;
184}192}
185 193
block/blk-mq.c+110−2
@@ -375,14 +375,30 @@ static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data) · 374 unmodified lines
375 e->type->ops.limit_depth(data->cmd_flags, data);375 e->type->ops.limit_depth(data->cmd_flags, data);
376 }376 }
377 377
378retry:
378 data->ctx = blk_mq_get_ctx(q);379 data->ctx = blk_mq_get_ctx(q);
379 data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);380 data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
380 if (!(data->flags & BLK_MQ_REQ_INTERNAL))381 if (!(data->flags & BLK_MQ_REQ_INTERNAL))
381 blk_mq_tag_busy(data->hctx);382 blk_mq_tag_busy(data->hctx);
382 383
384 /*
385 * Waiting allocations only fail because of an inactive hctx. In that
386 * case just retry the hctx assignment and tag allocation as CPU hotplug
387 * should have migrated us to an online CPU by now.
388 */
383 tag = blk_mq_get_tag(data);389 tag = blk_mq_get_tag(data);
384 if (tag == BLK_MQ_NO_TAG)390 if (tag == BLK_MQ_NO_TAG) {
385 return NULL;391 if (data->flags & BLK_MQ_REQ_NOWAIT)
392 return NULL;
393
394 /*
395 * Give up the CPU and sleep for a random short time to ensure
396 * that thread using a realtime scheduling class are migrated
397 * off the the CPU, and thus off the hctx that is going away.
398 */
399 msleep(3);
400 goto retry;
401 }
386 return blk_mq_rq_ctx_init(data, tag, alloc_time_ns);402 return blk_mq_rq_ctx_init(data, tag, alloc_time_ns);
387}403}
388 404
@@ -2335,6 +2351,86 @@ fail: · 1946 unmodified lines
2335 return -ENOMEM;2351 return -ENOMEM;
2336}2352}
2337 2353
2354struct rq_iter_data {
2355 struct blk_mq_hw_ctx *hctx;
2356 bool has_rq;
2357};
2358
2359static bool blk_mq_has_request(struct request *rq, void *data, bool reserved)
2360{
2361 struct rq_iter_data *iter_data = data;
2362
2363 if (rq->mq_hctx != iter_data->hctx)
2364 return true;
2365 iter_data->has_rq = true;
2366 return false;
2367}
2368
2369static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
2370{
2371 struct blk_mq_tags *tags = hctx->sched_tags ?
2372 hctx->sched_tags : hctx->tags;
2373 struct rq_iter_data data = {
2374 .hctx = hctx,
2375 };
2376
2377 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
2378 return data.has_rq;
2379}
2380
2381static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
2382 struct blk_mq_hw_ctx *hctx)
2383{
2384 if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu)
2385 return false;
2386 if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
2387 return false;
2388 return true;
2389}
2390
2391static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
2392{
2393 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
2394 struct blk_mq_hw_ctx, cpuhp_online);
2395
2396 if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
2397 !blk_mq_last_cpu_in_hctx(cpu, hctx))
2398 return 0;
2399
2400 /*
2401 * Prevent new request from being allocated on the current hctx.
2402 *
2403 * The smp_mb__after_atomic() Pairs with the implied barrier in
2404 * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is
2405 * seen once we return from the tag allocator.
2406 */
2407 set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
2408 smp_mb__after_atomic();
2409
2410 /*
2411 * Try to grab a reference to the queue and wait for any outstanding
2412 * requests. If we could not grab a reference the queue has been
2413 * frozen and there are no requests.
2414 */
2415 if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
2416 while (blk_mq_hctx_has_requests(hctx))
2417 msleep(5);
2418 percpu_ref_put(&hctx->queue->q_usage_counter);
2419 }
2420
2421 return 0;
2422}
2423
2424static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
2425{
2426 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
2427 struct blk_mq_hw_ctx, cpuhp_online);
2428
2429 if (cpumask_test_cpu(cpu, hctx->cpumask))
2430 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
2431 return 0;
2432}
2433
2338/*2434/*
2339 * 'cpu' is going away. splice any existing rq_list entries from this2435 * 'cpu' is going away. splice any existing rq_list entries from this
2340 * software queue to the hw queue dispatch list, and ensure that it2436 * software queue to the hw queue dispatch list, and ensure that it
@@ -2348,6 +2444,9 @@ static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) · 7 unmodified lines
2348 enum hctx_type type;2444 enum hctx_type type;
2349 2445
2350 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);2446 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
2447 if (!cpumask_test_cpu(cpu, hctx->cpumask))
2448 return 0;
2449
2351 ctx = __blk_mq_get_ctx(hctx->queue, cpu);2450 ctx = __blk_mq_get_ctx(hctx->queue, cpu);
2352 type = hctx->type;2451 type = hctx->type;
2353 2452
@@ -2371,6 +2470,9 @@ static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) · 17 unmodified lines
2371 2470
2372static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)2471static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
2373{2472{
2473 if (!(hctx->flags & BLK_MQ_F_STACKING))
2474 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
2475 &hctx->cpuhp_online);
2374 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,2476 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
2375 &hctx->cpuhp_dead);2477 &hctx->cpuhp_dead);
2376}2478}
@@ -2430,6 +2532,9 @@ static int blk_mq_init_hctx(struct request_queue *q, · 53 unmodified lines
2430{2532{
2431 hctx->queue_num = hctx_idx;2533 hctx->queue_num = hctx_idx;
2432 2534
2535 if (!(hctx->flags & BLK_MQ_F_STACKING))
2536 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
2537 &hctx->cpuhp_online);
2433 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);2538 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
2434 2539
2435 hctx->tags = set->tags[hctx_idx];2540 hctx->tags = set->tags[hctx_idx];
@@ -3684,6 +3789,9 @@ static int __init blk_mq_init(void) · 1248 unmodified lines
3684{3789{
3685 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,3790 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
3686 blk_mq_hctx_notify_dead);3791 blk_mq_hctx_notify_dead);
3792 cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
3793 blk_mq_hctx_notify_online,
3794 blk_mq_hctx_notify_offline);
3687 return 0;3795 return 0;
3688}3796}
3689subsys_initcall(blk_mq_init);3797subsys_initcall(blk_mq_init);
drivers/block/loop.c+1−1
@@ -2037,7 +2037,7 @@ static int loop_add(struct loop_device **l, int i) · 2036 unmodified lines
2037 lo->tag_set.queue_depth = 128;2037 lo->tag_set.queue_depth = 128;
2038 lo->tag_set.numa_node = NUMA_NO_NODE;2038 lo->tag_set.numa_node = NUMA_NO_NODE;
2039 lo->tag_set.cmd_size = sizeof(struct loop_cmd);2039 lo->tag_set.cmd_size = sizeof(struct loop_cmd);
2040 lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;2040 lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_STACKING;
2041 lo->tag_set.driver_data = lo;2041 lo->tag_set.driver_data = lo;
2042 2042
2043 err = blk_mq_alloc_tag_set(&lo->tag_set);2043 err = blk_mq_alloc_tag_set(&lo->tag_set);
drivers/md/dm-rq.c+1−1
@@ -547,7 +547,7 @@ int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t) · 546 unmodified lines
547 md->tag_set->ops = &dm_mq_ops;547 md->tag_set->ops = &dm_mq_ops;
548 md->tag_set->queue_depth = dm_get_blk_mq_queue_depth();548 md->tag_set->queue_depth = dm_get_blk_mq_queue_depth();
549 md->tag_set->numa_node = md->numa_node_id;549 md->tag_set->numa_node = md->numa_node_id;
550 md->tag_set->flags = BLK_MQ_F_SHOULD_MERGE;550 md->tag_set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_STACKING;
551 md->tag_set->nr_hw_queues = dm_get_blk_mq_nr_hw_queues();551 md->tag_set->nr_hw_queues = dm_get_blk_mq_nr_hw_queues();
552 md->tag_set->driver_data = md;552 md->tag_set->driver_data = md;
553 553
include/linux/blk-mq.h+10−0
@@ -140,6 +140,8 @@ struct blk_mq_hw_ctx { · 139 unmodified lines
140 */140 */
141 atomic_t nr_active;141 atomic_t nr_active;
142 142
143 /** @cpuhp_online: List to store request if CPU is going to die */
144 struct hlist_node cpuhp_online;
143 /** @cpuhp_dead: List to store request if some CPU die. */145 /** @cpuhp_dead: List to store request if some CPU die. */
144 struct hlist_node cpuhp_dead;146 struct hlist_node cpuhp_dead;
145 /** @kobj: Kernel object for sysfs. */147 /** @kobj: Kernel object for sysfs. */
@@ -391,6 +393,11 @@ struct blk_mq_ops { · 245 unmodified lines
391enum {393enum {
392 BLK_MQ_F_SHOULD_MERGE = 1 << 0,394 BLK_MQ_F_SHOULD_MERGE = 1 << 0,
393 BLK_MQ_F_TAG_SHARED = 1 << 1,395 BLK_MQ_F_TAG_SHARED = 1 << 1,
396 /*
397 * Set when this device requires underlying blk-mq device for
398 * completing IO:
399 */
400 BLK_MQ_F_STACKING = 1 << 2,
394 BLK_MQ_F_BLOCKING = 1 << 5,401 BLK_MQ_F_BLOCKING = 1 << 5,
395 BLK_MQ_F_NO_SCHED = 1 << 6,402 BLK_MQ_F_NO_SCHED = 1 << 6,
396 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,403 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
@@ -400,6 +407,9 @@ enum { · 3 unmodified lines
400 BLK_MQ_S_TAG_ACTIVE = 1,407 BLK_MQ_S_TAG_ACTIVE = 1,
401 BLK_MQ_S_SCHED_RESTART = 2,408 BLK_MQ_S_SCHED_RESTART = 2,
402 409
410 /* hw queue is inactive after all its CPUs become offline */
411 BLK_MQ_S_INACTIVE = 3,
412
403 BLK_MQ_MAX_DEPTH = 10240,413 BLK_MQ_MAX_DEPTH = 10240,
404 414
405 BLK_MQ_CPU_WORK_BATCH = 8,415 BLK_MQ_CPU_WORK_BATCH = 8,
include/linux/cpuhotplug.h+1−0
@@ -152,6 +152,7 @@ enum cpuhp_state { · 151 unmodified lines
152 CPUHP_AP_SMPBOOT_THREADS,152 CPUHP_AP_SMPBOOT_THREADS,
153 CPUHP_AP_X86_VDSO_VMA_ONLINE,153 CPUHP_AP_X86_VDSO_VMA_ONLINE,
154 CPUHP_AP_IRQ_AFFINITY_ONLINE,154 CPUHP_AP_IRQ_AFFINITY_ONLINE,
155 CPUHP_AP_BLK_MQ_ONLINE,
155 CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS,156 CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS,
156 CPUHP_AP_X86_INTEL_EPB_ONLINE,157 CPUHP_AP_X86_INTEL_EPB_ONLINE,
157 CPUHP_AP_PERF_ONLINE,158 CPUHP_AP_PERF_ONLINE,