History

blk-mq: drain I/O when all CPUs in a hctx are offline

Most of blk-mq drivers depend on managed IRQ's auto-affinity to setup up queue mapping. Thomas mentioned the following point[1]: "That was the constraint of managed interrupts from the very beginning: The driver/subsystem has to quiesce the interrupt line and the associated queue _before_ it gets shutdown in CPU unplug and not fiddle with it until it's restarted by the core when the CPU is plugged in again." However, current blk-mq implementation doesn't quiesce hw queue before the last CPU in the hctx is shutdown. Even worse, CPUHP_BLK_MQ_DEAD is a cpuhp state handled after the CPU is down, so there isn't any chance to quiesce the hctx before shutting down the CPU. Add new CPUHP_AP_BLK_MQ_ONLINE state to stop allocating from blk-mq hctxs where the last CPU goes away, and wait for completion of in-flight requests. This guarantees that there is no inflight I/O before shutting down the managed IRQ. Add a BLK_MQ_F_STACKING and set it for dm-rq and loop, so we don't need to wait for completion of in-flight requests from these drivers to avoid a potential dead-lock. It is safe to do this for stacking drivers as those do not use interrupts at all and their I/O completions are triggered by underlying devices I/O completion. [1] https://lore.kernel.org/linux-block/alpine.DEB.2.21.1904051331270.1802@nanos.tec.linutronix.de/ [hch: different retry mechanism, merged two patches, minor cleanups]

Ming Lei committed 6 years ago

Commit bf0beec1 parent 602380d7 files · +133 −4Browse files

Unified · Split

7 files
  1. block/blk-mq-debugfs.c +2 −0
  2. block/blk-mq-tag.c +8 −0
  3. block/blk-mq.c +110 −2
  4. drivers/block/loop.c +1 −1
  5. drivers/md/dm-rq.c +1 −1
  6. include/linux/blk-mq.h +10 −0
  7. include/linux/cpuhotplug.h +1 −0
block/blk-mq-debugfs.c+2−0
@@ -213,6 +213,7 @@ static const char *const hctx_state_name[] = { · 212 unmodified lines
213213 HCTX_STATE_NAME(STOPPED),
214214 HCTX_STATE_NAME(TAG_ACTIVE),
215215 HCTX_STATE_NAME(SCHED_RESTART),
216+ HCTX_STATE_NAME(INACTIVE),
216217 };
217218 #undef HCTX_STATE_NAME
218219
@@ -239,6 +240,7 @@ static const char *const hctx_flag_name[] = { · 20 unmodified lines
239240 HCTX_FLAG_NAME(TAG_SHARED),
240241 HCTX_FLAG_NAME(BLOCKING),
241242 HCTX_FLAG_NAME(NO_SCHED),
243+ HCTX_FLAG_NAME(STACKING),
242244 };
243245 #undef HCTX_FLAG_NAME
244246
block/blk-mq-tag.c+8−0
@@ -180,6 +180,14 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) · 179 unmodified lines
180180 sbitmap_finish_wait(bt, ws, &wait);
181181
182182 found_tag:
183+ /*
184+ * Give up this allocation if the hctx is inactive. The caller will
185+ * retry on an active hctx.
186+ */
187+ if (unlikely(test_bit(BLK_MQ_S_INACTIVE, &data->hctx->state))) {
188+ blk_mq_put_tag(tags, data->ctx, tag + tag_offset);
189+ return BLK_MQ_NO_TAG;
190+ }
183191 return tag + tag_offset;
184192 }
185193
block/blk-mq.c+110−2
@@ -375,14 +375,30 @@ static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data) · 374 unmodified lines
375375 e->type->ops.limit_depth(data->cmd_flags, data);
376376 }
377377
378+retry:
378379 data->ctx = blk_mq_get_ctx(q);
379380 data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
380381 if (!(data->flags & BLK_MQ_REQ_INTERNAL))
381382 blk_mq_tag_busy(data->hctx);
382383
384+ /*
385+ * Waiting allocations only fail because of an inactive hctx. In that
386+ * case just retry the hctx assignment and tag allocation as CPU hotplug
387+ * should have migrated us to an online CPU by now.
388+ */
383389 tag = blk_mq_get_tag(data);
384− if (tag == BLK_MQ_NO_TAG)
385− return NULL;
390+ if (tag == BLK_MQ_NO_TAG) {
391+ if (data->flags & BLK_MQ_REQ_NOWAIT)
392+ return NULL;
393+
394+ /*
395+ * Give up the CPU and sleep for a random short time to ensure
396+ * that thread using a realtime scheduling class are migrated
397+ * off the the CPU, and thus off the hctx that is going away.
398+ */
399+ msleep(3);
400+ goto retry;
401+ }
386402 return blk_mq_rq_ctx_init(data, tag, alloc_time_ns);
387403 }
388404
@@ -2335,6 +2351,86 @@ fail: · 1946 unmodified lines
23352351 return -ENOMEM;
23362352 }
23372353
2354+struct rq_iter_data {
2355+ struct blk_mq_hw_ctx *hctx;
2356+ bool has_rq;
2357+};
2358+
2359+static bool blk_mq_has_request(struct request *rq, void *data, bool reserved)
2360+{
2361+ struct rq_iter_data *iter_data = data;
2362+
2363+ if (rq->mq_hctx != iter_data->hctx)
2364+ return true;
2365+ iter_data->has_rq = true;
2366+ return false;
2367+}
2368+
2369+static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
2370+{
2371+ struct blk_mq_tags *tags = hctx->sched_tags ?
2372+ hctx->sched_tags : hctx->tags;
2373+ struct rq_iter_data data = {
2374+ .hctx = hctx,
2375+ };
2376+
2377+ blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
2378+ return data.has_rq;
2379+}
2380+
2381+static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
2382+ struct blk_mq_hw_ctx *hctx)
2383+{
2384+ if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu)
2385+ return false;
2386+ if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
2387+ return false;
2388+ return true;
2389+}
2390+
2391+static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
2392+{
2393+ struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
2394+ struct blk_mq_hw_ctx, cpuhp_online);
2395+
2396+ if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
2397+ !blk_mq_last_cpu_in_hctx(cpu, hctx))
2398+ return 0;
2399+
2400+ /*
2401+ * Prevent new request from being allocated on the current hctx.
2402+ *
2403+ * The smp_mb__after_atomic() Pairs with the implied barrier in
2404+ * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is
2405+ * seen once we return from the tag allocator.
2406+ */
2407+ set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
2408+ smp_mb__after_atomic();
2409+
2410+ /*
2411+ * Try to grab a reference to the queue and wait for any outstanding
2412+ * requests. If we could not grab a reference the queue has been
2413+ * frozen and there are no requests.
2414+ */
2415+ if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
2416+ while (blk_mq_hctx_has_requests(hctx))
2417+ msleep(5);
2418+ percpu_ref_put(&hctx->queue->q_usage_counter);
2419+ }
2420+
2421+ return 0;
2422+}
2423+
2424+static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
2425+{
2426+ struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
2427+ struct blk_mq_hw_ctx, cpuhp_online);
2428+
2429+ if (cpumask_test_cpu(cpu, hctx->cpumask))
2430+ clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
2431+ return 0;
2432+}
2433+
23382434 /*
23392435 * 'cpu' is going away. splice any existing rq_list entries from this
23402436 * software queue to the hw queue dispatch list, and ensure that it
@@ -2348,6 +2444,9 @@ static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) · 7 unmodified lines
23482444 enum hctx_type type;
23492445
23502446 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
2447+ if (!cpumask_test_cpu(cpu, hctx->cpumask))
2448+ return 0;
2449+
23512450 ctx = __blk_mq_get_ctx(hctx->queue, cpu);
23522451 type = hctx->type;
23532452
@@ -2371,6 +2470,9 @@ static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) · 17 unmodified lines
23712470
23722471 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
23732472 {
2473+ if (!(hctx->flags & BLK_MQ_F_STACKING))
2474+ cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
2475+ &hctx->cpuhp_online);
23742476 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
23752477 &hctx->cpuhp_dead);
23762478 }
@@ -2430,6 +2532,9 @@ static int blk_mq_init_hctx(struct request_queue *q, · 53 unmodified lines
24302532 {
24312533 hctx->queue_num = hctx_idx;
24322534
2535+ if (!(hctx->flags & BLK_MQ_F_STACKING))
2536+ cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
2537+ &hctx->cpuhp_online);
24332538 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
24342539
24352540 hctx->tags = set->tags[hctx_idx];
@@ -3684,6 +3789,9 @@ static int __init blk_mq_init(void) · 1248 unmodified lines
36843789 {
36853790 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
36863791 blk_mq_hctx_notify_dead);
3792+ cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
3793+ blk_mq_hctx_notify_online,
3794+ blk_mq_hctx_notify_offline);
36873795 return 0;
36883796 }
36893797 subsys_initcall(blk_mq_init);
drivers/block/loop.c+1−1
@@ -2037,7 +2037,7 @@ static int loop_add(struct loop_device **l, int i) · 2036 unmodified lines
20372037 lo->tag_set.queue_depth = 128;
20382038 lo->tag_set.numa_node = NUMA_NO_NODE;
20392039 lo->tag_set.cmd_size = sizeof(struct loop_cmd);
2040− lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
2040+ lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_STACKING;
20412041 lo->tag_set.driver_data = lo;
20422042
20432043 err = blk_mq_alloc_tag_set(&lo->tag_set);
drivers/md/dm-rq.c+1−1
@@ -547,7 +547,7 @@ int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t) · 546 unmodified lines
547547 md->tag_set->ops = &dm_mq_ops;
548548 md->tag_set->queue_depth = dm_get_blk_mq_queue_depth();
549549 md->tag_set->numa_node = md->numa_node_id;
550− md->tag_set->flags = BLK_MQ_F_SHOULD_MERGE;
550+ md->tag_set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_STACKING;
551551 md->tag_set->nr_hw_queues = dm_get_blk_mq_nr_hw_queues();
552552 md->tag_set->driver_data = md;
553553
include/linux/blk-mq.h+10−0
@@ -140,6 +140,8 @@ struct blk_mq_hw_ctx { · 139 unmodified lines
140140 */
141141 atomic_t nr_active;
142142
143+ /** @cpuhp_online: List to store request if CPU is going to die */
144+ struct hlist_node cpuhp_online;
143145 /** @cpuhp_dead: List to store request if some CPU die. */
144146 struct hlist_node cpuhp_dead;
145147 /** @kobj: Kernel object for sysfs. */
@@ -391,6 +393,11 @@ struct blk_mq_ops { · 245 unmodified lines
391393 enum {
392394 BLK_MQ_F_SHOULD_MERGE = 1 << 0,
393395 BLK_MQ_F_TAG_SHARED = 1 << 1,
396+ /*
397+ * Set when this device requires underlying blk-mq device for
398+ * completing IO:
399+ */
400+ BLK_MQ_F_STACKING = 1 << 2,
394401 BLK_MQ_F_BLOCKING = 1 << 5,
395402 BLK_MQ_F_NO_SCHED = 1 << 6,
396403 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
@@ -400,6 +407,9 @@ enum { · 3 unmodified lines
400407 BLK_MQ_S_TAG_ACTIVE = 1,
401408 BLK_MQ_S_SCHED_RESTART = 2,
402409
410+ /* hw queue is inactive after all its CPUs become offline */
411+ BLK_MQ_S_INACTIVE = 3,
412+
403413 BLK_MQ_MAX_DEPTH = 10240,
404414
405415 BLK_MQ_CPU_WORK_BATCH = 8,
include/linux/cpuhotplug.h+1−0
@@ -152,6 +152,7 @@ enum cpuhp_state { · 151 unmodified lines
152152 CPUHP_AP_SMPBOOT_THREADS,
153153 CPUHP_AP_X86_VDSO_VMA_ONLINE,
154154 CPUHP_AP_IRQ_AFFINITY_ONLINE,
155+ CPUHP_AP_BLK_MQ_ONLINE,
155156 CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS,
156157 CPUHP_AP_X86_INTEL_EPB_ONLINE,
157158 CPUHP_AP_PERF_ONLINE,