Most of blk-mq drivers depend on managed IRQ's auto-affinity to setup up queue mapping. Thomas mentioned the following point[1]: "That was the constraint of managed interrupts from the very beginning: The driver/subsystem has to quiesce the interrupt line and the associated queue _before_ it gets shutdown in CPU unplug and not fiddle with it until it's restarted by the core when the CPU is plugged in again." However, current blk-mq implementation doesn't quiesce hw queue before the last CPU in the hctx is shutdown. Even worse, CPUHP_BLK_MQ_DEAD is a cpuhp state handled after the CPU is down, so there isn't any chance to quiesce the hctx before shutting down the CPU. Add new CPUHP_AP_BLK_MQ_ONLINE state to stop allocating from blk-mq hctxs where the last CPU goes away, and wait for completion of in-flight requests. This guarantees that there is no inflight I/O before shutting down the managed IRQ. Add a BLK_MQ_F_STACKING and set it for dm-rq and loop, so we don't need to wait for completion of in-flight requests from these drivers to avoid a potential dead-lock. It is safe to do this for stacking drivers as those do not use interrupts at all and their I/O completions are triggered by underlying devices I/O completion. [1] https://lore.kernel.org/linux-block/alpine.DEB.2.21.1904051331270.1802@nanos.tec.linutronix.de/ [hch: different retry mechanism, merged two patches, minor cleanups]
Showing the changes to include/linux (2 of 7 files). Show all
Unified · Split
| @@ -140,6 +140,8 @@ struct blk_mq_hw_ctx { · 139 unmodified lines | ||
| 140 | 140 | */ |
| 141 | 141 | atomic_t nr_active; |
| 142 | 142 | |
| 143 | + /** @cpuhp_online: List to store request if CPU is going to die */ | |
| 144 | + struct hlist_node cpuhp_online; | |
| 143 | 145 | /** @cpuhp_dead: List to store request if some CPU die. */ |
| 144 | 146 | struct hlist_node cpuhp_dead; |
| 145 | 147 | /** @kobj: Kernel object for sysfs. */ |
| @@ -391,6 +393,11 @@ struct blk_mq_ops { · 245 unmodified lines | ||
| 391 | 393 | enum { |
| 392 | 394 | BLK_MQ_F_SHOULD_MERGE = 1 << 0, |
| 393 | 395 | BLK_MQ_F_TAG_SHARED = 1 << 1, |
| 396 | + /* | |
| 397 | + * Set when this device requires underlying blk-mq device for | |
| 398 | + * completing IO: | |
| 399 | + */ | |
| 400 | + BLK_MQ_F_STACKING = 1 << 2, | |
| 394 | 401 | BLK_MQ_F_BLOCKING = 1 << 5, |
| 395 | 402 | BLK_MQ_F_NO_SCHED = 1 << 6, |
| 396 | 403 | BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, |
| @@ -400,6 +407,9 @@ enum { · 3 unmodified lines | ||
| 400 | 407 | BLK_MQ_S_TAG_ACTIVE = 1, |
| 401 | 408 | BLK_MQ_S_SCHED_RESTART = 2, |
| 402 | 409 | |
| 410 | + /* hw queue is inactive after all its CPUs become offline */ | |
| 411 | + BLK_MQ_S_INACTIVE = 3, | |
| 412 | + | |
| 403 | 413 | BLK_MQ_MAX_DEPTH = 10240, |
| 404 | 414 | |
| 405 | 415 | BLK_MQ_CPU_WORK_BATCH = 8, |
| @@ -152,6 +152,7 @@ enum cpuhp_state { · 151 unmodified lines | ||
| 152 | 152 | CPUHP_AP_SMPBOOT_THREADS, |
| 153 | 153 | CPUHP_AP_X86_VDSO_VMA_ONLINE, |
| 154 | 154 | CPUHP_AP_IRQ_AFFINITY_ONLINE, |
| 155 | + CPUHP_AP_BLK_MQ_ONLINE, | |
| 155 | 156 | CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS, |
| 156 | 157 | CPUHP_AP_X86_INTEL_EPB_ONLINE, |
| 157 | 158 | CPUHP_AP_PERF_ONLINE, |