Intel VT-d driver was reworked to use common deferred flushing implementation. Previously there was one global per-cpu flush queue, afterwards - one per domain. Before deferring a flush, the queue should be allocated and initialized. Currently only domains with IOMMU_DOMAIN_DMA type initialize their flush queue. It's probably worth to init it for static or unmanaged domains too, but it may be arguable - I'm leaving it to iommu folks. Prevent queuing an iova flush if the domain doesn't have a queue. The defensive check seems to be worth to keep even if queue would be initialized for all kinds of domains. And is easy backportable. On 4.19.43 stable kernel it has a user-visible effect: previously for devices in si domain there were crashes, on sata devices: BUG: spinlock bad magic on CPU#6, swapper/0/1 lock: 0xffff88844f582008, .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0 CPU: 6 PID: 1 Comm: swapper/0 Not tainted 4.19.43 #1 Call Trace: <IRQ> dump_stack+0x61/0x7e spin_bug+0x9d/0xa3 do_raw_spin_lock+0x22/0x8e _raw_spin_lock_irqsave+0x32/0x3a queue_iova+0x45/0x115 intel_unmap+0x107/0x113 intel_unmap_sg+0x6b/0x76 __ata_qc_complete+0x7f/0x103 ata_qc_complete+0x9b/0x26a ata_qc_complete_multiple+0xd0/0xe3 ahci_handle_port_interrupt+0x3ee/0x48a ahci_handle_port_intr+0x73/0xa9 ahci_single_level_irq_intr+0x40/0x60 __handle_irq_event_percpu+0x7f/0x19a handle_irq_event_percpu+0x32/0x72 handle_irq_event+0x38/0x56 handle_edge_irq+0x102/0x121 handle_irq+0x147/0x15c do_IRQ+0x66/0xf2 common_interrupt+0xf/0xf RIP: 0010:__do_softirq+0x8c/0x2df The same for usb devices that use ehci-pci: BUG: spinlock bad magic on CPU#0, swapper/0/1 lock: 0xffff88844f402008, .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0 CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.19.43 #4 Call Trace: <IRQ> dump_stack+0x61/0x7e spin_bug+0x9d/0xa3 do_raw_spin_lock+0x22/0x8e _raw_spin_lock_irqsave+0x32/0x3a queue_iova+0x77/0x145 intel_unmap+0x107/0x113 intel_unmap_page+0xe/0x10 usb_hcd_unmap_urb_setup_for_dma+0x53/0x9d usb_hcd_unmap_urb_for_dma+0x17/0x100 unmap_urb_for_dma+0x22/0x24 __usb_hcd_giveback_urb+0x51/0xc3 usb_giveback_urb_bh+0x97/0xde tasklet_action_common.isra.4+0x5f/0xa1 tasklet_action+0x2d/0x30 __do_softirq+0x138/0x2df irq_exit+0x7d/0x8b smp_apic_timer_interrupt+0x10f/0x151 apic_timer_interrupt+0xf/0x20 </IRQ> RIP: 0010:_raw_spin_unlock_irqrestore+0x17/0x39
Unified · Split
| @@ -3561,7 +3561,8 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) · 3560 unmodified lines | ||
| 3561 | 3561 | |
| 3562 | 3562 | freelist = domain_unmap(domain, start_pfn, last_pfn); |
| 3563 | 3563 | |
| 3564 | − if (intel_iommu_strict || (pdev && pdev->untrusted)) { | |
| 3564 | + if (intel_iommu_strict || (pdev && pdev->untrusted) || | |
| 3565 | + !has_iova_flush_queue(&domain->iovad)) { | |
| 3565 | 3566 | iommu_flush_iotlb_psi(iommu, domain, start_pfn, |
| 3566 | 3567 | nrpages, !freelist, 0); |
| 3567 | 3568 | /* free iova */ |
| @@ -54,9 +54,14 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, · 53 unmodified lines | ||
| 54 | 54 | } |
| 55 | 55 | EXPORT_SYMBOL_GPL(init_iova_domain); |
| 56 | 56 | |
| 57 | +bool has_iova_flush_queue(struct iova_domain *iovad) | |
| 58 | +{ | |
| 59 | + return !!iovad->fq; | |
| 60 | +} | |
| 61 | + | |
| 57 | 62 | static void free_iova_flush_queue(struct iova_domain *iovad) |
| 58 | 63 | { |
| 59 | − if (!iovad->fq) | |
| 64 | + if (!has_iova_flush_queue(iovad)) | |
| 60 | 65 | return; |
| 61 | 66 | |
| 62 | 67 | if (timer_pending(&iovad->fq_timer)) |
| @@ -74,13 +79,14 @@ static void free_iova_flush_queue(struct iova_domain *iovad) · 11 unmodified lines | ||
| 74 | 79 | int init_iova_flush_queue(struct iova_domain *iovad, |
| 75 | 80 | iova_flush_cb flush_cb, iova_entry_dtor entry_dtor) |
| 76 | 81 | { |
| 82 | + struct iova_fq __percpu *queue; | |
| 77 | 83 | int cpu; |
| 78 | 84 | |
| 79 | 85 | atomic64_set(&iovad->fq_flush_start_cnt, 0); |
| 80 | 86 | atomic64_set(&iovad->fq_flush_finish_cnt, 0); |
| 81 | 87 | |
| 82 | − iovad->fq = alloc_percpu(struct iova_fq); | |
| 83 | − if (!iovad->fq) | |
| 88 | + queue = alloc_percpu(struct iova_fq); | |
| 89 | + if (!queue) | |
| 84 | 90 | return -ENOMEM; |
| 85 | 91 | |
| 86 | 92 | iovad->flush_cb = flush_cb; |
| @@ -89,13 +95,17 @@ int init_iova_flush_queue(struct iova_domain *iovad, · 2 unmodified lines | ||
| 89 | 95 | for_each_possible_cpu(cpu) { |
| 90 | 96 | struct iova_fq *fq; |
| 91 | 97 | |
| 92 | − fq = per_cpu_ptr(iovad->fq, cpu); | |
| 98 | + fq = per_cpu_ptr(queue, cpu); | |
| 93 | 99 | fq->head = 0; |
| 94 | 100 | fq->tail = 0; |
| 95 | 101 | |
| 96 | 102 | spin_lock_init(&fq->lock); |
| 97 | 103 | } |
| 98 | 104 | |
| 105 | + smp_wmb(); | |
| 106 | + | |
| 107 | + iovad->fq = queue; | |
| 108 | + | |
| 99 | 109 | timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); |
| 100 | 110 | atomic_set(&iovad->fq_timer_on, 0); |
| 101 | 111 | |
| @@ -155,6 +155,7 @@ struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, · 154 unmodified lines | ||
| 155 | 155 | void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to); |
| 156 | 156 | void init_iova_domain(struct iova_domain *iovad, unsigned long granule, |
| 157 | 157 | unsigned long start_pfn); |
| 158 | +bool has_iova_flush_queue(struct iova_domain *iovad); | |
| 158 | 159 | int init_iova_flush_queue(struct iova_domain *iovad, |
| 159 | 160 | iova_flush_cb flush_cb, iova_entry_dtor entry_dtor); |
| 160 | 161 | struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); |
| @@ -235,6 +236,11 @@ static inline void init_iova_domain(struct iova_domain *iovad, · 74 unmodified lines | ||
| 235 | 236 | { |
| 236 | 237 | } |
| 237 | 238 | |
| 239 | +bool has_iova_flush_queue(struct iova_domain *iovad) | |
| 240 | +{ | |
| 241 | + return false; | |
| 242 | +} | |
| 243 | + | |
| 238 | 244 | static inline int init_iova_flush_queue(struct iova_domain *iovad, |
| 239 | 245 | iova_flush_cb flush_cb, |
| 240 | 246 | iova_entry_dtor entry_dtor) |