Pull IOMMU fixes from Joerg Roedel: - fix a scheduling-while-atomic bug in the AMD IOMMU driver. It was found after the checker was enabled earlier. - a fix for the virtual APIC code in the AMD IOMMU driver which delivers device interrupts directly into KVM guests for assigned devices. - fixes for the recently merged lock-less page-table code for ARM. The redundant TLB syncs got reverted and locks added again around the TLB sync code. - fix for error handling in arm_smmu_add_device() - address sanitization fix for arm io-pgtable code * tag 'iommu-fixes-v4.13-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: iommu/amd: Fix schedule-while-atomic BUG in initialization code iommu/amd: Enable ga_log_intr when enabling guest_mode iommu/io-pgtable: Sanitise map/unmap addresses iommu/arm-smmu: Fix the error path in arm_smmu_add_device Revert "iommu/io-pgtable: Avoid redundant TLB syncs" iommu/mtk: Avoid redundant TLB syncs locally iommu/arm-smmu: Reintroduce locking around TLB sync operations
Unified · Split
| @@ -4452,6 +4452,7 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) · 4451 unmodified lines | ||
| 4452 | 4452 | /* Setting */ |
| 4453 | 4453 | irte->hi.fields.ga_root_ptr = (pi_data->base >> 12); |
| 4454 | 4454 | irte->hi.fields.vector = vcpu_pi_info->vector; |
| 4455 | + irte->lo.fields_vapic.ga_log_intr = 1; | |
| 4455 | 4456 | irte->lo.fields_vapic.guest_mode = 1; |
| 4456 | 4457 | irte->lo.fields_vapic.ga_tag = pi_data->ga_tag; |
| 4457 | 4458 | |
| @@ -2440,11 +2440,11 @@ static int __init state_next(void) · 2439 unmodified lines | ||
| 2440 | 2440 | break; |
| 2441 | 2441 | case IOMMU_ACPI_FINISHED: |
| 2442 | 2442 | early_enable_iommus(); |
| 2443 | − register_syscore_ops(&amd_iommu_syscore_ops); | |
| 2444 | 2443 | x86_platform.iommu_shutdown = disable_iommus; |
| 2445 | 2444 | init_state = IOMMU_ENABLED; |
| 2446 | 2445 | break; |
| 2447 | 2446 | case IOMMU_ENABLED: |
| 2447 | + register_syscore_ops(&amd_iommu_syscore_ops); | |
| 2448 | 2448 | ret = amd_iommu_init_pci(); |
| 2449 | 2449 | init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT; |
| 2450 | 2450 | enable_iommus_v2(); |
| @@ -400,6 +400,8 @@ struct arm_smmu_device { · 399 unmodified lines | ||
| 400 | 400 | |
| 401 | 401 | u32 cavium_id_base; /* Specific to Cavium */ |
| 402 | 402 | |
| 403 | + spinlock_t global_sync_lock; | |
| 404 | + | |
| 403 | 405 | /* IOMMU core code handle */ |
| 404 | 406 | struct iommu_device iommu; |
| 405 | 407 | }; |
| @@ -436,7 +438,7 @@ struct arm_smmu_domain { · 30 unmodified lines | ||
| 436 | 438 | struct arm_smmu_cfg cfg; |
| 437 | 439 | enum arm_smmu_domain_stage stage; |
| 438 | 440 | struct mutex init_mutex; /* Protects smmu pointer */ |
| 439 | − spinlock_t cb_lock; /* Serialises ATS1* ops */ | |
| 441 | + spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */ | |
| 440 | 442 | struct iommu_domain domain; |
| 441 | 443 | }; |
| 442 | 444 | |
| @@ -602,9 +604,12 @@ static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, · 159 unmodified lines | ||
| 602 | 604 | static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) |
| 603 | 605 | { |
| 604 | 606 | void __iomem *base = ARM_SMMU_GR0(smmu); |
| 607 | + unsigned long flags; | |
| 605 | 608 | |
| 609 | + spin_lock_irqsave(&smmu->global_sync_lock, flags); | |
| 606 | 610 | __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC, |
| 607 | 611 | base + ARM_SMMU_GR0_sTLBGSTATUS); |
| 612 | + spin_unlock_irqrestore(&smmu->global_sync_lock, flags); | |
| 608 | 613 | } |
| 609 | 614 | |
| 610 | 615 | static void arm_smmu_tlb_sync_context(void *cookie) |
| @@ -612,9 +617,12 @@ static void arm_smmu_tlb_sync_context(void *cookie) · 1 unmodified line | ||
| 612 | 617 | struct arm_smmu_domain *smmu_domain = cookie; |
| 613 | 618 | struct arm_smmu_device *smmu = smmu_domain->smmu; |
| 614 | 619 | void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx); |
| 620 | + unsigned long flags; | |
| 615 | 621 | |
| 622 | + spin_lock_irqsave(&smmu_domain->cb_lock, flags); | |
| 616 | 623 | __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC, |
| 617 | 624 | base + ARM_SMMU_CB_TLBSTATUS); |
| 625 | + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); | |
| 618 | 626 | } |
| 619 | 627 | |
| 620 | 628 | static void arm_smmu_tlb_sync_vmid(void *cookie) |
| @@ -1511,7 +1519,6 @@ static int arm_smmu_add_device(struct device *dev) · 890 unmodified lines | ||
| 1511 | 1519 | |
| 1512 | 1520 | if (using_legacy_binding) { |
| 1513 | 1521 | ret = arm_smmu_register_legacy_master(dev, &smmu); |
| 1514 | − fwspec = dev->iommu_fwspec; | |
| 1515 | 1522 | if (ret) |
| 1516 | 1523 | goto out_free; |
| 1517 | 1524 | } else if (fwspec && fwspec->ops == &arm_smmu_ops) { |
| @@ -1550,15 +1557,15 @@ static int arm_smmu_add_device(struct device *dev) · 32 unmodified lines | ||
| 1550 | 1557 | |
| 1551 | 1558 | ret = arm_smmu_master_alloc_smes(dev); |
| 1552 | 1559 | if (ret) |
| 1553 | − goto out_free; | |
| 1560 | + goto out_cfg_free; | |
| 1554 | 1561 | |
| 1555 | 1562 | iommu_device_link(&smmu->iommu, dev); |
| 1556 | 1563 | |
| 1557 | 1564 | return 0; |
| 1558 | 1565 | |
| 1566 | +out_cfg_free: | |
| 1567 | + kfree(cfg); | |
| 1559 | 1568 | out_free: |
| 1560 | − if (fwspec) | |
| 1561 | − kfree(fwspec->iommu_priv); | |
| 1562 | 1569 | iommu_fwspec_free(dev); |
| 1563 | 1570 | return ret; |
| 1564 | 1571 | } |
| @@ -1925,6 +1932,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 360 unmodified lines | ||
| 1925 | 1932 | |
| 1926 | 1933 | smmu->num_mapping_groups = size; |
| 1927 | 1934 | mutex_init(&smmu->stream_map_mutex); |
| 1935 | + spin_lock_init(&smmu->global_sync_lock); | |
| 1928 | 1936 | |
| 1929 | 1937 | if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { |
| 1930 | 1938 | smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; |
| @@ -479,6 +479,9 @@ static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, · 478 unmodified lines | ||
| 479 | 479 | if (!(prot & (IOMMU_READ | IOMMU_WRITE))) |
| 480 | 480 | return 0; |
| 481 | 481 | |
| 482 | + if (WARN_ON(upper_32_bits(iova) || upper_32_bits(paddr))) | |
| 483 | + return -ERANGE; | |
| 484 | + | |
| 482 | 485 | ret = __arm_v7s_map(data, iova, paddr, size, prot, 1, data->pgd); |
| 483 | 486 | /* |
| 484 | 487 | * Synchronise all PTE updates for the new mapping before there's |
| @@ -659,6 +662,9 @@ static int arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, · 174 unmodified lines | ||
| 659 | 662 | struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); |
| 660 | 663 | size_t unmapped; |
| 661 | 664 | |
| 665 | + if (WARN_ON(upper_32_bits(iova))) | |
| 666 | + return 0; | |
| 667 | + | |
| 662 | 668 | unmapped = __arm_v7s_unmap(data, iova, size, 1, data->pgd); |
| 663 | 669 | if (unmapped) |
| 664 | 670 | io_pgtable_tlb_sync(&data->iop); |
| @@ -452,6 +452,10 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, · 451 unmodified lines | ||
| 452 | 452 | if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) |
| 453 | 453 | return 0; |
| 454 | 454 | |
| 455 | + if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || | |
| 456 | + paddr >= (1ULL << data->iop.cfg.oas))) | |
| 457 | + return -ERANGE; | |
| 458 | + | |
| 455 | 459 | prot = arm_lpae_prot_to_pte(data, iommu_prot); |
| 456 | 460 | ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); |
| 457 | 461 | /* |
| @@ -610,6 +614,9 @@ static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, · 152 unmodified lines | ||
| 610 | 614 | arm_lpae_iopte *ptep = data->pgd; |
| 611 | 615 | int lvl = ARM_LPAE_START_LVL(data); |
| 612 | 616 | |
| 617 | + if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) | |
| 618 | + return 0; | |
| 619 | + | |
| 613 | 620 | unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep); |
| 614 | 621 | if (unmapped) |
| 615 | 622 | io_pgtable_tlb_sync(&data->iop); |
| @@ -158,14 +158,12 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops); · 157 unmodified lines | ||
| 158 | 158 | * @fmt: The page table format. |
| 159 | 159 | * @cookie: An opaque token provided by the IOMMU driver and passed back to |
| 160 | 160 | * any callback routines. |
| 161 | − * @tlb_sync_pending: Private flag for optimising out redundant syncs. | |
| 162 | 161 | * @cfg: A copy of the page table configuration. |
| 163 | 162 | * @ops: The page table operations in use for this set of page tables. |
| 164 | 163 | */ |
| 165 | 164 | struct io_pgtable { |
| 166 | 165 | enum io_pgtable_fmt fmt; |
| 167 | 166 | void *cookie; |
| 168 | − bool tlb_sync_pending; | |
| 169 | 167 | struct io_pgtable_cfg cfg; |
| 170 | 168 | struct io_pgtable_ops ops; |
| 171 | 169 | }; |
| @@ -175,22 +173,17 @@ struct io_pgtable { · 3 unmodified lines | ||
| 175 | 173 | static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) |
| 176 | 174 | { |
| 177 | 175 | iop->cfg.tlb->tlb_flush_all(iop->cookie); |
| 178 | − iop->tlb_sync_pending = true; | |
| 179 | 176 | } |
| 180 | 177 | |
| 181 | 178 | static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, |
| 182 | 179 | unsigned long iova, size_t size, size_t granule, bool leaf) |
| 183 | 180 | { |
| 184 | 181 | iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); |
| 185 | − iop->tlb_sync_pending = true; | |
| 186 | 182 | } |
| 187 | 183 | |
| 188 | 184 | static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) |
| 189 | 185 | { |
| 190 | − if (iop->tlb_sync_pending) { | |
| 191 | − iop->cfg.tlb->tlb_sync(iop->cookie); | |
| 192 | − iop->tlb_sync_pending = false; | |
| 193 | − } | |
| 186 | + iop->cfg.tlb->tlb_sync(iop->cookie); | |
| 194 | 187 | } |
| 195 | 188 | |
| 196 | 189 | /** |
| @@ -129,6 +129,7 @@ static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size, · 128 unmodified lines | ||
| 129 | 129 | writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A); |
| 130 | 130 | writel_relaxed(iova + size - 1, data->base + REG_MMU_INVLD_END_A); |
| 131 | 131 | writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); |
| 132 | + data->tlb_flush_active = true; | |
| 132 | 133 | } |
| 133 | 134 | |
| 134 | 135 | static void mtk_iommu_tlb_sync(void *cookie) |
| @@ -137,6 +138,10 @@ static void mtk_iommu_tlb_sync(void *cookie) · 2 unmodified lines | ||
| 137 | 138 | int ret; |
| 138 | 139 | u32 tmp; |
| 139 | 140 | |
| 141 | + /* Avoid timing out if there's nothing to wait for */ | |
| 142 | + if (!data->tlb_flush_active) | |
| 143 | + return; | |
| 144 | + | |
| 140 | 145 | ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp, |
| 141 | 146 | tmp != 0, 10, 100000); |
| 142 | 147 | if (ret) { |
| @@ -146,6 +151,7 @@ static void mtk_iommu_tlb_sync(void *cookie) · 3 unmodified lines | ||
| 146 | 151 | } |
| 147 | 152 | /* Clear the CPE status */ |
| 148 | 153 | writel_relaxed(0, data->base + REG_MMU_CPE_DONE); |
| 154 | + data->tlb_flush_active = false; | |
| 149 | 155 | } |
| 150 | 156 | |
| 151 | 157 | static const struct iommu_gather_ops mtk_iommu_gather_ops = { |
| @@ -47,6 +47,7 @@ struct mtk_iommu_data { · 46 unmodified lines | ||
| 47 | 47 | struct iommu_group *m4u_group; |
| 48 | 48 | struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */ |
| 49 | 49 | bool enable_4GB; |
| 50 | + bool tlb_flush_active; | |
| 50 | 51 | |
| 51 | 52 | struct iommu_device iommu; |
| 52 | 53 | }; |