Pull IOMMU updates from Joerg Roedel: - A big cleanup and optimization patch-set for the Tegra GART driver - Documentation updates and fixes for the IOMMU-API - Support for page request in Intel VT-d scalable mode - Intel VT-d dma_[un]map_resource() support - Updates to the ATS enabling code for PCI (acked by Bjorn) and Intel VT-d to align with the latest version of the ATS spec - Relaxed IRQ source checking in the Intel VT-d driver for some aliased devices, needed for future devices which send IRQ messages from more than on request-ID - IRQ remapping driver for Hyper-V - Patches to make generic IOVA and IO-Page-Table code usable outside of the IOMMU code - Various other small fixes and cleanups * tag 'iommu-updates-v5.1' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (60 commits) iommu/vt-d: Get domain ID before clear pasid entry iommu/vt-d: Fix NULL pointer reference in intel_svm_bind_mm() iommu/vt-d: Set context field after value initialized iommu/vt-d: Disable ATS support on untrusted devices iommu/mediatek: Fix semicolon code style issue MAINTAINERS: Add Hyper-V IOMMU driver into Hyper-V CORE AND DRIVERS scope iommu/hyper-v: Add Hyper-V stub IOMMU driver x86/Hyper-V: Set x2apic destination mode to physical when x2apic is available PCI/ATS: Add inline to pci_prg_resp_pasid_required() iommu/vt-d: Check identity map for hot-added devices iommu: Fix IOMMU debugfs fallout iommu: Document iommu_ops.is_attach_deferred() iommu: Document iommu_ops.iotlb_sync_map() iommu/vt-d: Enable ATS only if the device uses page aligned address. PCI/ATS: Add pci_ats_page_aligned() interface iommu/vt-d: Fix PRI/PASID dependency issue. PCI/ATS: Add pci_prg_resp_pasid_required() interface. iommu/vt-d: Allow interrupts from the entire bus for aliased devices iommu/vt-d: Add helper to set an IRTE to verify only the bus number iommu: Fix flush_tlb_all typo ...
Showing the changes to include/linux (6 of 44 files). Show all
Unified · Split
| @@ -374,20 +374,17 @@ enum { · 373 unmodified lines | |||
| 374 | #define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | ((u64)(pfsid & 0xfff) << 52)) | 374 | #define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | ((u64)(pfsid & 0xfff) << 52)) |
| 375 | #define QI_DEV_EIOTLB_MAX_INVS 32 | 375 | #define QI_DEV_EIOTLB_MAX_INVS 32 |
| 376 | 376 | ||
| 377 | #define QI_PGRP_IDX(idx) (((u64)(idx)) << 55) | 377 | /* Page group response descriptor QW0 */ |
| 378 | #define QI_PGRP_PRIV(priv) (((u64)(priv)) << 32) | ||
| 379 | #define QI_PGRP_RESP_CODE(res) ((u64)(res)) | ||
| 380 | #define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32) | ||
| 381 | #define QI_PGRP_DID(did) (((u64)(did)) << 16) | ||
| 382 | #define QI_PGRP_PASID_P(p) (((u64)(p)) << 4) | 378 | #define QI_PGRP_PASID_P(p) (((u64)(p)) << 4) |
| 379 | #define QI_PGRP_PDP(p) (((u64)(p)) << 5) | ||
| 380 | #define QI_PGRP_RESP_CODE(res) (((u64)(res)) << 12) | ||
| 381 | #define QI_PGRP_DID(rid) (((u64)(rid)) << 16) | ||
| 382 | #define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32) | ||
| 383 | |||
| 384 | /* Page group response descriptor QW1 */ | ||
| 385 | #define QI_PGRP_LPIG(x) (((u64)(x)) << 2) | ||
| 386 | #define QI_PGRP_IDX(idx) (((u64)(idx)) << 3) | ||
| 383 | 387 | ||
| 384 | #define QI_PSTRM_ADDR(addr) (((u64)(addr)) & VTD_PAGE_MASK) | ||
| 385 | #define QI_PSTRM_DEVFN(devfn) (((u64)(devfn)) << 4) | ||
| 386 | #define QI_PSTRM_RESP_CODE(res) ((u64)(res)) | ||
| 387 | #define QI_PSTRM_IDX(idx) (((u64)(idx)) << 55) | ||
| 388 | #define QI_PSTRM_PRIV(priv) (((u64)(priv)) << 32) | ||
| 389 | #define QI_PSTRM_BUS(bus) (((u64)(bus)) << 24) | ||
| 390 | #define QI_PSTRM_PASID(pasid) (((u64)(pasid)) << 4) | ||
| 391 | 388 | ||
| 392 | #define QI_RESP_SUCCESS 0x0 | 389 | #define QI_RESP_SUCCESS 0x0 |
| 393 | #define QI_RESP_INVALID 0x1 | 390 | #define QI_RESP_INVALID 0x1 |
| @@ -20,7 +20,7 @@ struct device; · 19 unmodified lines | |||
| 20 | 20 | ||
| 21 | struct svm_dev_ops { | 21 | struct svm_dev_ops { |
| 22 | void (*fault_cb)(struct device *dev, int pasid, u64 address, | 22 | void (*fault_cb)(struct device *dev, int pasid, u64 address, |
| 23 | u32 private, int rwxp, int response); | 23 | void *private, int rwxp, int response); |
| 24 | }; | 24 | }; |
| 25 | 25 | ||
| 26 | /* Values for rxwp in fault_cb callback */ | 26 | /* Values for rxwp in fault_cb callback */ |
| @@ -0,0 +1,213 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | #ifndef __IO_PGTABLE_H | ||
| 3 | #define __IO_PGTABLE_H | ||
| 4 | #include <linux/bitops.h> | ||
| 5 | |||
| 6 | /* | ||
| 7 | * Public API for use by IOMMU drivers | ||
| 8 | */ | ||
| 9 | enum io_pgtable_fmt { | ||
| 10 | ARM_32_LPAE_S1, | ||
| 11 | ARM_32_LPAE_S2, | ||
| 12 | ARM_64_LPAE_S1, | ||
| 13 | ARM_64_LPAE_S2, | ||
| 14 | ARM_V7S, | ||
| 15 | IO_PGTABLE_NUM_FMTS, | ||
| 16 | }; | ||
| 17 | |||
| 18 | /** | ||
| 19 | * struct iommu_gather_ops - IOMMU callbacks for TLB and page table management. | ||
| 20 | * | ||
| 21 | * @tlb_flush_all: Synchronously invalidate the entire TLB context. | ||
| 22 | * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. | ||
| 23 | * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and | ||
| 24 | * any corresponding page table updates are visible to the | ||
| 25 | * IOMMU. | ||
| 26 | * | ||
| 27 | * Note that these can all be called in atomic context and must therefore | ||
| 28 | * not block. | ||
| 29 | */ | ||
| 30 | struct iommu_gather_ops { | ||
| 31 | void (*tlb_flush_all)(void *cookie); | ||
| 32 | void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule, | ||
| 33 | bool leaf, void *cookie); | ||
| 34 | void (*tlb_sync)(void *cookie); | ||
| 35 | }; | ||
| 36 | |||
| 37 | /** | ||
| 38 | * struct io_pgtable_cfg - Configuration data for a set of page tables. | ||
| 39 | * | ||
| 40 | * @quirks: A bitmap of hardware quirks that require some special | ||
| 41 | * action by the low-level page table allocator. | ||
| 42 | * @pgsize_bitmap: A bitmap of page sizes supported by this set of page | ||
| 43 | * tables. | ||
| 44 | * @ias: Input address (iova) size, in bits. | ||
| 45 | * @oas: Output address (paddr) size, in bits. | ||
| 46 | * @tlb: TLB management callbacks for this set of tables. | ||
| 47 | * @iommu_dev: The device representing the DMA configuration for the | ||
| 48 | * page table walker. | ||
| 49 | */ | ||
| 50 | struct io_pgtable_cfg { | ||
| 51 | /* | ||
| 52 | * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in | ||
| 53 | * stage 1 PTEs, for hardware which insists on validating them | ||
| 54 | * even in non-secure state where they should normally be ignored. | ||
| 55 | * | ||
| 56 | * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and | ||
| 57 | * IOMMU_NOEXEC flags and map everything with full access, for | ||
| 58 | * hardware which does not implement the permissions of a given | ||
| 59 | * format, and/or requires some format-specific default value. | ||
| 60 | * | ||
| 61 | * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid | ||
| 62 | * (unmapped) entries but the hardware might do so anyway, perform | ||
| 63 | * TLB maintenance when mapping as well as when unmapping. | ||
| 64 | * | ||
| 65 | * IO_PGTABLE_QUIRK_ARM_MTK_4GB: (ARM v7s format) Set bit 9 in all | ||
| 66 | * PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit | ||
| 67 | * when the SoC is in "4GB mode" and they can only access the high | ||
| 68 | * remap of DRAM (0x1_00000000 to 0x1_ffffffff). | ||
| 69 | * | ||
| 70 | * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever | ||
| 71 | * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a | ||
| 72 | * software-emulated IOMMU), such that pagetable updates need not | ||
| 73 | * be treated as explicit DMA data. | ||
| 74 | * | ||
| 75 | * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs | ||
| 76 | * on unmap, for DMA domains using the flush queue mechanism for | ||
| 77 | * delayed invalidation. | ||
| 78 | */ | ||
| 79 | #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) | ||
| 80 | #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) | ||
| 81 | #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) | ||
| 82 | #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3) | ||
| 83 | #define IO_PGTABLE_QUIRK_NO_DMA BIT(4) | ||
| 84 | #define IO_PGTABLE_QUIRK_NON_STRICT BIT(5) | ||
| 85 | unsigned long quirks; | ||
| 86 | unsigned long pgsize_bitmap; | ||
| 87 | unsigned int ias; | ||
| 88 | unsigned int oas; | ||
| 89 | const struct iommu_gather_ops *tlb; | ||
| 90 | struct device *iommu_dev; | ||
| 91 | |||
| 92 | /* Low-level data specific to the table format */ | ||
| 93 | union { | ||
| 94 | struct { | ||
| 95 | u64 ttbr[2]; | ||
| 96 | u64 tcr; | ||
| 97 | u64 mair[2]; | ||
| 98 | } arm_lpae_s1_cfg; | ||
| 99 | |||
| 100 | struct { | ||
| 101 | u64 vttbr; | ||
| 102 | u64 vtcr; | ||
| 103 | } arm_lpae_s2_cfg; | ||
| 104 | |||
| 105 | struct { | ||
| 106 | u32 ttbr[2]; | ||
| 107 | u32 tcr; | ||
| 108 | u32 nmrr; | ||
| 109 | u32 prrr; | ||
| 110 | } arm_v7s_cfg; | ||
| 111 | }; | ||
| 112 | }; | ||
| 113 | |||
| 114 | /** | ||
| 115 | * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers. | ||
| 116 | * | ||
| 117 | * @map: Map a physically contiguous memory region. | ||
| 118 | * @unmap: Unmap a physically contiguous memory region. | ||
| 119 | * @iova_to_phys: Translate iova to physical address. | ||
| 120 | * | ||
| 121 | * These functions map directly onto the iommu_ops member functions with | ||
| 122 | * the same names. | ||
| 123 | */ | ||
| 124 | struct io_pgtable_ops { | ||
| 125 | int (*map)(struct io_pgtable_ops *ops, unsigned long iova, | ||
| 126 | phys_addr_t paddr, size_t size, int prot); | ||
| 127 | size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, | ||
| 128 | size_t size); | ||
| 129 | phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, | ||
| 130 | unsigned long iova); | ||
| 131 | }; | ||
| 132 | |||
| 133 | /** | ||
| 134 | * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. | ||
| 135 | * | ||
| 136 | * @fmt: The page table format. | ||
| 137 | * @cfg: The page table configuration. This will be modified to represent | ||
| 138 | * the configuration actually provided by the allocator (e.g. the | ||
| 139 | * pgsize_bitmap may be restricted). | ||
| 140 | * @cookie: An opaque token provided by the IOMMU driver and passed back to | ||
| 141 | * the callback routines in cfg->tlb. | ||
| 142 | */ | ||
| 143 | struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, | ||
| 144 | struct io_pgtable_cfg *cfg, | ||
| 145 | void *cookie); | ||
| 146 | |||
| 147 | /** | ||
| 148 | * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller | ||
| 149 | * *must* ensure that the page table is no longer | ||
| 150 | * live, but the TLB can be dirty. | ||
| 151 | * | ||
| 152 | * @ops: The ops returned from alloc_io_pgtable_ops. | ||
| 153 | */ | ||
| 154 | void free_io_pgtable_ops(struct io_pgtable_ops *ops); | ||
| 155 | |||
| 156 | |||
| 157 | /* | ||
| 158 | * Internal structures for page table allocator implementations. | ||
| 159 | */ | ||
| 160 | |||
| 161 | /** | ||
| 162 | * struct io_pgtable - Internal structure describing a set of page tables. | ||
| 163 | * | ||
| 164 | * @fmt: The page table format. | ||
| 165 | * @cookie: An opaque token provided by the IOMMU driver and passed back to | ||
| 166 | * any callback routines. | ||
| 167 | * @cfg: A copy of the page table configuration. | ||
| 168 | * @ops: The page table operations in use for this set of page tables. | ||
| 169 | */ | ||
| 170 | struct io_pgtable { | ||
| 171 | enum io_pgtable_fmt fmt; | ||
| 172 | void *cookie; | ||
| 173 | struct io_pgtable_cfg cfg; | ||
| 174 | struct io_pgtable_ops ops; | ||
| 175 | }; | ||
| 176 | |||
| 177 | #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) | ||
| 178 | |||
| 179 | static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) | ||
| 180 | { | ||
| 181 | iop->cfg.tlb->tlb_flush_all(iop->cookie); | ||
| 182 | } | ||
| 183 | |||
| 184 | static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, | ||
| 185 | unsigned long iova, size_t size, size_t granule, bool leaf) | ||
| 186 | { | ||
| 187 | iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); | ||
| 188 | } | ||
| 189 | |||
| 190 | static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) | ||
| 191 | { | ||
| 192 | iop->cfg.tlb->tlb_sync(iop->cookie); | ||
| 193 | } | ||
| 194 | |||
| 195 | /** | ||
| 196 | * struct io_pgtable_init_fns - Alloc/free a set of page tables for a | ||
| 197 | * particular format. | ||
| 198 | * | ||
| 199 | * @alloc: Allocate a set of page tables described by cfg. | ||
| 200 | * @free: Free the page tables associated with iop. | ||
| 201 | */ | ||
| 202 | struct io_pgtable_init_fns { | ||
| 203 | struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); | ||
| 204 | void (*free)(struct io_pgtable *iop); | ||
| 205 | }; | ||
| 206 | |||
| 207 | extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; | ||
| 208 | extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; | ||
| 209 | extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; | ||
| 210 | extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; | ||
| 211 | extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; | ||
| 212 | |||
| 213 | #endif /* __IO_PGTABLE_H */ | ||
| @@ -167,8 +167,9 @@ struct iommu_resv_region { · 166 unmodified lines | |||
| 167 | * @detach_dev: detach device from an iommu domain | 167 | * @detach_dev: detach device from an iommu domain |
| 168 | * @map: map a physically contiguous memory region to an iommu domain | 168 | * @map: map a physically contiguous memory region to an iommu domain |
| 169 | * @unmap: unmap a physically contiguous memory region from an iommu domain | 169 | * @unmap: unmap a physically contiguous memory region from an iommu domain |
| 170 | * @flush_tlb_all: Synchronously flush all hardware TLBs for this domain | 170 | * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain |
| 171 | * @iotlb_range_add: Add a given iova range to the flush queue for this domain | 171 | * @iotlb_range_add: Add a given iova range to the flush queue for this domain |
| 172 | * @iotlb_sync_map: Sync mappings created recently using @map to the hardware | ||
| 172 | * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush | 173 | * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush |
| 173 | * queue | 174 | * queue |
| 174 | * @iova_to_phys: translate iova to physical address | 175 | * @iova_to_phys: translate iova to physical address |
| @@ -183,6 +184,8 @@ struct iommu_resv_region { · 8 unmodified lines | |||
| 183 | * @domain_window_enable: Configure and enable a particular window for a domain | 184 | * @domain_window_enable: Configure and enable a particular window for a domain |
| 184 | * @domain_window_disable: Disable a particular window for a domain | 185 | * @domain_window_disable: Disable a particular window for a domain |
| 185 | * @of_xlate: add OF master IDs to iommu grouping | 186 | * @of_xlate: add OF master IDs to iommu grouping |
| 187 | * @is_attach_deferred: Check if domain attach should be deferred from iommu | ||
| 188 | * driver init to device driver init (default no) | ||
| 186 | * @pgsize_bitmap: bitmap of all possible supported page sizes | 189 | * @pgsize_bitmap: bitmap of all possible supported page sizes |
| 187 | */ | 190 | */ |
| 188 | struct iommu_ops { | 191 | struct iommu_ops { |
| @@ -201,6 +204,7 @@ struct iommu_ops { · 12 unmodified lines | |||
| 201 | void (*flush_iotlb_all)(struct iommu_domain *domain); | 204 | void (*flush_iotlb_all)(struct iommu_domain *domain); |
| 202 | void (*iotlb_range_add)(struct iommu_domain *domain, | 205 | void (*iotlb_range_add)(struct iommu_domain *domain, |
| 203 | unsigned long iova, size_t size); | 206 | unsigned long iova, size_t size); |
| 207 | void (*iotlb_sync_map)(struct iommu_domain *domain); | ||
| 204 | void (*iotlb_sync)(struct iommu_domain *domain); | 208 | void (*iotlb_sync)(struct iommu_domain *domain); |
| 205 | phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova); | 209 | phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova); |
| 206 | int (*add_device)(struct device *dev); | 210 | int (*add_device)(struct device *dev); |
| @@ -40,6 +40,7 @@ void pci_disable_pasid(struct pci_dev *pdev); · 39 unmodified lines | |||
| 40 | void pci_restore_pasid_state(struct pci_dev *pdev); | 40 | void pci_restore_pasid_state(struct pci_dev *pdev); |
| 41 | int pci_pasid_features(struct pci_dev *pdev); | 41 | int pci_pasid_features(struct pci_dev *pdev); |
| 42 | int pci_max_pasids(struct pci_dev *pdev); | 42 | int pci_max_pasids(struct pci_dev *pdev); |
| 43 | int pci_prg_resp_pasid_required(struct pci_dev *pdev); | ||
| 43 | 44 | ||
| 44 | #else /* CONFIG_PCI_PASID */ | 45 | #else /* CONFIG_PCI_PASID */ |
| 45 | 46 | ||
| @@ -66,6 +67,10 @@ static inline int pci_max_pasids(struct pci_dev *pdev) · 20 unmodified lines | |||
| 66 | return -EINVAL; | 67 | return -EINVAL; |
| 67 | } | 68 | } |
| 68 | 69 | ||
| 70 | static inline int pci_prg_resp_pasid_required(struct pci_dev *pdev) | ||
| 71 | { | ||
| 72 | return 0; | ||
| 73 | } | ||
| 69 | #endif /* CONFIG_PCI_PASID */ | 74 | #endif /* CONFIG_PCI_PASID */ |
| 70 | 75 | ||
| 71 | 76 | ||
| @@ -1527,11 +1527,13 @@ void pci_ats_init(struct pci_dev *dev); · 1526 unmodified lines | |||
| 1527 | int pci_enable_ats(struct pci_dev *dev, int ps); | 1527 | int pci_enable_ats(struct pci_dev *dev, int ps); |
| 1528 | void pci_disable_ats(struct pci_dev *dev); | 1528 | void pci_disable_ats(struct pci_dev *dev); |
| 1529 | int pci_ats_queue_depth(struct pci_dev *dev); | 1529 | int pci_ats_queue_depth(struct pci_dev *dev); |
| 1530 | int pci_ats_page_aligned(struct pci_dev *dev); | ||
| 1530 | #else | 1531 | #else |
| 1531 | static inline void pci_ats_init(struct pci_dev *d) { } | 1532 | static inline void pci_ats_init(struct pci_dev *d) { } |
| 1532 | static inline int pci_enable_ats(struct pci_dev *d, int ps) { return -ENODEV; } | 1533 | static inline int pci_enable_ats(struct pci_dev *d, int ps) { return -ENODEV; } |
| 1533 | static inline void pci_disable_ats(struct pci_dev *d) { } | 1534 | static inline void pci_disable_ats(struct pci_dev *d) { } |
| 1534 | static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; } | 1535 | static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; } |
| 1536 | static inline int pci_ats_page_aligned(struct pci_dev *dev) { return 0; } | ||
| 1535 | #endif | 1537 | #endif |
| 1536 | 1538 | ||
| 1537 | #ifdef CONFIG_PCIE_PTM | 1539 | #ifdef CONFIG_PCIE_PTM |