// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. * Author: Joerg Roedel <jroedel@suse.de> * Leo Duran <leo.duran@amd.com> */ #define pr_fmt(fmt) "AMD-Vi: " fmt#define dev_fmt(fmt) pr_fmt(fmt) #include <linux/pci.h>#include <linux/acpi.h>#include <linux/list.h>#include <linux/bitmap.h>#include <linux/syscore_ops.h>#include <linux/interrupt.h>#include <linux/msi.h>#include <linux/irq.h>#include <linux/amd-iommu.h>#include <linux/export.h>#include <linux/kmemleak.h>#include <linux/cc_platform.h>#include <linux/iopoll.h>#include <asm/pci-direct.h>#include <asm/iommu.h>#include <asm/apic.h>#include <asm/gart.h>#include <asm/x86_init.h>#include <asm/io_apic.h>#include <asm/irq_remapping.h>#include <asm/set_memory.h>#include <asm/sev.h> #include <linux/crash_dump.h> #include "amd_iommu.h"#include "../irq_remapping.h"#include "../iommu-pages.h" /* * definitions for the ACPI scanning code */#define IVRS_HEADER_LENGTH 48 #define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40#define ACPI_IVMD_TYPE_ALL 0x20#define ACPI_IVMD_TYPE 0x21#define ACPI_IVMD_TYPE_RANGE 0x22 #define IVHD_DEV_ALL 0x01#define IVHD_DEV_SELECT 0x02#define IVHD_DEV_SELECT_RANGE_START 0x03#define IVHD_DEV_RANGE_END 0x04#define IVHD_DEV_ALIAS 0x42#define IVHD_DEV_ALIAS_RANGE 0x43#define IVHD_DEV_EXT_SELECT 0x46#define IVHD_DEV_EXT_SELECT_RANGE 0x47#define IVHD_DEV_SPECIAL 0x48#define IVHD_DEV_ACPI_HID 0xf0 #define UID_NOT_PRESENT 0#define UID_IS_INTEGER 1#define UID_IS_CHARACTER 2 #define IVHD_SPECIAL_IOAPIC 1#define IVHD_SPECIAL_HPET 2 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01#define IVHD_FLAG_PASSPW_EN_MASK 0x02#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04#define IVHD_FLAG_ISOC_EN_MASK 0x08 #define IVMD_FLAG_EXCL_RANGE 0x08#define IVMD_FLAG_IW 0x04#define IVMD_FLAG_IR 0x02#define IVMD_FLAG_UNITY_MAP 0x01 #define ACPI_DEVFLAG_INITPASS 0x01#define ACPI_DEVFLAG_EXTINT 0x02#define ACPI_DEVFLAG_NMI 0x04#define ACPI_DEVFLAG_SYSMGT1 0x10#define ACPI_DEVFLAG_SYSMGT2 0x20#define ACPI_DEVFLAG_LINT0 0x40#define ACPI_DEVFLAG_LINT1 0x80#define ACPI_DEVFLAG_ATSDIS 0x10000000 #define IVRS_GET_SBDF_ID(seg, bus, dev, fn) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \ | ((dev & 0x1f) << 3) | (fn & 0x7)) /* * ACPI table definitions * * These data structures are laid over the table to parse the important values * out of it. */ /* * structure describing one IOMMU in the ACPI table. Typically followed by one * or more ivhd_entrys. */struct ivhd_header { u8 type; u8 flags; u16 length; u16 devid; u16 cap_ptr; u64 mmio_phys; u16 pci_seg; u16 info; u32 efr_attr; /* Following only valid on IVHD type 11h and 40h */ u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */ u64 efr_reg2;} __attribute__((packed)); /* * A device entry describing which devices a specific IOMMU translates and * which requestor ids they use. */struct ivhd_entry { u8 type; u16 devid; u8 flags; struct_group(ext_hid, u32 ext; u32 hidh; ); u64 cid; u8 uidf; u8 uidl; u8 uid;} __attribute__((packed)); int amd_iommu_evtlog_size = EVTLOG_SIZE_DEF;int amd_iommu_pprlog_size = PPRLOG_SIZE_DEF; /* * An AMD IOMMU memory definition structure. It defines things like exclusion * ranges for devices and regions that should be unity mapped. */struct ivmd_header { u8 type; u8 flags; u16 length; u16 devid; u16 aux; u16 pci_seg; u8 resv[6]; u64 range_start; u64 range_length;} __attribute__((packed)); bool amd_iommu_dump;bool amd_iommu_irq_remap __read_mostly; enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1;/* Virtual address size */u8 amd_iommu_hpt_vasize;/* Guest page table level */int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; static bool amd_iommu_detected;static bool amd_iommu_disabled __initdata;static bool amd_iommu_force_enable __initdata;static bool amd_iommu_irtcachedis;static int amd_iommu_target_ivhd_type; /* Global EFR and EFR2 registers */u64 amd_iommu_efr;u64 amd_iommu_efr2; /* Host (v1) page table is not supported*/bool amd_iommu_hatdis; /* SNP is enabled on the system? */bool amd_iommu_snp_en;EXPORT_SYMBOL(amd_iommu_snp_en); /* SNP page mode 0 support */bool amd_iommu_snp_mode0_sup; LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the system */LIST_HEAD(amd_ivhd_dev_flags_list); /* list of all IVHD device entry settings */ /* Number of IOMMUs present in the system */static int amd_iommus_present; /* IOMMUs have a non-present cache? */bool amd_iommu_np_cache __read_mostly;bool amd_iommu_iotlb_sup __read_mostly = true; static bool amd_iommu_pc_present __read_mostly;bool amdr_ivrs_remap_support __read_mostly; bool amd_iommu_force_isolation __read_mostly; unsigned long amd_iommu_pgsize_bitmap __ro_after_init = AMD_IOMMU_PGSIZES; enum iommu_init_state { IOMMU_START_STATE, IOMMU_IVRS_DETECTED, IOMMU_ACPI_FINISHED, IOMMU_ENABLED, IOMMU_PCI_INIT, IOMMU_INTERRUPTS_EN, IOMMU_INITIALIZED, IOMMU_NOT_FOUND, IOMMU_INIT_ERROR, IOMMU_CMDLINE_DISABLED,}; /* Early ioapic and hpet maps from kernel command line */#define EARLY_MAP_SIZE 4static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE]; static int __initdata early_ioapic_map_size;static int __initdata early_hpet_map_size;static int __initdata early_acpihid_map_size; static bool __initdata cmdline_maps; static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void);static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg); static bool amd_iommu_pre_enabled = true; static u32 amd_iommu_ivinfo __initdata; bool translation_pre_enabled(struct amd_iommu *iommu){ return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);} static void clear_translation_pre_enabled(struct amd_iommu *iommu){ iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;} static void init_translation_status(struct amd_iommu *iommu){ u64 ctrl; ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); if (ctrl & (1<<CONTROL_IOMMU_EN)) iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;} int amd_iommu_get_num_iommus(void){ return amd_iommus_present;} bool amd_iommu_ht_range_ignore(void){ return check_feature2(FEATURE_HT_RANGE_IGNORE);} /* * Iterate through all the IOMMUs to get common EFR * masks among all IOMMUs and warn if found inconsistency. */static __init void get_global_efr(void){ struct amd_iommu *iommu; for_each_iommu(iommu) { u64 tmp = iommu->features; u64 tmp2 = iommu->features2; if (list_is_first(&iommu->list, &amd_iommu_list)) { amd_iommu_efr = tmp; amd_iommu_efr2 = tmp2; continue; } if (amd_iommu_efr == tmp && amd_iommu_efr2 == tmp2) continue; pr_err(FW_BUG "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n", tmp, tmp2, amd_iommu_efr, amd_iommu_efr2, iommu->index, iommu->pci_seg->id, PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid), PCI_FUNC(iommu->devid)); amd_iommu_efr &= tmp; amd_iommu_efr2 &= tmp2; } pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2);} /* * For IVHD type 0x11/0x40, EFR is also available via IVHD. * Default to IVHD EFR since it is available sooner * (i.e. before PCI init). */static void __init early_iommu_features_init(struct amd_iommu *iommu, struct ivhd_header *h){ if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) { iommu->features = h->efr_reg; iommu->features2 = h->efr_reg2; } if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP) amdr_ivrs_remap_support = true;} /* Access to l1 and l2 indexed register spaces */ static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address){ u32 val; pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); pci_read_config_dword(iommu->dev, 0xfc, &val); return val;} static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val){ pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31)); pci_write_config_dword(iommu->dev, 0xfc, val); pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));} static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address){ u32 val; pci_write_config_dword(iommu->dev, 0xf0, address); pci_read_config_dword(iommu->dev, 0xf4, &val); return val;} static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val){ pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8)); pci_write_config_dword(iommu->dev, 0xf4, val);} /**************************************************************************** * * AMD IOMMU MMIO register space handling functions * * These functions are used to program the IOMMU device registers in * MMIO space required for that driver. * ****************************************************************************/ static void iommu_set_cwwb_range(struct amd_iommu *iommu){ u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem); u64 entry = start & PM_ADDR_MASK; if (!check_feature(FEATURE_SNP)) return; /* Note: * Re-purpose Exclusion base/limit registers for Completion wait * write-back base/limit. */ memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, &entry, sizeof(entry)); /* Note: * Default to 4 Kbytes, which can be specified by setting base * address equal to the limit address. */ memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, &entry, sizeof(entry));} /* Programs the physical address of the device table into the IOMMU hardware */static void iommu_set_device_table(struct amd_iommu *iommu){ u64 entry; u32 dev_table_size = iommu->pci_seg->dev_table_size; void *dev_table = (void *)get_dev_table(iommu); BUG_ON(iommu->mmio_base == NULL); if (is_kdump_kernel()) return; entry = iommu_virt_to_phys(dev_table); entry |= (dev_table_size >> 12) - 1; memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, &entry, sizeof(entry));} static void iommu_feature_set(struct amd_iommu *iommu, u64 val, u64 mask, u8 shift){ u64 ctrl; ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); mask <<= shift; ctrl &= ~mask; ctrl |= (val << shift) & mask; writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);} /* Generic functions to enable/disable certain features of the IOMMU. */void iommu_feature_enable(struct amd_iommu *iommu, u8 bit){ iommu_feature_set(iommu, 1ULL, 1ULL, bit);} static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit){ iommu_feature_set(iommu, 0ULL, 1ULL, bit);} /* Function to enable the hardware */static void iommu_enable(struct amd_iommu *iommu){ iommu_feature_enable(iommu, CONTROL_IOMMU_EN);} static void iommu_disable(struct amd_iommu *iommu){ if (!iommu->mmio_base) return; /* Disable command buffer */ iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); /* Disable event logging and event interrupts */ iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); /* Disable IOMMU GA_LOG */ iommu_feature_disable(iommu, CONTROL_GALOG_EN); iommu_feature_disable(iommu, CONTROL_GAINT_EN); /* Disable IOMMU PPR logging */ iommu_feature_disable(iommu, CONTROL_PPRLOG_EN); iommu_feature_disable(iommu, CONTROL_PPRINT_EN); /* Disable IOMMU hardware itself */ iommu_feature_disable(iommu, CONTROL_IOMMU_EN); /* Clear IRTE cache disabling bit */ iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);} /* * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in * the system has one. */static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end){ if (!request_mem_region(address, end, "amd_iommu")) { pr_err("Can not reserve memory region %llx-%llx for mmio\n", address, end); pr_err("This is a BIOS bug. Please contact your hardware vendor\n"); return NULL; } return (u8 __iomem *)ioremap(address, end);} static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu){ if (iommu->mmio_base) iounmap(iommu->mmio_base); release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);} static inline u32 get_ivhd_header_size(struct ivhd_header *h){ u32 size = 0; switch (h->type) { case 0x10: size = 24; break; case 0x11: case 0x40: size = 40; break; } return size;} /**************************************************************************** * * The functions below belong to the first pass of AMD IOMMU ACPI table * parsing. In this pass we try to find out the highest device id this * code has to handle. Upon this information the size of the shared dataLines 1–500 of 4157. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. * Author: Joerg Roedel <jroedel@suse.de> * Leo Duran <leo.duran@amd.com> */ #define pr_fmt(fmt) "AMD-Vi: " fmt #define dev_fmt(fmt) pr_fmt(fmt) #include <linux/pci.h> #include <linux/acpi.h> #include <linux/list.h> #include <linux/bitmap.h> #include <linux/syscore_ops.h> #include <linux/interrupt.h> #include <linux/msi.h> #include <linux/irq.h> #include <linux/amd-iommu.h> #include <linux/export.h> #include <linux/kmemleak.h> #include <linux/cc_platform.h> #include <linux/iopoll.h> #include <asm/pci-direct.h> #include <asm/iommu.h> #include <asm/apic.h> #include <asm/gart.h> #include <asm/x86_init.h> #include <asm/io_apic.h> #include <asm/irq_remapping.h> #include <asm/set_memory.h> #include <asm/sev.h> #include <linux/crash_dump.h> #include "amd_iommu.h" #include "../irq_remapping.h" #include "../iommu-pages.h" /* * definitions for the ACPI scanning code */ #define IVRS_HEADER_LENGTH 48 #define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40 #define ACPI_IVMD_TYPE_ALL 0x20 #define ACPI_IVMD_TYPE 0x21 #define ACPI_IVMD_TYPE_RANGE 0x22 #define IVHD_DEV_ALL 0x01 #define IVHD_DEV_SELECT 0x02 #define IVHD_DEV_SELECT_RANGE_START 0x03 #define IVHD_DEV_RANGE_END 0x04 #define IVHD_DEV_ALIAS 0x42 #define IVHD_DEV_ALIAS_RANGE 0x43 #define IVHD_DEV_EXT_SELECT 0x46 #define IVHD_DEV_EXT_SELECT_RANGE 0x47 #define IVHD_DEV_SPECIAL 0x48 #define IVHD_DEV_ACPI_HID 0xf0 #define UID_NOT_PRESENT 0 #define UID_IS_INTEGER 1 #define UID_IS_CHARACTER 2 #define IVHD_SPECIAL_IOAPIC 1 #define IVHD_SPECIAL_HPET 2 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01 #define IVHD_FLAG_PASSPW_EN_MASK 0x02 #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04 #define IVHD_FLAG_ISOC_EN_MASK 0x08 #define IVMD_FLAG_EXCL_RANGE 0x08 #define IVMD_FLAG_IW 0x04 #define IVMD_FLAG_IR 0x02 #define IVMD_FLAG_UNITY_MAP 0x01 #define ACPI_DEVFLAG_INITPASS 0x01 #define ACPI_DEVFLAG_EXTINT 0x02 #define ACPI_DEVFLAG_NMI 0x04 #define ACPI_DEVFLAG_SYSMGT1 0x10 #define ACPI_DEVFLAG_SYSMGT2 0x20 #define ACPI_DEVFLAG_LINT0 0x40 #define ACPI_DEVFLAG_LINT1 0x80 #define ACPI_DEVFLAG_ATSDIS 0x10000000 #define IVRS_GET_SBDF_ID(seg, bus, dev, fn) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \ | ((dev & 0x1f) << 3) | (fn & 0x7)) /* * ACPI table definitions * * These data structures are laid over the table to parse the important values * out of it. */ /* * structure describing one IOMMU in the ACPI table. Typically followed by one * or more ivhd_entrys. */ struct ivhd_header { u8 type; u8 flags; u16 length; u16 devid; u16 cap_ptr; u64 mmio_phys; u16 pci_seg; u16 info; u32 efr_attr; /* Following only valid on IVHD type 11h and 40h */ u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */ u64 efr_reg2; } __attribute__((packed)); /* * A device entry describing which devices a specific IOMMU translates and * which requestor ids they use. */ struct ivhd_entry { u8 type; u16 devid; u8 flags; struct_group(ext_hid, u32 ext; u32 hidh; ); u64 cid; u8 uidf; u8 uidl; u8 uid; } __attribute__((packed)); int amd_iommu_evtlog_size = EVTLOG_SIZE_DEF; int amd_iommu_pprlog_size = PPRLOG_SIZE_DEF; /* * An AMD IOMMU memory definition structure. It defines things like exclusion * ranges for devices and regions that should be unity mapped. */ struct ivmd_header { u8 type; u8 flags; u16 length; u16 devid; u16 aux; u16 pci_seg; u8 resv[6]; u64 range_start; u64 range_length; } __attribute__((packed)); bool amd_iommu_dump; bool amd_iommu_irq_remap __read_mostly; enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1; /* Virtual address size */ u8 amd_iommu_hpt_vasize; /* Guest page table level */ int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; static bool amd_iommu_detected; static bool amd_iommu_disabled __initdata; static bool amd_iommu_force_enable __initdata; static bool amd_iommu_irtcachedis; static int amd_iommu_target_ivhd_type; /* Global EFR and EFR2 registers */ u64 amd_iommu_efr; u64 amd_iommu_efr2; /* Host (v1) page table is not supported*/ bool amd_iommu_hatdis; /* SNP is enabled on the system? */ bool amd_iommu_snp_en; EXPORT_SYMBOL(amd_iommu_snp_en); /* SNP page mode 0 support */ bool amd_iommu_snp_mode0_sup; LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */ LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the system */ LIST_HEAD(amd_ivhd_dev_flags_list); /* list of all IVHD device entry settings */ /* Number of IOMMUs present in the system */ static int amd_iommus_present; /* IOMMUs have a non-present cache? */ bool amd_iommu_np_cache __read_mostly; bool amd_iommu_iotlb_sup __read_mostly = true; static bool amd_iommu_pc_present __read_mostly; bool amdr_ivrs_remap_support __read_mostly; bool amd_iommu_force_isolation __read_mostly; unsigned long amd_iommu_pgsize_bitmap __ro_after_init = AMD_IOMMU_PGSIZES; enum iommu_init_state { IOMMU_START_STATE, IOMMU_IVRS_DETECTED, IOMMU_ACPI_FINISHED, IOMMU_ENABLED, IOMMU_PCI_INIT, IOMMU_INTERRUPTS_EN, IOMMU_INITIALIZED, IOMMU_NOT_FOUND, IOMMU_INIT_ERROR, IOMMU_CMDLINE_DISABLED, }; /* Early ioapic and hpet maps from kernel command line */ #define EARLY_MAP_SIZE 4 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE]; static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE]; static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE]; static int __initdata early_ioapic_map_size; static int __initdata early_hpet_map_size; static int __initdata early_acpihid_map_size; static bool __initdata cmdline_maps; static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg); static bool amd_iommu_pre_enabled = true; static u32 amd_iommu_ivinfo __initdata; bool translation_pre_enabled(struct amd_iommu *iommu) { return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED); } static void clear_translation_pre_enabled(struct amd_iommu *iommu) { iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; } static void init_translation_status(struct amd_iommu *iommu) { u64 ctrl; ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); if (ctrl & (1<<CONTROL_IOMMU_EN)) iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; } int amd_iommu_get_num_iommus(void) { return amd_iommus_present; } bool amd_iommu_ht_range_ignore(void) { return check_feature2(FEATURE_HT_RANGE_IGNORE); } /* * Iterate through all the IOMMUs to get common EFR * masks among all IOMMUs and warn if found inconsistency. */ static __init void get_global_efr(void) { struct amd_iommu *iommu; for_each_iommu(iommu) { u64 tmp = iommu->features; u64 tmp2 = iommu->features2; if (list_is_first(&iommu->list, &amd_iommu_list)) { amd_iommu_efr = tmp; amd_iommu_efr2 = tmp2; continue; } if (amd_iommu_efr == tmp && amd_iommu_efr2 == tmp2) continue; pr_err(FW_BUG "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n", tmp, tmp2, amd_iommu_efr, amd_iommu_efr2, iommu->index, iommu->pci_seg->id, PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid), PCI_FUNC(iommu->devid)); amd_iommu_efr &= tmp; amd_iommu_efr2 &= tmp2; } pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2); } /* * For IVHD type 0x11/0x40, EFR is also available via IVHD. * Default to IVHD EFR since it is available sooner * (i.e. before PCI init). */ static void __init early_iommu_features_init(struct amd_iommu *iommu, struct ivhd_header *h) { if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) { iommu->features = h->efr_reg; iommu->features2 = h->efr_reg2; } if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP) amdr_ivrs_remap_support = true; } /* Access to l1 and l2 indexed register spaces */ static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address) { u32 val; pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); pci_read_config_dword(iommu->dev, 0xfc, &val); return val; } static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val) { pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31)); pci_write_config_dword(iommu->dev, 0xfc, val); pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); } static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address) { u32 val; pci_write_config_dword(iommu->dev, 0xf0, address); pci_read_config_dword(iommu->dev, 0xf4, &val); return val; } static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val) { pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8)); pci_write_config_dword(iommu->dev, 0xf4, val); } /**************************************************************************** * * AMD IOMMU MMIO register space handling functions * * These functions are used to program the IOMMU device registers in * MMIO space required for that driver. * ****************************************************************************/ static void iommu_set_cwwb_range(struct amd_iommu *iommu) { u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem); u64 entry = start & PM_ADDR_MASK; if (!check_feature(FEATURE_SNP)) return; /* Note: * Re-purpose Exclusion base/limit registers for Completion wait * write-back base/limit. */ memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, &entry, sizeof(entry)); /* Note: * Default to 4 Kbytes, which can be specified by setting base * address equal to the limit address. */ memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, &entry, sizeof(entry)); } /* Programs the physical address of the device table into the IOMMU hardware */ static void iommu_set_device_table(struct amd_iommu *iommu) { u64 entry; u32 dev_table_size = iommu->pci_seg->dev_table_size; void *dev_table = (void *)get_dev_table(iommu); BUG_ON(iommu->mmio_base == NULL); if (is_kdump_kernel()) return; entry = iommu_virt_to_phys(dev_table); entry |= (dev_table_size >> 12) - 1; memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, &entry, sizeof(entry)); } static void iommu_feature_set(struct amd_iommu *iommu, u64 val, u64 mask, u8 shift) { u64 ctrl; ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); mask <<= shift; ctrl &= ~mask; ctrl |= (val << shift) & mask; writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); } /* Generic functions to enable/disable certain features of the IOMMU. */ void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) { iommu_feature_set(iommu, 1ULL, 1ULL, bit); } static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) { iommu_feature_set(iommu, 0ULL, 1ULL, bit); } /* Function to enable the hardware */ static void iommu_enable(struct amd_iommu *iommu) { iommu_feature_enable(iommu, CONTROL_IOMMU_EN); } static void iommu_disable(struct amd_iommu *iommu) { if (!iommu->mmio_base) return; /* Disable command buffer */ iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); /* Disable event logging and event interrupts */ iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); /* Disable IOMMU GA_LOG */ iommu_feature_disable(iommu, CONTROL_GALOG_EN); iommu_feature_disable(iommu, CONTROL_GAINT_EN); /* Disable IOMMU PPR logging */ iommu_feature_disable(iommu, CONTROL_PPRLOG_EN); iommu_feature_disable(iommu, CONTROL_PPRINT_EN); /* Disable IOMMU hardware itself */ iommu_feature_disable(iommu, CONTROL_IOMMU_EN); /* Clear IRTE cache disabling bit */ iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS); } /* * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in * the system has one. */ static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end) { if (!request_mem_region(address, end, "amd_iommu")) { pr_err("Can not reserve memory region %llx-%llx for mmio\n", address, end); pr_err("This is a BIOS bug. Please contact your hardware vendor\n"); return NULL; } return (u8 __iomem *)ioremap(address, end); } static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) { if (iommu->mmio_base) iounmap(iommu->mmio_base); release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end); } static inline u32 get_ivhd_header_size(struct ivhd_header *h) { u32 size = 0; switch (h->type) { case 0x10: size = 24; break; case 0x11: case 0x40: size = 40; break; } return size; } /**************************************************************************** * * The functions below belong to the first pass of AMD IOMMU ACPI table * parsing. In this pass we try to find out the highest device id this * code has to handle. Upon this information the size of the shared data