lael-labs/linux

Documentation/devicetree/bindings/iommu/iommu.txt+6−0

 disable the IOMMU's device tree node in the first place because it would
 prevent any driver from properly setting up the translations.
+Optional properties:
+--------------------
+- pasid-num-bits: Some masters support multiple address spaces for DMA, by
+  tagging DMA transactions with an address space identifier. By default,
+  this is 0, which means that the device only has one address space.
 Notes:
 ======

drivers/acpi/arm64/iort.c+20−2

 #define pr_fmt(fmt)	"ACPI: IORT: " fmt
 #include <linux/acpi_iort.h>
+#include <linux/bitfield.h>
 #include <linux/iommu.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 {
 	switch (type) {
 	case ACPI_IORT_NODE_SMMU_V3:
-		return IS_BUILTIN(CONFIG_ARM_SMMU_V3);
+		return IS_ENABLED(CONFIG_ARM_SMMU_V3);
 	case ACPI_IORT_NODE_SMMU:
-		return IS_BUILTIN(CONFIG_ARM_SMMU);
+		return IS_ENABLED(CONFIG_ARM_SMMU);
 	default:
 		pr_warn("IORT node type %u does not describe an SMMU\n", type);
 		return false;
 	return iort_iommu_xlate(info->dev, parent, streamid);
 }
+static void iort_named_component_init(struct device *dev,
+				      struct acpi_iort_node *node)
+{
+	struct acpi_iort_named_component *nc;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	if (!fwspec)
+		return;
+	nc = (struct acpi_iort_named_component *)node->node_data;
+	fwspec->num_pasid_bits = FIELD_GET(ACPI_IORT_NC_PASID_BITS,
+					   nc->node_flags);
+}
 /**
  * iort_iommu_configure - Set-up IOMMU configuration for a device.
  *
 			if (parent)
 				err = iort_iommu_xlate(dev, parent, streamid);
 		} while (parent && !err);
+		if (!err)
+			iort_named_component_init(dev, node);
 	}
 	/*

drivers/iommu/Kconfig+30−5

 config IOMMU_DEFAULT_PASSTHROUGH
 	bool "IOMMU passthrough by default"
 	depends on IOMMU_API
-        help
+	help
 	  Enable passthrough by default, removing the need to pass in
 	  iommu.passthrough=on or iommu=pt through command line. If this
 	  is enabled, you can still disable with iommu.passthrough=off
 	  If unsure, say N here.
 config OF_IOMMU
-       def_bool y
+	def_bool y
-       depends on OF && IOMMU_API
+	depends on OF && IOMMU_API
 # IOMMU-agnostic DMA-mapping layer
 config IOMMU_DMA
 	select PCI_PASID
 	select PCI_PRI
 	select MMU_NOTIFIER
+	select IOASID
 	help
 	  Shared Virtual Memory (SVM) provides a facility for devices
 	  to access DMA resources through process address space by
 	  workaround will setup a 1:1 mapping for the first
 	  16MiB to make floppy (an ISA device) work.
+config INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON
+	bool "Enable Intel IOMMU scalable mode by default"
+	depends on INTEL_IOMMU
+	help
+	  Selecting this option will enable by default the scalable mode if
+	  hardware presents the capability. The scalable mode is defined in
+	  VT-d 3.0. The scalable mode capability could be checked by reading
+	  /sys/devices/virtual/iommu/dmar*/intel-iommu/ecap. If this option
+	  is not selected, scalable mode support could also be enabled by
+	  passing intel_iommu=sm_on to the kernel. If not sure, please use
+	  the default value.
 config IRQ_REMAP
 	bool "Support for Interrupt Remapping"
 	depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI
 # ARM IOMMU support
 config ARM_SMMU
-	bool "ARM Ltd. System MMU (SMMU) Support"
+	tristate "ARM Ltd. System MMU (SMMU) Support"
 	depends on (ARM64 || ARM) && MMU
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_LPAE
 	  Say Y here if your SoC includes an IOMMU device implementing
 	  the ARM SMMU architecture.
+config ARM_SMMU_LEGACY_DT_BINDINGS
+	bool "Support the legacy \"mmu-masters\" devicetree bindings"
+	depends on ARM_SMMU=y && OF
+	help
+	  Support for the badly designed and deprecated "mmu-masters"
+	  devicetree bindings. This allows some DMA masters to attach
+	  to the SMMU but does not provide any support via the DMA API.
+	  If you're lucky, you might be able to get VFIO up and running.
+	  If you say Y here then you'll make me very sad. Instead, say N
+	  and move your firmware to the utopian future that was 2016.
 config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT
 	bool "Default to disabling bypass on ARM SMMU v1 and v2"
 	depends on ARM_SMMU
 	  config.
 config ARM_SMMU_V3
-	bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
+	tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
 	depends on ARM64
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_LPAE

drivers/iommu/Makefile+2−1

 obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o amd_iommu_quirks.o
 obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o
 obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
-obj-$(CONFIG_ARM_SMMU) += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o
+obj-$(CONFIG_ARM_SMMU) += arm-smmu-mod.o
+arm-smmu-mod-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o
 obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o
 obj-$(CONFIG_DMAR_TABLE) += dmar.o
 obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o

drivers/iommu/amd_iommu.c+1−11

 int __init amd_iommu_init_dma_ops(void)
 {
 	swiotlb        = (iommu_default_passthrough() || sme_me_mask) ? 1 : 0;
-	iommu_detected = 1;
 	if (amd_iommu_unmap_flush)
 		pr_info("IO/TLB flush on unmap enabled\n");
 	list_add_tail(&region->list, head);
 }
-static void amd_iommu_put_resv_regions(struct device *dev,
-				     struct list_head *head)
-{
-	struct iommu_resv_region *entry, *next;
-	list_for_each_entry_safe(entry, next, head, list)
-		kfree(entry);
-}
 static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
 					 struct device *dev)
 {
 	.device_group = amd_iommu_device_group,
 	.domain_get_attr = amd_iommu_domain_get_attr,
 	.get_resv_regions = amd_iommu_get_resv_regions,
-	.put_resv_regions = amd_iommu_put_resv_regions,
+	.put_resv_regions = generic_iommu_put_resv_regions,
 	.is_attach_deferred = amd_iommu_is_attach_deferred,
 	.pgsize_bitmap	= AMD_IOMMU_PGSIZES,
 	.flush_iotlb_all = amd_iommu_flush_iotlb_all,

drivers/iommu/amd_iommu_init.c+25−30

 #define IVHD_FLAG_ISOC_EN_MASK          0x08
 #define IVMD_FLAG_EXCL_RANGE            0x08
+#define IVMD_FLAG_IW                    0x04
+#define IVMD_FLAG_IR                    0x02
 #define IVMD_FLAG_UNITY_MAP             0x01
 #define ACPI_DEVFLAG_INITPASS           0x01
 bool amd_iommu_irq_remap __read_mostly;
 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
-static int amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
+static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
 static bool amd_iommu_detected;
 static bool __initdata amd_iommu_disabled;
 	writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
 	writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
-	iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);
+	iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);
 	iommu_feature_enable(iommu, CONTROL_PPR_EN);
 }
  */
 static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
 {
-	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
 	if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
 		return;
-	if (iommu) {
+	/*
-		/*
+	 * Treat per-device exclusion ranges as r/w unity-mapped regions
-		 * We only can configure exclusion ranges per IOMMU, not
+	 * since some buggy BIOSes might lead to the overwritten exclusion
-		 * per device. But we can enable the exclusion range per
+	 * range (exclusion_start and exclusion_length members). This
-		 * device. This is done here
+	 * happens when there are multiple exclusion ranges (IVMD entries)
-		 */
+	 * defined in ACPI table.
-		set_dev_entry_bit(devid, DEV_ENTRY_EX);
+	 */
-		iommu->exclusion_start = m->range_start;
+	m->flags = (IVMD_FLAG_IW | IVMD_FLAG_IR | IVMD_FLAG_UNITY_MAP);
-		iommu->exclusion_length = m->range_length;
-	}
 }
 /*
 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
 		if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
-		if (((h->efr_attr & (0x1 << IOMMU_FEAT_XTSUP_SHIFT)) == 0))
-			amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
 		break;
 	case 0x11:
 	case 0x40:
 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
 		if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0))
 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
-		if (((h->efr_reg & (0x1 << IOMMU_EFR_XTSUP_SHIFT)) == 0))
+		/*
-			amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
+		 * Note: Since iommu_update_intcapxt() leverages
+		 * the IOMMU MMIO access to MSI capability block registers
+		 * for MSI address lo/hi/data, we need to check both
+		 * EFR[XtSup] and EFR[MsiCapMmioSup] for x2APIC support.
+		 */
+		if ((h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) &&
+		    (h->efr_reg & BIT(IOMMU_EFR_MSICAPMMIOSUP_SHIFT)))
+			amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
 		break;
 	default:
 		return -EINVAL;
 static int __init iommu_init_pci(struct amd_iommu *iommu)
 {
 	int cap_ptr = iommu->cap_ptr;
-	u32 range, misc, low, high;
 	int ret;
 	iommu->dev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(iommu->devid),
 	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
 			      &iommu->cap);
-	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,
-			      &range);
-	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,
-			      &misc);
 	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
 		amd_iommu_iotlb_sup = false;
 	/* read extended feature bits */
-	low  = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
+	iommu->features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);
-	high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
-	iommu->features = ((u64)high << 32) | low;
 	if (iommu_feature(iommu, FEATURE_GT)) {
 		int glxval;
 	struct irq_affinity_notify *notify = &iommu->intcapxt_notify;
 	/**
-	 * IntCapXT requires XTSup=1, which can be inferred
+	 * IntCapXT requires XTSup=1 and MsiCapMmioSup=1,
-	 * amd_iommu_xt_mode.
+	 * which can be inferred from amd_iommu_xt_mode.
 	 */
 	if (amd_iommu_xt_mode != IRQ_REMAP_X2APIC_MODE)
 		return 0;
 	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
 	if (iommu->ppr_log != NULL)
-		iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
+		iommu_feature_enable(iommu, CONTROL_PPRINT_EN);
 	iommu_ga_log_enable(iommu);

drivers/iommu/amd_iommu_types.h+3−4

 #define CONTROL_COHERENT_EN     0x0aULL
 #define CONTROL_ISOC_EN         0x0bULL
 #define CONTROL_CMDBUF_EN       0x0cULL
-#define CONTROL_PPFLOG_EN       0x0dULL
+#define CONTROL_PPRLOG_EN       0x0dULL
-#define CONTROL_PPFINT_EN       0x0eULL
+#define CONTROL_PPRINT_EN       0x0eULL
 #define CONTROL_PPR_EN          0x0fULL
 #define CONTROL_GT_EN           0x10ULL
 #define CONTROL_GA_EN           0x11ULL
 #define IOMMU_CAP_EFR     27
 /* IOMMU Feature Reporting Field (for IVHD type 10h */
-#define IOMMU_FEAT_XTSUP_SHIFT	0
 #define IOMMU_FEAT_GASUP_SHIFT	6
 /* IOMMU Extended Feature Register (EFR) */
 #define IOMMU_EFR_XTSUP_SHIFT	2
 #define IOMMU_EFR_GASUP_SHIFT	7
+#define IOMMU_EFR_MSICAPMMIOSUP_SHIFT	46
 #define MAX_DOMAIN_ID 65536
  * independent of their use.
  */
 struct protection_domain {
-	struct list_head list;  /* for list of all protection domains */
 	struct list_head dev_list; /* List of all devices in this domain */
 	struct iommu_domain domain; /* generic domain handle used by
 				       iommu core code */

drivers/iommu/arm-smmu-impl.c+1−1

@@ -119,7 +119,7 @@ int arm_mmu500_reset(struct arm_smmu_device *smmu) · 118 unmodified lines
119	* Secure has also cleared SACR.CACHE_LOCK for this to take effect...	119	* Secure has also cleared SACR.CACHE_LOCK for this to take effect...
120	*/	120	*/
121	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);	121	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
122	major = FIELD_GET(ID7_MAJOR, reg);	122	major = FIELD_GET(ARM_SMMU_ID7_MAJOR, reg);
123	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);	123	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
124	if (major >= 2)	124	if (major >= 2)
125	reg &= ~ARM_MMU500_ACR_CACHE_LOCK;	125	reg &= ~ARM_MMU500_ACR_CACHE_LOCK;

drivers/iommu/arm-smmu-v3.c+450−150

@@ -21,8 +21,7 @@ · 20 unmodified lines
21	#include <linux/io-pgtable.h>	21	#include <linux/io-pgtable.h>
22	#include <linux/iommu.h>	22	#include <linux/iommu.h>
23	#include <linux/iopoll.h>	23	#include <linux/iopoll.h>
24	#include <linux/init.h>	24	#include <linux/module.h>
25	#include <linux/moduleparam.h>
26	#include <linux/msi.h>	25	#include <linux/msi.h>
27	#include <linux/of.h>	26	#include <linux/of.h>
28	#include <linux/of_address.h>	27	#include <linux/of_address.h>
@@ -224,9 +223,15 @@ · 195 unmodified lines
224		223
225	#define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)	224	#define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
226	#define STRTAB_STE_0_S1FMT_LINEAR 0	225	#define STRTAB_STE_0_S1FMT_LINEAR 0
		226	#define STRTAB_STE_0_S1FMT_64K_L2 2
227	#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)	227	#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
228	#define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)	228	#define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
229		229
		230	#define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
		231	#define STRTAB_STE_1_S1DSS_TERMINATE 0x0
		232	#define STRTAB_STE_1_S1DSS_BYPASS 0x1
		233	#define STRTAB_STE_1_S1DSS_SSID0 0x2
		234
230	#define STRTAB_STE_1_S1C_CACHE_NC 0UL	235	#define STRTAB_STE_1_S1C_CACHE_NC 0UL
231	#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL	236	#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
232	#define STRTAB_STE_1_S1C_CACHE_WT 2UL	237	#define STRTAB_STE_1_S1C_CACHE_WT 2UL
@@ -251,6 +256,13 @@ · 18 unmodified lines
251		256
252	#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)	257	#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
253	#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)	258	#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
		259	#define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
		260	#define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
		261	#define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
		262	#define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
		263	#define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
		264	#define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
		265	#define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
254	#define STRTAB_STE_2_S2AA64 (1UL << 51)	266	#define STRTAB_STE_2_S2AA64 (1UL << 51)
255	#define STRTAB_STE_2_S2ENDI (1UL << 52)	267	#define STRTAB_STE_2_S2ENDI (1UL << 52)
256	#define STRTAB_STE_2_S2PTW (1UL << 54)	268	#define STRTAB_STE_2_S2PTW (1UL << 54)
@@ -258,30 +270,34 @@ · 1 unmodified line
258		270
259	#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)	271	#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
260		272
261	/* Context descriptor (stage-1 only) */	273	/*
		274	* Context descriptors.
		275	*
		276	* Linear: when less than 1024 SSIDs are supported
		277	* 2lvl: at most 1024 L1 entries,
		278	* 1024 lazy entries per table.
		279	*/
		280	#define CTXDESC_SPLIT 10
		281	#define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
		282
		283	#define CTXDESC_L1_DESC_DWORDS 1
		284	#define CTXDESC_L1_DESC_V (1UL << 0)
		285	#define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
		286
262	#define CTXDESC_CD_DWORDS 8	287	#define CTXDESC_CD_DWORDS 8
263	#define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)	288	#define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
264	#define ARM64_TCR_T0SZ GENMASK_ULL(5, 0)
265	#define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)	289	#define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
266	#define ARM64_TCR_TG0 GENMASK_ULL(15, 14)
267	#define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)	290	#define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
268	#define ARM64_TCR_IRGN0 GENMASK_ULL(9, 8)
269	#define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)	291	#define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
270	#define ARM64_TCR_ORGN0 GENMASK_ULL(11, 10)
271	#define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)	292	#define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
272	#define ARM64_TCR_SH0 GENMASK_ULL(13, 12)
273	#define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)	293	#define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
274	#define ARM64_TCR_EPD0 (1ULL << 7)
275	#define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)	294	#define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
276	#define ARM64_TCR_EPD1 (1ULL << 23)
277		295
278	#define CTXDESC_CD_0_ENDI (1UL << 15)	296	#define CTXDESC_CD_0_ENDI (1UL << 15)
279	#define CTXDESC_CD_0_V (1UL << 31)	297	#define CTXDESC_CD_0_V (1UL << 31)
280		298
281	#define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)	299	#define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
282	#define ARM64_TCR_IPS GENMASK_ULL(34, 32)
283	#define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)	300	#define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
284	#define ARM64_TCR_TBI0 (1ULL << 37)
285		301
286	#define CTXDESC_CD_0_AA64 (1UL << 41)	302	#define CTXDESC_CD_0_AA64 (1UL << 41)
287	#define CTXDESC_CD_0_S (1UL << 44)	303	#define CTXDESC_CD_0_S (1UL << 44)
@@ -292,9 +308,11 @@ · 4 unmodified lines
292		308
293	#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)	309	#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
294		310
295	/* Convert between AArch64 (CPU) TCR format and SMMU CD format */	311	/*
296	#define ARM_SMMU_TCR2CD(tcr, fld) FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \	312	* When the SMMU only supports linear context descriptor tables, pick a
297	FIELD_GET(ARM64_TCR_##fld, tcr))	313	* reasonable size limit (64kB).
		314	*/
		315	#define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
298		316
299	/* Command queue */	317	/* Command queue */
300	#define CMDQ_ENT_SZ_SHIFT 4	318	#define CMDQ_ENT_SZ_SHIFT 4
@@ -323,6 +341,7 @@ · 22 unmodified lines
323	#define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)	341	#define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
324	#define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)	342	#define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
325		343
		344	#define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
326	#define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)	345	#define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
327	#define CMDQ_CFGI_1_LEAF (1UL << 0)	346	#define CMDQ_CFGI_1_LEAF (1UL << 0)
328	#define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)	347	#define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
@@ -384,10 +403,6 @@ · 55 unmodified lines
384	#define MSI_IOVA_BASE 0x8000000	403	#define MSI_IOVA_BASE 0x8000000
385	#define MSI_IOVA_LENGTH 0x100000	404	#define MSI_IOVA_LENGTH 0x100000
386		405
387	/*
388	* not really modular, but the easiest way to keep compat with existing
389	* bootargs behaviour is to continue using module_param_named here.
390	*/
391	static bool disable_bypass = 1;	406	static bool disable_bypass = 1;
392	module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);	407	module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
393	MODULE_PARM_DESC(disable_bypass,	408	MODULE_PARM_DESC(disable_bypass,
@@ -440,8 +455,11 @@ struct arm_smmu_cmdq_ent { · 46 unmodified lines
440		455
441	#define CMDQ_OP_CFGI_STE 0x3	456	#define CMDQ_OP_CFGI_STE 0x3
442	#define CMDQ_OP_CFGI_ALL 0x4	457	#define CMDQ_OP_CFGI_ALL 0x4
		458	#define CMDQ_OP_CFGI_CD 0x5
		459	#define CMDQ_OP_CFGI_CD_ALL 0x6
443	struct {	460	struct {
444	u32 sid;	461	u32 sid;
		462	u32 ssid;
445	union {	463	union {
446	bool leaf;	464	bool leaf;
447	u8 span;	465	u8 span;
@@ -547,16 +565,30 @@ struct arm_smmu_strtab_l1_desc { · 99 unmodified lines
547	dma_addr_t l2ptr_dma;	565	dma_addr_t l2ptr_dma;
548	};	566	};
549		567
		568	struct arm_smmu_ctx_desc {
		569	u16 asid;
		570	u64 ttbr;
		571	u64 tcr;
		572	u64 mair;
		573	};
		574
		575	struct arm_smmu_l1_ctx_desc {
		576	__le64 *l2ptr;
		577	dma_addr_t l2ptr_dma;
		578	};
		579
		580	struct arm_smmu_ctx_desc_cfg {
		581	__le64 *cdtab;
		582	dma_addr_t cdtab_dma;
		583	struct arm_smmu_l1_ctx_desc *l1_desc;
		584	unsigned int num_l1_ents;
		585	};
		586
550	struct arm_smmu_s1_cfg {	587	struct arm_smmu_s1_cfg {
551	__le64 *cdptr;	588	struct arm_smmu_ctx_desc_cfg cdcfg;
552	dma_addr_t cdptr_dma;	589	struct arm_smmu_ctx_desc cd;
553		590	u8 s1fmt;
554	struct arm_smmu_ctx_desc {	591	u8 s1cdmax;
555	u16 asid;
556	u64 ttbr;
557	u64 tcr;
558	u64 mair;
559	} cd;
560	};	592	};
561		593
562	struct arm_smmu_s2_cfg {	594	struct arm_smmu_s2_cfg {
@@ -638,6 +670,7 @@ struct arm_smmu_master { · 75 unmodified lines
638	u32 *sids;	670	u32 *sids;
639	unsigned int num_sids;	671	unsigned int num_sids;
640	bool ats_enabled;	672	bool ats_enabled;
		673	unsigned int ssid_bits;
641	};	674	};
642		675
643	/* SMMU private data for an IOMMU domain */	676	/* SMMU private data for an IOMMU domain */
@@ -847,15 +880,22 @@ static int arm_smmu_cmdq_build_cmd(u64 cmd, struct arm_smmu_cmdq_ent ent) · 203 unmodified lines
847	cmd[1] \|= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);	880	cmd[1] \|= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
848	cmd[1] \|= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;	881	cmd[1] \|= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
849	break;	882	break;
		883	case CMDQ_OP_CFGI_CD:
		884	cmd[0] \|= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
		885	/* Fallthrough */
850	case CMDQ_OP_CFGI_STE:	886	case CMDQ_OP_CFGI_STE:
851	cmd[0] \|= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);	887	cmd[0] \|= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
852	cmd[1] \|= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);	888	cmd[1] \|= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
853	break;	889	break;
		890	case CMDQ_OP_CFGI_CD_ALL:
		891	cmd[0] \|= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
		892	break;
854	case CMDQ_OP_CFGI_ALL:	893	case CMDQ_OP_CFGI_ALL:
855	/* Cover the entire SID range */	894	/* Cover the entire SID range */
856	cmd[1] \|= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);	895	cmd[1] \|= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
857	break;	896	break;
858	case CMDQ_OP_TLBI_NH_VA:	897	case CMDQ_OP_TLBI_NH_VA:
		898	cmd[0] \|= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
859	cmd[0] \|= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);	899	cmd[0] \|= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
860	cmd[1] \|= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);	900	cmd[1] \|= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
861	cmd[1] \|= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;	901	cmd[1] \|= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
@@ -1443,50 +1483,238 @@ static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu) · 581 unmodified lines
1443	}	1483	}
1444		1484
1445	/* Context descriptor manipulation functions */	1485	/* Context descriptor manipulation functions */
1446	static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)	1486	static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
		1487	int ssid, bool leaf)
1447	{	1488	{
1448	u64 val = 0;	1489	size_t i;
		1490	unsigned long flags;
		1491	struct arm_smmu_master *master;
		1492	struct arm_smmu_device *smmu = smmu_domain->smmu;
		1493	struct arm_smmu_cmdq_ent cmd = {
		1494	.opcode = CMDQ_OP_CFGI_CD,
		1495	.cfgi = {
		1496	.ssid = ssid,
		1497	.leaf = leaf,
		1498	},
		1499	};
		1500
		1501	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
		1502	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
		1503	for (i = 0; i < master->num_sids; i++) {
		1504	cmd.cfgi.sid = master->sids[i];
		1505	arm_smmu_cmdq_issue_cmd(smmu, &cmd);
		1506	}
		1507	}
		1508	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
		1509
		1510	arm_smmu_cmdq_issue_sync(smmu);
		1511	}
1449		1512
1450	/* Repack the TCR. Just care about TTBR0 for now */	1513	static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
1451	val \|= ARM_SMMU_TCR2CD(tcr, T0SZ);	1514	struct arm_smmu_l1_ctx_desc *l1_desc)
1452	val \|= ARM_SMMU_TCR2CD(tcr, TG0);	1515	{
1453	val \|= ARM_SMMU_TCR2CD(tcr, IRGN0);	1516	size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1454	val \|= ARM_SMMU_TCR2CD(tcr, ORGN0);
1455	val \|= ARM_SMMU_TCR2CD(tcr, SH0);
1456	val \|= ARM_SMMU_TCR2CD(tcr, EPD0);
1457	val \|= ARM_SMMU_TCR2CD(tcr, EPD1);
1458	val \|= ARM_SMMU_TCR2CD(tcr, IPS);
1459		1517
1460	return val;	1518	l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
		1519	&l1_desc->l2ptr_dma, GFP_KERNEL);
		1520	if (!l1_desc->l2ptr) {
		1521	dev_warn(smmu->dev,
		1522	"failed to allocate context descriptor table\n");
		1523	return -ENOMEM;
		1524	}
		1525	return 0;
1461	}	1526	}
1462		1527
1463	static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,	1528	static void arm_smmu_write_cd_l1_desc(__le64 *dst,
1464	struct arm_smmu_s1_cfg *cfg)	1529	struct arm_smmu_l1_ctx_desc *l1_desc)
1465	{	1530	{
1466	u64 val;	1531	u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) \|
		1532	CTXDESC_L1_DESC_V;
		1533
		1534	WRITE_ONCE(*dst, cpu_to_le64(val));
		1535	}
		1536
		1537	static __le64 arm_smmu_get_cd_ptr(struct arm_smmu_domain smmu_domain,
		1538	u32 ssid)
		1539	{
		1540	__le64 *l1ptr;
		1541	unsigned int idx;
		1542	struct arm_smmu_l1_ctx_desc *l1_desc;
		1543	struct arm_smmu_device *smmu = smmu_domain->smmu;
		1544	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1467		1545
		1546	if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
		1547	return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
		1548
		1549	idx = ssid >> CTXDESC_SPLIT;
		1550	l1_desc = &cdcfg->l1_desc[idx];
		1551	if (!l1_desc->l2ptr) {
		1552	if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
		1553	return NULL;
		1554
		1555	l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
		1556	arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
		1557	/* An invalid L1CD can be cached */
		1558	arm_smmu_sync_cd(smmu_domain, ssid, false);
		1559	}
		1560	idx = ssid & (CTXDESC_L2_ENTRIES - 1);
		1561	return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
		1562	}
		1563
		1564	static int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain,
		1565	int ssid, struct arm_smmu_ctx_desc *cd)
		1566	{
1468	/*	1567	/*
1469	* We don't need to issue any invalidation here, as we'll invalidate	1568	* This function handles the following cases:
1470	* the STE when installing the new entry anyway.	1569	*
		1570	* (1) Install primary CD, for normal DMA traffic (SSID = 0).
		1571	* (2) Install a secondary CD, for SID+SSID traffic.
		1572	* (3) Update ASID of a CD. Atomically write the first 64 bits of the
		1573	* CD, then invalidate the old entry and mappings.
		1574	* (4) Remove a secondary CD.
1471	*/	1575	*/
1472	val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) \|	1576	u64 val;
		1577	bool cd_live;
		1578	__le64 *cdptr;
		1579	struct arm_smmu_device *smmu = smmu_domain->smmu;
		1580
		1581	if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
		1582	return -E2BIG;
		1583
		1584	cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
		1585	if (!cdptr)
		1586	return -ENOMEM;
		1587
		1588	val = le64_to_cpu(cdptr[0]);
		1589	cd_live = !!(val & CTXDESC_CD_0_V);
		1590
		1591	if (!cd) { /* (4) */
		1592	val = 0;
		1593	} else if (cd_live) { /* (3) */
		1594	val &= ~CTXDESC_CD_0_ASID;
		1595	val \|= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
		1596	/*
		1597	* Until CD+TLB invalidation, both ASIDs may be used for tagging
		1598	* this substream's traffic
		1599	*/
		1600	} else { /* (1) and (2) */
		1601	cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
		1602	cdptr[2] = 0;
		1603	cdptr[3] = cpu_to_le64(cd->mair);
		1604
		1605	/*
		1606	* STE is live, and the SMMU might read dwords of this CD in any
		1607	* order. Ensure that it observes valid values before reading
		1608	* V=1.
		1609	*/
		1610	arm_smmu_sync_cd(smmu_domain, ssid, true);
		1611
		1612	val = cd->tcr \|
1473	#ifdef __BIG_ENDIAN	1613	#ifdef __BIG_ENDIAN
1474	CTXDESC_CD_0_ENDI \|	1614	CTXDESC_CD_0_ENDI \|
1475	#endif	1615	#endif
1476	CTXDESC_CD_0_R \| CTXDESC_CD_0_A \| CTXDESC_CD_0_ASET \|	1616	CTXDESC_CD_0_R \| CTXDESC_CD_0_A \| CTXDESC_CD_0_ASET \|
1477	CTXDESC_CD_0_AA64 \| FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) \|	1617	CTXDESC_CD_0_AA64 \|
1478	CTXDESC_CD_0_V;	1618	FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) \|
		1619	CTXDESC_CD_0_V;
1479		1620
1480	/* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */	1621	/* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1481	if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)	1622	if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1482	val \|= CTXDESC_CD_0_S;	1623	val \|= CTXDESC_CD_0_S;
		1624	}
1483		1625
1484	cfg->cdptr[0] = cpu_to_le64(val);	1626	/*
		1627	* The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
		1628	* "Configuration structures and configuration invalidation completion"
		1629	*
		1630	* The size of single-copy atomic reads made by the SMMU is
		1631	* IMPLEMENTATION DEFINED but must be at least 64 bits. Any single
		1632	* field within an aligned 64-bit span of a structure can be altered
		1633	* without first making the structure invalid.
		1634	*/
		1635	WRITE_ONCE(cdptr[0], cpu_to_le64(val));
		1636	arm_smmu_sync_cd(smmu_domain, ssid, true);
		1637	return 0;
		1638	}
		1639
		1640	static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
		1641	{
		1642	int ret;
		1643	size_t l1size;
		1644	size_t max_contexts;
		1645	struct arm_smmu_device *smmu = smmu_domain->smmu;
		1646	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
		1647	struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
		1648
		1649	max_contexts = 1 << cfg->s1cdmax;
		1650
		1651	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) \|\|
		1652	max_contexts <= CTXDESC_L2_ENTRIES) {
		1653	cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
		1654	cdcfg->num_l1_ents = max_contexts;
1485		1655
1486	val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;	1656	l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
1487	cfg->cdptr[1] = cpu_to_le64(val);	1657	} else {
		1658	cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
		1659	cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
		1660	CTXDESC_L2_ENTRIES);
		1661
		1662	cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
		1663	sizeof(*cdcfg->l1_desc),
		1664	GFP_KERNEL);
		1665	if (!cdcfg->l1_desc)
		1666	return -ENOMEM;
		1667
		1668	l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
		1669	}
		1670
		1671	cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
		1672	GFP_KERNEL);
		1673	if (!cdcfg->cdtab) {
		1674	dev_warn(smmu->dev, "failed to allocate context descriptor\n");
		1675	ret = -ENOMEM;
		1676	goto err_free_l1;
		1677	}
		1678
		1679	return 0;
1488		1680
1489	cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);	1681	err_free_l1:
		1682	if (cdcfg->l1_desc) {
		1683	devm_kfree(smmu->dev, cdcfg->l1_desc);
		1684	cdcfg->l1_desc = NULL;
		1685	}
		1686	return ret;
		1687	}
		1688
		1689	static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
		1690	{
		1691	int i;
		1692	size_t size, l1size;
		1693	struct arm_smmu_device *smmu = smmu_domain->smmu;
		1694	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
		1695
		1696	if (cdcfg->l1_desc) {
		1697	size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
		1698
		1699	for (i = 0; i < cdcfg->num_l1_ents; i++) {
		1700	if (!cdcfg->l1_desc[i].l2ptr)
		1701	continue;
		1702
		1703	dmam_free_coherent(smmu->dev, size,
		1704	cdcfg->l1_desc[i].l2ptr,
		1705	cdcfg->l1_desc[i].l2ptr_dma);
		1706	}
		1707	devm_kfree(smmu->dev, cdcfg->l1_desc);
		1708	cdcfg->l1_desc = NULL;
		1709
		1710	l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
		1711	} else {
		1712	l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
		1713	}
		1714
		1715	dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
		1716	cdcfg->cdtab_dma = 0;
		1717	cdcfg->cdtab = NULL;
1490	}	1718	}
1491		1719
1492	/* Stream table manipulation functions */	1720	/* Stream table manipulation functions */
@@ -1608,6 +1836,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, · 115 unmodified lines
1608	if (s1_cfg) {	1836	if (s1_cfg) {
1609	BUG_ON(ste_live);	1837	BUG_ON(ste_live);
1610	dst[1] = cpu_to_le64(	1838	dst[1] = cpu_to_le64(
		1839	FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) \|
1611	FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) \|	1840	FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) \|
1612	FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) \|	1841	FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) \|
1613	FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) \|	1842	FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) \|
@@ -1617,8 +1846,10 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, · 3 unmodified lines
1617	!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))	1846	!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1618	dst[1] \|= cpu_to_le64(STRTAB_STE_1_S1STALLD);	1847	dst[1] \|= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1619		1848
1620	val \|= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) \|	1849	val \|= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) \|
1621	FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS);	1850	FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) \|
		1851	FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) \|
		1852	FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
1622	}	1853	}
1623		1854
1624	if (s2_cfg) {	1855	if (s2_cfg) {
@@ -1642,7 +1873,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, · 17 unmodified lines
1642	STRTAB_STE_1_EATS_TRANS));	1873	STRTAB_STE_1_EATS_TRANS));
1643		1874
1644	arm_smmu_sync_ste_for_sid(smmu, sid);	1875	arm_smmu_sync_ste_for_sid(smmu, sid);
1645	dst[0] = cpu_to_le64(val);	1876	/* See comment in arm_smmu_write_ctx_desc() */
		1877	WRITE_ONCE(dst[0], cpu_to_le64(val));
1646	arm_smmu_sync_ste_for_sid(smmu, sid);	1878	arm_smmu_sync_ste_for_sid(smmu, sid);
1647		1879
1648	/* It's likely that we'll want to use the new STE soon */	1880	/* It's likely that we'll want to use the new STE soon */
@@ -1675,7 +1907,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) · 26 unmodified lines
1675		1907
1676	desc->span = STRTAB_SPLIT + 1;	1908	desc->span = STRTAB_SPLIT + 1;
1677	desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,	1909	desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1678	GFP_KERNEL \| __GFP_ZERO);	1910	GFP_KERNEL);
1679	if (!desc->l2ptr) {	1911	if (!desc->l2ptr) {
1680	dev_err(smmu->dev,	1912	dev_err(smmu->dev,
1681	"failed to allocate l2 stream table for SID %u\n",	1913	"failed to allocate l2 stream table for SID %u\n",
@@ -2131,12 +2363,8 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) · 449 unmodified lines
2131	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {	2363	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2132	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;	2364	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2133		2365
2134	if (cfg->cdptr) {	2366	if (cfg->cdcfg.cdtab) {
2135	dmam_free_coherent(smmu_domain->smmu->dev,	2367	arm_smmu_free_cd_tables(smmu_domain);
2136	CTXDESC_CD_DWORDS << 3,
2137	cfg->cdptr,
2138	cfg->cdptr_dma);
2139
2140	arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);	2368	arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
2141	}	2369	}
2142	} else {	2370	} else {
@@ -2149,55 +2377,82 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) · 6 unmodified lines
2149	}	2377	}
2150		2378
2151	static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,	2379	static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
		2380	struct arm_smmu_master *master,
2152	struct io_pgtable_cfg *pgtbl_cfg)	2381	struct io_pgtable_cfg *pgtbl_cfg)
2153	{	2382	{
2154	int ret;	2383	int ret;
2155	int asid;	2384	int asid;
2156	struct arm_smmu_device *smmu = smmu_domain->smmu;	2385	struct arm_smmu_device *smmu = smmu_domain->smmu;
2157	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;	2386	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
		2387	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2158		2388
2159	asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);	2389	asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
2160	if (asid < 0)	2390	if (asid < 0)
2161	return asid;	2391	return asid;
2162		2392
2163	cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,	2393	cfg->s1cdmax = master->ssid_bits;
2164	&cfg->cdptr_dma,	2394
2165	GFP_KERNEL \| __GFP_ZERO);	2395	ret = arm_smmu_alloc_cd_tables(smmu_domain);
2166	if (!cfg->cdptr) {	2396	if (ret)
2167	dev_warn(smmu->dev, "failed to allocate context descriptor\n");
2168	ret = -ENOMEM;
2169	goto out_free_asid;	2397	goto out_free_asid;
2170	}
2171		2398
2172	cfg->cd.asid = (u16)asid;	2399	cfg->cd.asid = (u16)asid;
2173	cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];	2400	cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
2174	cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;	2401	cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) \|
		2402	FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) \|
		2403	FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) \|
		2404	FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) \|
		2405	FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) \|
		2406	FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) \|
		2407	CTXDESC_CD_0_TCR_EPD1 \| CTXDESC_CD_0_AA64;
2175	cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;	2408	cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;
		2409
		2410	/*
		2411	* Note that this will end up calling arm_smmu_sync_cd() before
		2412	* the master has been added to the devices list for this domain.
		2413	* This isn't an issue because the STE hasn't been installed yet.
		2414	*/
		2415	ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd);
		2416	if (ret)
		2417	goto out_free_cd_tables;
		2418
2176	return 0;	2419	return 0;
2177		2420
		2421	out_free_cd_tables:
		2422	arm_smmu_free_cd_tables(smmu_domain);
2178	out_free_asid:	2423	out_free_asid:
2179	arm_smmu_bitmap_free(smmu->asid_map, asid);	2424	arm_smmu_bitmap_free(smmu->asid_map, asid);
2180	return ret;	2425	return ret;
2181	}	2426	}
2182		2427
2183	static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,	2428	static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
		2429	struct arm_smmu_master *master,
2184	struct io_pgtable_cfg *pgtbl_cfg)	2430	struct io_pgtable_cfg *pgtbl_cfg)
2185	{	2431	{
2186	int vmid;	2432	int vmid;
2187	struct arm_smmu_device *smmu = smmu_domain->smmu;	2433	struct arm_smmu_device *smmu = smmu_domain->smmu;
2188	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;	2434	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
		2435	typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;
2189		2436
2190	vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);	2437	vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
2191	if (vmid < 0)	2438	if (vmid < 0)
2192	return vmid;	2439	return vmid;
2193		2440
		2441	vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2194	cfg->vmid = (u16)vmid;	2442	cfg->vmid = (u16)vmid;
2195	cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;	2443	cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2196	cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;	2444	cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) \|
		2445	FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) \|
		2446	FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) \|
		2447	FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) \|
		2448	FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) \|
		2449	FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) \|
		2450	FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
2197	return 0;	2451	return 0;
2198	}	2452	}
2199		2453
2200	static int arm_smmu_domain_finalise(struct iommu_domain *domain)	2454	static int arm_smmu_domain_finalise(struct iommu_domain *domain,
		2455	struct arm_smmu_master *master)
2201	{	2456	{
2202	int ret;	2457	int ret;
2203	unsigned long ias, oas;	2458	unsigned long ias, oas;
@@ -2205,6 +2460,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) · 1 unmodified line
2205	struct io_pgtable_cfg pgtbl_cfg;	2460	struct io_pgtable_cfg pgtbl_cfg;
2206	struct io_pgtable_ops *pgtbl_ops;	2461	struct io_pgtable_ops *pgtbl_ops;
2207	int (finalise_stage_fn)(struct arm_smmu_domain ,	2462	int (finalise_stage_fn)(struct arm_smmu_domain ,
		2463	struct arm_smmu_master *,
2208	struct io_pgtable_cfg *);	2464	struct io_pgtable_cfg *);
2209	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);	2465	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2210	struct arm_smmu_device *smmu = smmu_domain->smmu;	2466	struct arm_smmu_device *smmu = smmu_domain->smmu;
@@ -2259,7 +2515,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) · 48 unmodified lines
2259	domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;	2515	domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2260	domain->geometry.force_aperture = true;	2516	domain->geometry.force_aperture = true;
2261		2517
2262	ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);	2518	ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);
2263	if (ret < 0) {	2519	if (ret < 0) {
2264	free_io_pgtable_ops(pgtbl_ops);	2520	free_io_pgtable_ops(pgtbl_ops);
2265	return ret;	2521	return ret;
@@ -2412,7 +2668,7 @@ static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev) · 146 unmodified lines
2412		2668
2413	if (!smmu_domain->smmu) {	2669	if (!smmu_domain->smmu) {
2414	smmu_domain->smmu = smmu;	2670	smmu_domain->smmu = smmu;
2415	ret = arm_smmu_domain_finalise(domain);	2671	ret = arm_smmu_domain_finalise(domain, master);
2416	if (ret) {	2672	if (ret) {
2417	smmu_domain->smmu = NULL;	2673	smmu_domain->smmu = NULL;
2418	goto out_unlock;	2674	goto out_unlock;
@@ -2424,6 +2680,13 @@ static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev) · 5 unmodified lines
2424	dev_name(smmu->dev));	2680	dev_name(smmu->dev));
2425	ret = -ENXIO;	2681	ret = -ENXIO;
2426	goto out_unlock;	2682	goto out_unlock;
		2683	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
		2684	master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
		2685	dev_err(dev,
		2686	"cannot attach to incompatible domain (%u SSID bits != %u)\n",
		2687	smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);
		2688	ret = -EINVAL;
		2689	goto out_unlock;
2427	}	2690	}
2428		2691
2429	master->domain = smmu_domain;	2692	master->domain = smmu_domain;
@@ -2431,9 +2694,6 @@ static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev) · 1 unmodified line
2431	if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)	2694	if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2432	master->ats_enabled = arm_smmu_ats_supported(master);	2695	master->ats_enabled = arm_smmu_ats_supported(master);
2433		2696
2434	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2435	arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg);
2436
2437	arm_smmu_install_ste_for_dev(master);	2697	arm_smmu_install_ste_for_dev(master);
2438		2698
2439	spin_lock_irqsave(&smmu_domain->devices_lock, flags);	2699	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
@@ -2534,51 +2794,66 @@ static int arm_smmu_add_device(struct device *dev) · 94 unmodified lines
2534		2794
2535	if (!fwspec \|\| fwspec->ops != &arm_smmu_ops)	2795	if (!fwspec \|\| fwspec->ops != &arm_smmu_ops)
2536	return -ENODEV;	2796	return -ENODEV;
2537	/*
2538	* We _can_ actually withstand dodgy bus code re-calling add_device()
2539	* without an intervening remove_device()/of_xlate() sequence, but
2540	* we're not going to do so quietly...
2541	*/
2542	if (WARN_ON_ONCE(fwspec->iommu_priv)) {
2543	master = fwspec->iommu_priv;
2544	smmu = master->smmu;
2545	} else {
2546	smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2547	if (!smmu)
2548	return -ENODEV;
2549	master = kzalloc(sizeof(*master), GFP_KERNEL);
2550	if (!master)
2551	return -ENOMEM;
2552		2797
2553	master->dev = dev;	2798	if (WARN_ON_ONCE(fwspec->iommu_priv))
2554	master->smmu = smmu;	2799	return -EBUSY;
2555	master->sids = fwspec->ids;	2800
2556	master->num_sids = fwspec->num_ids;	2801	smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2557	fwspec->iommu_priv = master;	2802	if (!smmu)
2558	}	2803	return -ENODEV;
		2804
		2805	master = kzalloc(sizeof(*master), GFP_KERNEL);
		2806	if (!master)
		2807	return -ENOMEM;
		2808
		2809	master->dev = dev;
		2810	master->smmu = smmu;
		2811	master->sids = fwspec->ids;
		2812	master->num_sids = fwspec->num_ids;
		2813	fwspec->iommu_priv = master;
2559		2814
2560	/* Check the SIDs are in range of the SMMU and our stream table */	2815	/* Check the SIDs are in range of the SMMU and our stream table */
2561	for (i = 0; i < master->num_sids; i++) {	2816	for (i = 0; i < master->num_sids; i++) {
2562	u32 sid = master->sids[i];	2817	u32 sid = master->sids[i];
2563		2818
2564	if (!arm_smmu_sid_in_range(smmu, sid))	2819	if (!arm_smmu_sid_in_range(smmu, sid)) {
2565	return -ERANGE;	2820	ret = -ERANGE;
		2821	goto err_free_master;
		2822	}
2566		2823
2567	/* Ensure l2 strtab is initialised */	2824	/* Ensure l2 strtab is initialised */
2568	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {	2825	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2569	ret = arm_smmu_init_l2_strtab(smmu, sid);	2826	ret = arm_smmu_init_l2_strtab(smmu, sid);
2570	if (ret)	2827	if (ret)
2571	return ret;	2828	goto err_free_master;
2572	}	2829	}
2573	}	2830	}
2574		2831
		2832	master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits);
		2833
		2834	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
		2835	master->ssid_bits = min_t(u8, master->ssid_bits,
		2836	CTXDESC_LINEAR_CDMAX);
		2837
		2838	ret = iommu_device_link(&smmu->iommu, dev);
		2839	if (ret)
		2840	goto err_free_master;
		2841
2575	group = iommu_group_get_for_dev(dev);	2842	group = iommu_group_get_for_dev(dev);
2576	if (!IS_ERR(group)) {	2843	if (IS_ERR(group)) {
2577	iommu_group_put(group);	2844	ret = PTR_ERR(group);
2578	iommu_device_link(&smmu->iommu, dev);	2845	goto err_unlink;
2579	}	2846	}
2580		2847
2581	return PTR_ERR_OR_ZERO(group);	2848	iommu_group_put(group);
		2849	return 0;
		2850
		2851	err_unlink:
		2852	iommu_device_unlink(&smmu->iommu, dev);
		2853	err_free_master:
		2854	kfree(master);
		2855	fwspec->iommu_priv = NULL;
		2856	return ret;
2582	}	2857	}
2583		2858
2584	static void arm_smmu_remove_device(struct device *dev)	2859	static void arm_smmu_remove_device(struct device *dev)
@@ -2710,15 +2985,6 @@ static void arm_smmu_get_resv_regions(struct device *dev, · 125 unmodified lines
2710	iommu_dma_get_resv_regions(dev, head);	2985	iommu_dma_get_resv_regions(dev, head);
2711	}	2986	}
2712		2987
2713	static void arm_smmu_put_resv_regions(struct device *dev,
2714	struct list_head *head)
2715	{
2716	struct iommu_resv_region entry, next;
2717
2718	list_for_each_entry_safe(entry, next, head, list)
2719	kfree(entry);
2720	}
2721
2722	static struct iommu_ops arm_smmu_ops = {	2988	static struct iommu_ops arm_smmu_ops = {
2723	.capable = arm_smmu_capable,	2989	.capable = arm_smmu_capable,
2724	.domain_alloc = arm_smmu_domain_alloc,	2990	.domain_alloc = arm_smmu_domain_alloc,
@@ -2736,7 +3002,7 @@ static struct iommu_ops arm_smmu_ops = { · 11 unmodified lines
2736	.domain_set_attr = arm_smmu_domain_set_attr,	3002	.domain_set_attr = arm_smmu_domain_set_attr,
2737	.of_xlate = arm_smmu_of_xlate,	3003	.of_xlate = arm_smmu_of_xlate,
2738	.get_resv_regions = arm_smmu_get_resv_regions,	3004	.get_resv_regions = arm_smmu_get_resv_regions,
2739	.put_resv_regions = arm_smmu_put_resv_regions,	3005	.put_resv_regions = generic_iommu_put_resv_regions,
2740	.pgsize_bitmap = -1UL, /* Restricted during device attach */	3006	.pgsize_bitmap = -1UL, /* Restricted during device attach */
2741	};	3007	};
2742		3008
@@ -2883,7 +3149,7 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) · 140 unmodified lines
2883		3149
2884	l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);	3150	l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2885	strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,	3151	strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2886	GFP_KERNEL \| __GFP_ZERO);	3152	GFP_KERNEL);
2887	if (!strtab) {	3153	if (!strtab) {
2888	dev_err(smmu->dev,	3154	dev_err(smmu->dev,
2889	"failed to allocate l1 stream table (%u bytes)\n",	3155	"failed to allocate l1 stream table (%u bytes)\n",
@@ -2910,7 +3176,7 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) · 20 unmodified lines
2910		3176
2911	size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);	3177	size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2912	strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,	3178	strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2913	GFP_KERNEL \| __GFP_ZERO);	3179	GFP_KERNEL);
2914	if (!strtab) {	3180	if (!strtab) {
2915	dev_err(smmu->dev,	3181	dev_err(smmu->dev,
2916	"failed to allocate linear stream table (%u bytes)\n",	3182	"failed to allocate linear stream table (%u bytes)\n",
@@ -3570,6 +3836,43 @@ static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) · 653 unmodified lines
3570	return SZ_128K;	3836	return SZ_128K;
3571	}	3837	}
3572		3838
		3839	static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
		3840	{
		3841	int err;
		3842
		3843	#ifdef CONFIG_PCI
		3844	if (pci_bus_type.iommu_ops != ops) {
		3845	err = bus_set_iommu(&pci_bus_type, ops);
		3846	if (err)
		3847	return err;
		3848	}
		3849	#endif
		3850	#ifdef CONFIG_ARM_AMBA
		3851	if (amba_bustype.iommu_ops != ops) {
		3852	err = bus_set_iommu(&amba_bustype, ops);
		3853	if (err)
		3854	goto err_reset_pci_ops;
		3855	}
		3856	#endif
		3857	if (platform_bus_type.iommu_ops != ops) {
		3858	err = bus_set_iommu(&platform_bus_type, ops);
		3859	if (err)
		3860	goto err_reset_amba_ops;
		3861	}
		3862
		3863	return 0;
		3864
		3865	err_reset_amba_ops:
		3866	#ifdef CONFIG_ARM_AMBA
		3867	bus_set_iommu(&amba_bustype, NULL);
		3868	#endif
		3869	err_reset_pci_ops: __maybe_unused;
		3870	#ifdef CONFIG_PCI
		3871	bus_set_iommu(&pci_bus_type, NULL);
		3872	#endif
		3873	return err;
		3874	}
		3875
3573	static int arm_smmu_device_probe(struct platform_device *pdev)	3876	static int arm_smmu_device_probe(struct platform_device *pdev)
3574	{	3877	{
3575	int irq, ret;	3878	int irq, ret;
@@ -3599,7 +3902,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 23 unmodified lines
3599		3902
3600	/* Base address */	3903	/* Base address */
3601	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);	3904	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3602	if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) {	3905	if (resource_size(res) < arm_smmu_resource_size(smmu)) {
3603	dev_err(dev, "MMIO region too small (%pr)\n", res);	3906	dev_err(dev, "MMIO region too small (%pr)\n", res);
3604	return -EINVAL;	3907	return -EINVAL;
3605	}	3908	}
@@ -3660,48 +3963,45 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 54 unmodified lines
3660	return ret;	3963	return ret;
3661	}	3964	}
3662		3965
3663	#ifdef CONFIG_PCI	3966	return arm_smmu_set_bus_ops(&arm_smmu_ops);
3664	if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
3665	pci_request_acs();
3666	ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
3667	if (ret)
3668	return ret;
3669	}
3670	#endif
3671	#ifdef CONFIG_ARM_AMBA
3672	if (amba_bustype.iommu_ops != &arm_smmu_ops) {
3673	ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
3674	if (ret)
3675	return ret;
3676	}
3677	#endif
3678	if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
3679	ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
3680	if (ret)
3681	return ret;
3682	}
3683	return 0;
3684	}	3967	}
3685		3968
3686	static void arm_smmu_device_shutdown(struct platform_device *pdev)	3969	static int arm_smmu_device_remove(struct platform_device *pdev)
3687	{	3970	{
3688	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);	3971	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3689		3972
		3973	arm_smmu_set_bus_ops(NULL);
		3974	iommu_device_unregister(&smmu->iommu);
		3975	iommu_device_sysfs_remove(&smmu->iommu);
3690	arm_smmu_device_disable(smmu);	3976	arm_smmu_device_disable(smmu);
		3977
		3978	return 0;
		3979	}
		3980
		3981	static void arm_smmu_device_shutdown(struct platform_device *pdev)
		3982	{
		3983	arm_smmu_device_remove(pdev);
3691	}	3984	}
3692		3985
3693	static const struct of_device_id arm_smmu_of_match[] = {	3986	static const struct of_device_id arm_smmu_of_match[] = {
3694	{ .compatible = "arm,smmu-v3", },	3987	{ .compatible = "arm,smmu-v3", },
3695	{ },	3988	{ },
3696	};	3989	};
		3990	MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
3697		3991
3698	static struct platform_driver arm_smmu_driver = {	3992	static struct platform_driver arm_smmu_driver = {
3699	.driver = {	3993	.driver = {
3700	.name = "arm-smmu-v3",	3994	.name = "arm-smmu-v3",
3701	.of_match_table = of_match_ptr(arm_smmu_of_match),	3995	.of_match_table = arm_smmu_of_match,
3702	.suppress_bind_attrs = true,	3996	.suppress_bind_attrs = true,
3703	},	3997	},
3704	.probe = arm_smmu_device_probe,	3998	.probe = arm_smmu_device_probe,
		3999	.remove = arm_smmu_device_remove,
3705	.shutdown = arm_smmu_device_shutdown,	4000	.shutdown = arm_smmu_device_shutdown,
3706	};	4001	};
3707	builtin_platform_driver(arm_smmu_driver);	4002	module_platform_driver(arm_smmu_driver);
		4003
		4004	MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
		4005	MODULE_AUTHOR("Will Deacon <will@kernel.org>");
		4006	MODULE_ALIAS("platform:arm-smmu-v3");
		4007	MODULE_LICENSE("GPL v2");

drivers/iommu/arm-smmu.c+198−136

@@ -27,8 +27,7 @@ · 26 unmodified lines
27	#include <linux/interrupt.h>	27	#include <linux/interrupt.h>
28	#include <linux/io.h>	28	#include <linux/io.h>
29	#include <linux/iopoll.h>	29	#include <linux/iopoll.h>
30	#include <linux/init.h>	30	#include <linux/module.h>
31	#include <linux/moduleparam.h>
32	#include <linux/of.h>	31	#include <linux/of.h>
33	#include <linux/of_address.h>	32	#include <linux/of_address.h>
34	#include <linux/of_device.h>	33	#include <linux/of_device.h>
@@ -60,10 +59,6 @@ · 25 unmodified lines
60	#define MSI_IOVA_LENGTH 0x100000	59	#define MSI_IOVA_LENGTH 0x100000
61		60
62	static int force_stage;	61	static int force_stage;
63	/*
64	* not really modular, but the easiest way to keep compat with existing
65	* bootargs behaviour is to continue using module_param() here.
66	*/
67	module_param(force_stage, int, S_IRUGO);	62	module_param(force_stage, int, S_IRUGO);
68	MODULE_PARM_DESC(force_stage,	63	MODULE_PARM_DESC(force_stage,
69	"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");	64	"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
@@ -131,6 +126,12 @@ static struct arm_smmu_domain to_smmu_domain(struct iommu_domain dom) · 61 unmodified lines
131	return container_of(dom, struct arm_smmu_domain, domain);	126	return container_of(dom, struct arm_smmu_domain, domain);
132	}	127	}
133		128
		129	static struct platform_driver arm_smmu_driver;
		130	static struct iommu_ops arm_smmu_ops;
		131
		132	#ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
		133	static int arm_smmu_bus_init(struct iommu_ops *ops);
		134
134	static struct device_node dev_get_dev_node(struct device dev)	135	static struct device_node dev_get_dev_node(struct device dev)
135	{	136	{
136	if (dev_is_pci(dev)) {	137	if (dev_is_pci(dev)) {
@@ -166,9 +167,6 @@ static int __find_legacy_master_phandle(struct device dev, void data) · 29 unmodified lines
166	return err == -ENOENT ? 0 : err;	167	return err == -ENOENT ? 0 : err;
167	}	168	}
168		169
169	static struct platform_driver arm_smmu_driver;
170	static struct iommu_ops arm_smmu_ops;
171
172	static int arm_smmu_register_legacy_master(struct device *dev,	170	static int arm_smmu_register_legacy_master(struct device *dev,
173	struct arm_smmu_device **smmu)	171	struct arm_smmu_device **smmu)
174	{	172	{
@@ -220,6 +218,27 @@ static int arm_smmu_register_legacy_master(struct device *dev, · 45 unmodified lines
220	return err;	218	return err;
221	}	219	}
222		220
		221	/*
		222	* With the legacy DT binding in play, we have no guarantees about
		223	* probe order, but then we're also not doing default domains, so we can
		224	* delay setting bus ops until we're sure every possible SMMU is ready,
		225	* and that way ensure that no add_device() calls get missed.
		226	*/
		227	static int arm_smmu_legacy_bus_init(void)
		228	{
		229	if (using_legacy_binding)
		230	return arm_smmu_bus_init(&arm_smmu_ops);
		231	return 0;
		232	}
		233	device_initcall_sync(arm_smmu_legacy_bus_init);
		234	#else
		235	static int arm_smmu_register_legacy_master(struct device *dev,
		236	struct arm_smmu_device **smmu)
		237	{
		238	return -ENODEV;
		239	}
		240	#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
		241
223	static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)	242	static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
224	{	243	{
225	int idx;	244	int idx;
@@ -252,7 +271,7 @@ static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, · 26 unmodified lines
252	for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {	271	for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
253	for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {	272	for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
254	reg = arm_smmu_readl(smmu, page, status);	273	reg = arm_smmu_readl(smmu, page, status);
255	if (!(reg & sTLBGSTATUS_GSACTIVE))	274	if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
256	return;	275	return;
257	cpu_relax();	276	cpu_relax();
258	}	277	}
@@ -459,7 +478,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) · 200 unmodified lines
459	int idx = smmu_domain->cfg.cbndx;	478	int idx = smmu_domain->cfg.cbndx;
460		479
461	fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);	480	fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
462	if (!(fsr & FSR_FAULT))	481	if (!(fsr & ARM_SMMU_FSR_FAULT))
463	return IRQ_NONE;	482	return IRQ_NONE;
464		483
465	fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);	484	fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
@@ -491,7 +510,7 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) · 25 unmodified lines
491		510
492	if (__ratelimit(&rs)) {	511	if (__ratelimit(&rs)) {
493	if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&	512	if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
494	(gfsr & sGFSR_USF))	513	(gfsr & ARM_SMMU_sGFSR_USF))
495	dev_err(smmu->dev,	514	dev_err(smmu->dev,
496	"Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",	515	"Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
497	(u16)gfsynr1);	516	(u16)gfsynr1);
@@ -521,26 +540,28 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, · 23 unmodified lines
521	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {	540	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
522	cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;	541	cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
523	} else {	542	} else {
524	cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;	543	cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
525	cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;	544	cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
526	cb->tcr[1] \|= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
527	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)	545	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
528	cb->tcr[1] \|= TCR2_AS;	546	cb->tcr[1] \|= ARM_SMMU_TCR2_AS;
		547	else
		548	cb->tcr[0] \|= ARM_SMMU_TCR_EAE;
529	}	549	}
530	} else {	550	} else {
531	cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;	551	cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
532	}	552	}
533		553
534	/* TTBRs */	554	/* TTBRs */
535	if (stage1) {	555	if (stage1) {
536	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {	556	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
537	cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];	557	cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
538	cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];	558	cb->ttbr[1] = 0;
539	} else {	559	} else {
540	cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];	560	cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
541	cb->ttbr[0] \|= FIELD_PREP(TTBRn_ASID, cfg->asid);	561	cb->ttbr[0] \|= FIELD_PREP(ARM_SMMU_TTBRn_ASID,
542	cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];	562	cfg->asid);
543	cb->ttbr[1] \|= FIELD_PREP(TTBRn_ASID, cfg->asid);	563	cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
		564	cfg->asid);
544	}	565	}
545	} else {	566	} else {
546	cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;	567	cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
@@ -576,31 +597,33 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) · 29 unmodified lines
576	/* CBA2R */	597	/* CBA2R */
577	if (smmu->version > ARM_SMMU_V1) {	598	if (smmu->version > ARM_SMMU_V1) {
578	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)	599	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
579	reg = CBA2R_VA64;	600	reg = ARM_SMMU_CBA2R_VA64;
580	else	601	else
581	reg = 0;	602	reg = 0;
582	/* 16-bit VMIDs live in CBA2R */	603	/* 16-bit VMIDs live in CBA2R */
583	if (smmu->features & ARM_SMMU_FEAT_VMID16)	604	if (smmu->features & ARM_SMMU_FEAT_VMID16)
584	reg \|= FIELD_PREP(CBA2R_VMID16, cfg->vmid);	605	reg \|= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
585		606
586	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);	607	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
587	}	608	}
588		609
589	/* CBAR */	610	/* CBAR */
590	reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);	611	reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
591	if (smmu->version < ARM_SMMU_V2)	612	if (smmu->version < ARM_SMMU_V2)
592	reg \|= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);	613	reg \|= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
593		614
594	/*	615	/*
595	* Use the weakest shareability/memory types, so they are	616	* Use the weakest shareability/memory types, so they are
596	* overridden by the ttbcr/pte.	617	* overridden by the ttbcr/pte.
597	*/	618	*/
598	if (stage1) {	619	if (stage1) {
599	reg \|= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) \|	620	reg \|= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
600	FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);	621	ARM_SMMU_CBAR_S1_BPSHCFG_NSH) \|
		622	FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
		623	ARM_SMMU_CBAR_S1_MEMATTR_WB);
601	} else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {	624	} else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
602	/* 8-bit VMIDs live in CBAR */	625	/* 8-bit VMIDs live in CBAR */
603	reg \|= FIELD_PREP(CBAR_VMID, cfg->vmid);	626	reg \|= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
604	}	627	}
605	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);	628	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
606		629
@@ -632,11 +655,12 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) · 25 unmodified lines
632	}	655	}
633		656
634	/* SCTLR */	657	/* SCTLR */
635	reg = SCTLR_CFIE \| SCTLR_CFRE \| SCTLR_AFE \| SCTLR_TRE \| SCTLR_M;	658	reg = ARM_SMMU_SCTLR_CFIE \| ARM_SMMU_SCTLR_CFRE \| ARM_SMMU_SCTLR_AFE \|
		659	ARM_SMMU_SCTLR_TRE \| ARM_SMMU_SCTLR_M;
636	if (stage1)	660	if (stage1)
637	reg \|= SCTLR_S1_ASIDPNE;	661	reg \|= ARM_SMMU_SCTLR_S1_ASIDPNE;
638	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))	662	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
639	reg \|= SCTLR_E;	663	reg \|= ARM_SMMU_SCTLR_E;
640		664
641	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);	665	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
642	}	666	}
@@ -818,7 +842,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, · 175 unmodified lines
818	if (ret < 0) {	842	if (ret < 0) {
819	dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",	843	dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
820	cfg->irptndx, irq);	844	cfg->irptndx, irq);
821	cfg->irptndx = INVALID_IRPTNDX;	845	cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
822	}	846	}
823		847
824	mutex_unlock(&smmu_domain->init_mutex);	848	mutex_unlock(&smmu_domain->init_mutex);
@@ -856,7 +880,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) · 31 unmodified lines
856	smmu->cbs[cfg->cbndx].cfg = NULL;	880	smmu->cbs[cfg->cbndx].cfg = NULL;
857	arm_smmu_write_context_bank(smmu, cfg->cbndx);	881	arm_smmu_write_context_bank(smmu, cfg->cbndx);
858		882
859	if (cfg->irptndx != INVALID_IRPTNDX) {	883	if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
860	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];	884	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
861	devm_free_irq(smmu->dev, irq, domain);	885	devm_free_irq(smmu->dev, irq, domain);
862	}	886	}
@@ -912,23 +936,24 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) · 49 unmodified lines
912	static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)	936	static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
913	{	937	{
914	struct arm_smmu_smr *smr = smmu->smrs + idx;	938	struct arm_smmu_smr *smr = smmu->smrs + idx;
915	u32 reg = FIELD_PREP(SMR_ID, smr->id) \| FIELD_PREP(SMR_MASK, smr->mask);	939	u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) \|
		940	FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
916		941
917	if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)	942	if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
918	reg \|= SMR_VALID;	943	reg \|= ARM_SMMU_SMR_VALID;
919	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);	944	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
920	}	945	}
921		946
922	static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)	947	static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
923	{	948	{
924	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;	949	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
925	u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) \|	950	u32 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) \|
926	FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) \|	951	FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) \|
927	FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);	952	FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
928		953
929	if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&	954	if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
930	smmu->smrs[idx].valid)	955	smmu->smrs[idx].valid)
931	reg \|= S2CR_EXIDVALID;	956	reg \|= ARM_SMMU_S2CR_EXIDVALID;
932	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);	957	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
933	}	958	}
934		959
@@ -946,24 +971,37 @@ static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) · 11 unmodified lines
946	static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)	971	static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
947	{	972	{
948	u32 smr;	973	u32 smr;
		974	int i;
949		975
950	if (!smmu->smrs)	976	if (!smmu->smrs)
951	return;	977	return;
952		978	/*
		979	* If we've had to accommodate firmware memory regions, we may
		980	* have live SMRs by now; tread carefully...
		981	*
		982	* Somewhat perversely, not having a free SMR for this test implies we
		983	* can get away without it anyway, as we'll only be able to 'allocate'
		984	* these SMRs for the ID/mask values we're already trusting to be OK.
		985	*/
		986	for (i = 0; i < smmu->num_mapping_groups; i++)
		987	if (!smmu->smrs[i].valid)
		988	goto smr_ok;
		989	return;
		990	smr_ok:
953	/*	991	/*
954	* SMR.ID bits may not be preserved if the corresponding MASK	992	* SMR.ID bits may not be preserved if the corresponding MASK
955	* bits are set, so check each one separately. We can reject	993	* bits are set, so check each one separately. We can reject
956	* masters later if they try to claim IDs outside these masks.	994	* masters later if they try to claim IDs outside these masks.
957	*/	995	*/
958	smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);	996	smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
959	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);	997	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
960	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));	998	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
961	smmu->streamid_mask = FIELD_GET(SMR_ID, smr);	999	smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
962		1000
963	smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);	1001	smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
964	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);	1002	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
965	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));	1003	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
966	smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);	1004	smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
967	}	1005	}
968		1006
969	static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)	1007	static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
@@ -1032,8 +1070,8 @@ static int arm_smmu_master_alloc_smes(struct device *dev) · 62 unmodified lines
1032	mutex_lock(&smmu->stream_map_mutex);	1070	mutex_lock(&smmu->stream_map_mutex);
1033	/* Figure out a viable stream map entry allocation */	1071	/* Figure out a viable stream map entry allocation */
1034	for_each_cfg_sme(fwspec, i, idx) {	1072	for_each_cfg_sme(fwspec, i, idx) {
1035	u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);	1073	u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1036	u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);	1074	u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1037		1075
1038	if (idx != INVALID_SMENDX) {	1076	if (idx != INVALID_SMENDX) {
1039	ret = -EEXIST;	1077	ret = -EEXIST;
@@ -1277,7 +1315,8 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, · 237 unmodified lines
1277	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);	1315	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1278		1316
1279	reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;	1317	reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1280	if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) {	1318	if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
		1319	5, 50)) {
1281	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);	1320	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1282	dev_err(dev,	1321	dev_err(dev,
1283	"iova to phys timed out on %pad. Falling back to software table walk.\n",	1322	"iova to phys timed out on %pad. Falling back to software table walk.\n",
@@ -1287,7 +1326,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, · 3 unmodified lines
1287		1326
1288	phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);	1327	phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1289	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);	1328	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1290	if (phys & CB_PAR_F) {	1329	if (phys & ARM_SMMU_CB_PAR_F) {
1291	dev_err(dev, "translation fault!\n");	1330	dev_err(dev, "translation fault!\n");
1292	dev_err(dev, "PAR = 0x%llx\n", phys);	1331	dev_err(dev, "PAR = 0x%llx\n", phys);
1293	return 0;	1332	return 0;
@@ -1368,8 +1407,8 @@ static int arm_smmu_add_device(struct device *dev) · 74 unmodified lines
1368		1407
1369	ret = -EINVAL;	1408	ret = -EINVAL;
1370	for (i = 0; i < fwspec->num_ids; i++) {	1409	for (i = 0; i < fwspec->num_ids; i++) {
1371	u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);	1410	u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1372	u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);	1411	u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1373		1412
1374	if (sid & ~smmu->streamid_mask) {	1413	if (sid & ~smmu->streamid_mask) {
1375	dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",	1414	dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
@@ -1550,12 +1589,12 @@ static int arm_smmu_of_xlate(struct device dev, struct of_phandle_args args) · 174 unmodified lines
1550	u32 mask, fwid = 0;	1589	u32 mask, fwid = 0;
1551		1590
1552	if (args->args_count > 0)	1591	if (args->args_count > 0)
1553	fwid \|= FIELD_PREP(SMR_ID, args->args[0]);	1592	fwid \|= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1554		1593
1555	if (args->args_count > 1)	1594	if (args->args_count > 1)
1556	fwid \|= FIELD_PREP(SMR_MASK, args->args[1]);	1595	fwid \|= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1557	else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))	1596	else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1558	fwid \|= FIELD_PREP(SMR_MASK, mask);	1597	fwid \|= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1559		1598
1560	return iommu_fwspec_add_ids(dev, &fwid, 1);	1599	return iommu_fwspec_add_ids(dev, &fwid, 1);
1561	}	1600	}
@@ -1576,15 +1615,6 @@ static void arm_smmu_get_resv_regions(struct device *dev, · 14 unmodified lines
1576	iommu_dma_get_resv_regions(dev, head);	1615	iommu_dma_get_resv_regions(dev, head);
1577	}	1616	}
1578		1617
1579	static void arm_smmu_put_resv_regions(struct device *dev,
1580	struct list_head *head)
1581	{
1582	struct iommu_resv_region entry, next;
1583
1584	list_for_each_entry_safe(entry, next, head, list)
1585	kfree(entry);
1586	}
1587
1588	static struct iommu_ops arm_smmu_ops = {	1618	static struct iommu_ops arm_smmu_ops = {
1589	.capable = arm_smmu_capable,	1619	.capable = arm_smmu_capable,
1590	.domain_alloc = arm_smmu_domain_alloc,	1620	.domain_alloc = arm_smmu_domain_alloc,
@@ -1602,7 +1632,7 @@ static struct iommu_ops arm_smmu_ops = { · 11 unmodified lines
1602	.domain_set_attr = arm_smmu_domain_set_attr,	1632	.domain_set_attr = arm_smmu_domain_set_attr,
1603	.of_xlate = arm_smmu_of_xlate,	1633	.of_xlate = arm_smmu_of_xlate,
1604	.get_resv_regions = arm_smmu_get_resv_regions,	1634	.get_resv_regions = arm_smmu_get_resv_regions,
1605	.put_resv_regions = arm_smmu_put_resv_regions,	1635	.put_resv_regions = generic_iommu_put_resv_regions,
1606	.pgsize_bitmap = -1UL, /* Restricted during device attach */	1636	.pgsize_bitmap = -1UL, /* Restricted during device attach */
1607	};	1637	};
1608		1638
@@ -1625,7 +1655,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) · 16 unmodified lines
1625	/* Make sure all context banks are disabled and clear CB_FSR */	1655	/* Make sure all context banks are disabled and clear CB_FSR */
1626	for (i = 0; i < smmu->num_context_banks; ++i) {	1656	for (i = 0; i < smmu->num_context_banks; ++i) {
1627	arm_smmu_write_context_bank(smmu, i);	1657	arm_smmu_write_context_bank(smmu, i);
1628	arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT);	1658	arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1629	}	1659	}
1630		1660
1631	/* Invalidate the TLB, just in case */	1661	/* Invalidate the TLB, just in case */
@@ -1635,29 +1665,30 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) · 3 unmodified lines
1635	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);	1665	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1636		1666
1637	/* Enable fault reporting */	1667	/* Enable fault reporting */
1638	reg \|= (sCR0_GFRE \| sCR0_GFIE \| sCR0_GCFGFRE \| sCR0_GCFGFIE);	1668	reg \|= (ARM_SMMU_sCR0_GFRE \| ARM_SMMU_sCR0_GFIE \|
		1669	ARM_SMMU_sCR0_GCFGFRE \| ARM_SMMU_sCR0_GCFGFIE);
1639		1670
1640	/* Disable TLB broadcasting. */	1671	/* Disable TLB broadcasting. */
1641	reg \|= (sCR0_VMIDPNE \| sCR0_PTM);	1672	reg \|= (ARM_SMMU_sCR0_VMIDPNE \| ARM_SMMU_sCR0_PTM);
1642		1673
1643	/* Enable client access, handling unmatched streams as appropriate */	1674	/* Enable client access, handling unmatched streams as appropriate */
1644	reg &= ~sCR0_CLIENTPD;	1675	reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1645	if (disable_bypass)	1676	if (disable_bypass)
1646	reg \|= sCR0_USFCFG;	1677	reg \|= ARM_SMMU_sCR0_USFCFG;
1647	else	1678	else
1648	reg &= ~sCR0_USFCFG;	1679	reg &= ~ARM_SMMU_sCR0_USFCFG;
1649		1680
1650	/* Disable forced broadcasting */	1681	/* Disable forced broadcasting */
1651	reg &= ~sCR0_FB;	1682	reg &= ~ARM_SMMU_sCR0_FB;
1652		1683
1653	/* Don't upgrade barriers */	1684	/* Don't upgrade barriers */
1654	reg &= ~(sCR0_BSU);	1685	reg &= ~(ARM_SMMU_sCR0_BSU);
1655		1686
1656	if (smmu->features & ARM_SMMU_FEAT_VMID16)	1687	if (smmu->features & ARM_SMMU_FEAT_VMID16)
1657	reg \|= sCR0_VMID16EN;	1688	reg \|= ARM_SMMU_sCR0_VMID16EN;
1658		1689
1659	if (smmu->features & ARM_SMMU_FEAT_EXIDS)	1690	if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1660	reg \|= sCR0_EXIDENABLE;	1691	reg \|= ARM_SMMU_sCR0_EXIDENABLE;
1661		1692
1662	if (smmu->impl && smmu->impl->reset)	1693	if (smmu->impl && smmu->impl->reset)
1663	smmu->impl->reset(smmu);	1694	smmu->impl->reset(smmu);
@@ -1702,21 +1733,21 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 38 unmodified lines
1702		1733
1703	/* Restrict available stages based on module parameter */	1734	/* Restrict available stages based on module parameter */
1704	if (force_stage == 1)	1735	if (force_stage == 1)
1705	id &= ~(ID0_S2TS \| ID0_NTS);	1736	id &= ~(ARM_SMMU_ID0_S2TS \| ARM_SMMU_ID0_NTS);
1706	else if (force_stage == 2)	1737	else if (force_stage == 2)
1707	id &= ~(ID0_S1TS \| ID0_NTS);	1738	id &= ~(ARM_SMMU_ID0_S1TS \| ARM_SMMU_ID0_NTS);
1708		1739
1709	if (id & ID0_S1TS) {	1740	if (id & ARM_SMMU_ID0_S1TS) {
1710	smmu->features \|= ARM_SMMU_FEAT_TRANS_S1;	1741	smmu->features \|= ARM_SMMU_FEAT_TRANS_S1;
1711	dev_notice(smmu->dev, "\tstage 1 translation\n");	1742	dev_notice(smmu->dev, "\tstage 1 translation\n");
1712	}	1743	}
1713		1744
1714	if (id & ID0_S2TS) {	1745	if (id & ARM_SMMU_ID0_S2TS) {
1715	smmu->features \|= ARM_SMMU_FEAT_TRANS_S2;	1746	smmu->features \|= ARM_SMMU_FEAT_TRANS_S2;
1716	dev_notice(smmu->dev, "\tstage 2 translation\n");	1747	dev_notice(smmu->dev, "\tstage 2 translation\n");
1717	}	1748	}
1718		1749
1719	if (id & ID0_NTS) {	1750	if (id & ARM_SMMU_ID0_NTS) {
1720	smmu->features \|= ARM_SMMU_FEAT_TRANS_NESTED;	1751	smmu->features \|= ARM_SMMU_FEAT_TRANS_NESTED;
1721	dev_notice(smmu->dev, "\tnested translation\n");	1752	dev_notice(smmu->dev, "\tnested translation\n");
1722	}	1753	}
@@ -1727,8 +1758,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 4 unmodified lines
1727	return -ENODEV;	1758	return -ENODEV;
1728	}	1759	}
1729		1760
1730	if ((id & ID0_S1TS) &&	1761	if ((id & ARM_SMMU_ID0_S1TS) &&
1731	((smmu->version < ARM_SMMU_V2) \|\| !(id & ID0_ATOSNS))) {	1762	((smmu->version < ARM_SMMU_V2) \|\| !(id & ARM_SMMU_ID0_ATOSNS))) {
1732	smmu->features \|= ARM_SMMU_FEAT_TRANS_OPS;	1763	smmu->features \|= ARM_SMMU_FEAT_TRANS_OPS;
1733	dev_notice(smmu->dev, "\taddress translation ops\n");	1764	dev_notice(smmu->dev, "\taddress translation ops\n");
1734	}	1765	}
@@ -1739,7 +1770,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 4 unmodified lines
1739	* Fortunately, this also opens up a workaround for systems where the	1770	* Fortunately, this also opens up a workaround for systems where the
1740	* ID register value has ended up configured incorrectly.	1771	* ID register value has ended up configured incorrectly.
1741	*/	1772	*/
1742	cttw_reg = !!(id & ID0_CTTW);	1773	cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1743	if (cttw_fw \|\| cttw_reg)	1774	if (cttw_fw \|\| cttw_reg)
1744	dev_notice(smmu->dev, "\t%scoherent table walk\n",	1775	dev_notice(smmu->dev, "\t%scoherent table walk\n",
1745	cttw_fw ? "" : "non-");	1776	cttw_fw ? "" : "non-");
@@ -1748,16 +1779,16 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 2 unmodified lines
1748	"\t(IDR0.CTTW overridden by FW configuration)\n");	1779	"\t(IDR0.CTTW overridden by FW configuration)\n");
1749		1780
1750	/* Max. number of entries we have for stream matching/indexing */	1781	/* Max. number of entries we have for stream matching/indexing */
1751	if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {	1782	if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1752	smmu->features \|= ARM_SMMU_FEAT_EXIDS;	1783	smmu->features \|= ARM_SMMU_FEAT_EXIDS;
1753	size = 1 << 16;	1784	size = 1 << 16;
1754	} else {	1785	} else {
1755	size = 1 << FIELD_GET(ID0_NUMSIDB, id);	1786	size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1756	}	1787	}
1757	smmu->streamid_mask = size - 1;	1788	smmu->streamid_mask = size - 1;
1758	if (id & ID0_SMS) {	1789	if (id & ARM_SMMU_ID0_SMS) {
1759	smmu->features \|= ARM_SMMU_FEAT_STREAM_MATCH;	1790	smmu->features \|= ARM_SMMU_FEAT_STREAM_MATCH;
1760	size = FIELD_GET(ID0_NUMSMRG, id);	1791	size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1761	if (size == 0) {	1792	if (size == 0) {
1762	dev_err(smmu->dev,	1793	dev_err(smmu->dev,
1763	"stream-matching supported, but no SMRs present!\n");	1794	"stream-matching supported, but no SMRs present!\n");
@@ -1785,18 +1816,19 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 21 unmodified lines
1785	mutex_init(&smmu->stream_map_mutex);	1816	mutex_init(&smmu->stream_map_mutex);
1786	spin_lock_init(&smmu->global_sync_lock);	1817	spin_lock_init(&smmu->global_sync_lock);
1787		1818
1788	if (smmu->version < ARM_SMMU_V2 \|\| !(id & ID0_PTFS_NO_AARCH32)) {	1819	if (smmu->version < ARM_SMMU_V2 \|\|
		1820	!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1789	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH32_L;	1821	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH32_L;
1790	if (!(id & ID0_PTFS_NO_AARCH32S))	1822	if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1791	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH32_S;	1823	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH32_S;
1792	}	1824	}
1793		1825
1794	/* ID1 */	1826	/* ID1 */
1795	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);	1827	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1796	smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;	1828	smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1797		1829
1798	/* Check for size mismatch of SMMU address space from mapped region */	1830	/* Check for size mismatch of SMMU address space from mapped region */
1799	size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);	1831	size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1800	if (smmu->numpage != 2 * size << smmu->pgshift)	1832	if (smmu->numpage != 2 * size << smmu->pgshift)
1801	dev_warn(smmu->dev,	1833	dev_warn(smmu->dev,
1802	"SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",	1834	"SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
@@ -1804,8 +1836,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 1 unmodified line
1804	/* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */	1836	/* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1805	smmu->numpage = size;	1837	smmu->numpage = size;
1806		1838
1807	smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);	1839	smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1808	smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);	1840	smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1809	if (smmu->num_s2_context_banks > smmu->num_context_banks) {	1841	if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1810	dev_err(smmu->dev, "impossible number of S2 context banks!\n");	1842	dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1811	return -ENODEV;	1843	return -ENODEV;
@@ -1819,14 +1851,14 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 7 unmodified lines
1819		1851
1820	/* ID2 */	1852	/* ID2 */
1821	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);	1853	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1822	size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));	1854	size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1823	smmu->ipa_size = size;	1855	smmu->ipa_size = size;
1824		1856
1825	/* The output mask is also applied for bypass */	1857	/* The output mask is also applied for bypass */
1826	size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));	1858	size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1827	smmu->pa_size = size;	1859	smmu->pa_size = size;
1828		1860
1829	if (id & ID2_VMID16)	1861	if (id & ARM_SMMU_ID2_VMID16)
1830	smmu->features \|= ARM_SMMU_FEAT_VMID16;	1862	smmu->features \|= ARM_SMMU_FEAT_VMID16;
1831		1863
1832	/*	1864	/*
@@ -1843,13 +1875,13 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 10 unmodified lines
1843	if (smmu->version == ARM_SMMU_V1_64K)	1875	if (smmu->version == ARM_SMMU_V1_64K)
1844	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_64K;	1876	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1845	} else {	1877	} else {
1846	size = FIELD_GET(ID2_UBS, id);	1878	size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1847	smmu->va_size = arm_smmu_id_size_to_bits(size);	1879	smmu->va_size = arm_smmu_id_size_to_bits(size);
1848	if (id & ID2_PTFS_4K)	1880	if (id & ARM_SMMU_ID2_PTFS_4K)
1849	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_4K;	1881	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1850	if (id & ID2_PTFS_16K)	1882	if (id & ARM_SMMU_ID2_PTFS_16K)
1851	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_16K;	1883	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1852	if (id & ID2_PTFS_64K)	1884	if (id & ARM_SMMU_ID2_PTFS_64K)
1853	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_64K;	1885	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1854	}	1886	}
1855		1887
@@ -1911,6 +1943,7 @@ static const struct of_device_id arm_smmu_of_match[] = { · 55 unmodified lines
1911	{ .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },	1943	{ .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1912	{ },	1944	{ },
1913	};	1945	};
		1946	MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1914		1947
1915	#ifdef CONFIG_ACPI	1948	#ifdef CONFIG_ACPI
1916	static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)	1949	static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
@@ -1997,8 +2030,10 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev, · 80 unmodified lines
1997		2030
1998	legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);	2031	legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1999	if (legacy_binding && !using_generic_binding) {	2032	if (legacy_binding && !using_generic_binding) {
2000	if (!using_legacy_binding)	2033	if (!using_legacy_binding) {
2001	pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");	2034	pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
		2035	IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
		2036	}
2002	using_legacy_binding = true;	2037	using_legacy_binding = true;
2003	} else if (!legacy_binding && !using_legacy_binding) {	2038	} else if (!legacy_binding && !using_legacy_binding) {
2004	using_generic_binding = true;	2039	using_generic_binding = true;
@@ -2013,25 +2048,50 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev, · 8 unmodified lines
2013	return 0;	2048	return 0;
2014	}	2049	}
2015		2050
2016	static void arm_smmu_bus_init(void)	2051	static int arm_smmu_bus_init(struct iommu_ops *ops)
2017	{	2052	{
		2053	int err;
		2054
2018	/* Oh, for a proper bus abstraction */	2055	/* Oh, for a proper bus abstraction */
2019	if (!iommu_present(&platform_bus_type))	2056	if (!iommu_present(&platform_bus_type)) {
2020	bus_set_iommu(&platform_bus_type, &arm_smmu_ops);	2057	err = bus_set_iommu(&platform_bus_type, ops);
		2058	if (err)
		2059	return err;
		2060	}
2021	#ifdef CONFIG_ARM_AMBA	2061	#ifdef CONFIG_ARM_AMBA
2022	if (!iommu_present(&amba_bustype))	2062	if (!iommu_present(&amba_bustype)) {
2023	bus_set_iommu(&amba_bustype, &arm_smmu_ops);	2063	err = bus_set_iommu(&amba_bustype, ops);
		2064	if (err)
		2065	goto err_reset_platform_ops;
		2066	}
2024	#endif	2067	#endif
2025	#ifdef CONFIG_PCI	2068	#ifdef CONFIG_PCI
2026	if (!iommu_present(&pci_bus_type)) {	2069	if (!iommu_present(&pci_bus_type)) {
2027	pci_request_acs();	2070	err = bus_set_iommu(&pci_bus_type, ops);
2028	bus_set_iommu(&pci_bus_type, &arm_smmu_ops);	2071	if (err)
		2072	goto err_reset_amba_ops;
2029	}	2073	}
2030	#endif	2074	#endif
2031	#ifdef CONFIG_FSL_MC_BUS	2075	#ifdef CONFIG_FSL_MC_BUS
2032	if (!iommu_present(&fsl_mc_bus_type))	2076	if (!iommu_present(&fsl_mc_bus_type)) {
2033	bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);	2077	err = bus_set_iommu(&fsl_mc_bus_type, ops);
		2078	if (err)
		2079	goto err_reset_pci_ops;
		2080	}
2034	#endif	2081	#endif
		2082	return 0;
		2083
		2084	err_reset_pci_ops: __maybe_unused;
		2085	#ifdef CONFIG_PCI
		2086	bus_set_iommu(&pci_bus_type, NULL);
		2087	#endif
		2088	err_reset_amba_ops: __maybe_unused;
		2089	#ifdef CONFIG_ARM_AMBA
		2090	bus_set_iommu(&amba_bustype, NULL);
		2091	#endif
		2092	err_reset_platform_ops: __maybe_unused;
		2093	bus_set_iommu(&platform_bus_type, NULL);
		2094	return err;
2035	}	2095	}
2036		2096
2037	static int arm_smmu_device_probe(struct platform_device *pdev)	2097	static int arm_smmu_device_probe(struct platform_device *pdev)
@@ -2177,38 +2237,28 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 139 unmodified lines
2177	* ready to handle default domain setup as soon as any SMMU exists.	2237	* ready to handle default domain setup as soon as any SMMU exists.
2178	*/	2238	*/
2179	if (!using_legacy_binding)	2239	if (!using_legacy_binding)
2180	arm_smmu_bus_init();	2240	return arm_smmu_bus_init(&arm_smmu_ops);
2181		2241
2182	return 0;	2242	return 0;
2183	}	2243	}
2184		2244
2185	/*	2245	static int arm_smmu_device_remove(struct platform_device *pdev)
2186	* With the legacy DT binding in play, though, we have no guarantees about
2187	* probe order, but then we're also not doing default domains, so we can
2188	* delay setting bus ops until we're sure every possible SMMU is ready,
2189	* and that way ensure that no add_device() calls get missed.
2190	*/
2191	static int arm_smmu_legacy_bus_init(void)
2192	{
2193	if (using_legacy_binding)
2194	arm_smmu_bus_init();
2195	return 0;
2196	}
2197	device_initcall_sync(arm_smmu_legacy_bus_init);
2198
2199	static void arm_smmu_device_shutdown(struct platform_device *pdev)
2200	{	2246	{
2201	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);	2247	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2202		2248
2203	if (!smmu)	2249	if (!smmu)
2204	return;	2250	return -ENODEV;
2205		2251
2206	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))	2252	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2207	dev_err(&pdev->dev, "removing device with active domains!\n");	2253	dev_err(&pdev->dev, "removing device with active domains!\n");
2208		2254
		2255	arm_smmu_bus_init(NULL);
		2256	iommu_device_unregister(&smmu->iommu);
		2257	iommu_device_sysfs_remove(&smmu->iommu);
		2258
2209	arm_smmu_rpm_get(smmu);	2259	arm_smmu_rpm_get(smmu);
2210	/* Turn the thing off */	2260	/* Turn the thing off */
2211	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD);	2261	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2212	arm_smmu_rpm_put(smmu);	2262	arm_smmu_rpm_put(smmu);
2213		2263
2214	if (pm_runtime_enabled(smmu->dev))	2264	if (pm_runtime_enabled(smmu->dev))
@@ -2217,6 +2267,12 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev) · 2 unmodified lines
2217	clk_bulk_disable(smmu->num_clks, smmu->clks);	2267	clk_bulk_disable(smmu->num_clks, smmu->clks);
2218		2268
2219	clk_bulk_unprepare(smmu->num_clks, smmu->clks);	2269	clk_bulk_unprepare(smmu->num_clks, smmu->clks);
		2270	return 0;
		2271	}
		2272
		2273	static void arm_smmu_device_shutdown(struct platform_device *pdev)
		2274	{
		2275	arm_smmu_device_remove(pdev);
2220	}	2276	}
2221		2277
2222	static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)	2278	static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
@@ -2267,11 +2323,17 @@ static const struct dev_pm_ops arm_smmu_pm_ops = { · 44 unmodified lines
2267	static struct platform_driver arm_smmu_driver = {	2323	static struct platform_driver arm_smmu_driver = {
2268	.driver = {	2324	.driver = {
2269	.name = "arm-smmu",	2325	.name = "arm-smmu",
2270	.of_match_table = of_match_ptr(arm_smmu_of_match),	2326	.of_match_table = arm_smmu_of_match,
2271	.pm = &arm_smmu_pm_ops,	2327	.pm = &arm_smmu_pm_ops,
2272	.suppress_bind_attrs = true,	2328	.suppress_bind_attrs = true,
2273	},	2329	},
2274	.probe = arm_smmu_device_probe,	2330	.probe = arm_smmu_device_probe,
		2331	.remove = arm_smmu_device_remove,
2275	.shutdown = arm_smmu_device_shutdown,	2332	.shutdown = arm_smmu_device_shutdown,
2276	};	2333	};
2277	builtin_platform_driver(arm_smmu_driver);	2334	module_platform_driver(arm_smmu_driver);
		2335
		2336	MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
		2337	MODULE_AUTHOR("Will Deacon <will@kernel.org>");
		2338	MODULE_ALIAS("platform:arm-smmu");
		2339	MODULE_LICENSE("GPL v2");

drivers/iommu/arm-smmu.h+141−87

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	#define _ARM_SMMU_H	11	#define _ARM_SMMU_H
12		12
13	#include <linux/atomic.h>	13	#include <linux/atomic.h>
		14	#include <linux/bitfield.h>
14	#include <linux/bits.h>	15	#include <linux/bits.h>
15	#include <linux/clk.h>	16	#include <linux/clk.h>
16	#include <linux/device.h>	17	#include <linux/device.h>
@@ -23,51 +24,51 @@ · 6 unmodified lines
23		24
24	/* Configuration registers */	25	/* Configuration registers */
25	#define ARM_SMMU_GR0_sCR0 0x0	26	#define ARM_SMMU_GR0_sCR0 0x0
26	#define sCR0_VMID16EN BIT(31)	27	#define ARM_SMMU_sCR0_VMID16EN BIT(31)
27	#define sCR0_BSU GENMASK(15, 14)	28	#define ARM_SMMU_sCR0_BSU GENMASK(15, 14)
28	#define sCR0_FB BIT(13)	29	#define ARM_SMMU_sCR0_FB BIT(13)
29	#define sCR0_PTM BIT(12)	30	#define ARM_SMMU_sCR0_PTM BIT(12)
30	#define sCR0_VMIDPNE BIT(11)	31	#define ARM_SMMU_sCR0_VMIDPNE BIT(11)
31	#define sCR0_USFCFG BIT(10)	32	#define ARM_SMMU_sCR0_USFCFG BIT(10)
32	#define sCR0_GCFGFIE BIT(5)	33	#define ARM_SMMU_sCR0_GCFGFIE BIT(5)
33	#define sCR0_GCFGFRE BIT(4)	34	#define ARM_SMMU_sCR0_GCFGFRE BIT(4)
34	#define sCR0_EXIDENABLE BIT(3)	35	#define ARM_SMMU_sCR0_EXIDENABLE BIT(3)
35	#define sCR0_GFIE BIT(2)	36	#define ARM_SMMU_sCR0_GFIE BIT(2)
36	#define sCR0_GFRE BIT(1)	37	#define ARM_SMMU_sCR0_GFRE BIT(1)
37	#define sCR0_CLIENTPD BIT(0)	38	#define ARM_SMMU_sCR0_CLIENTPD BIT(0)
38		39
39	/* Auxiliary Configuration register */	40	/* Auxiliary Configuration register */
40	#define ARM_SMMU_GR0_sACR 0x10	41	#define ARM_SMMU_GR0_sACR 0x10
41		42
42	/* Identification registers */	43	/* Identification registers */
43	#define ARM_SMMU_GR0_ID0 0x20	44	#define ARM_SMMU_GR0_ID0 0x20
44	#define ID0_S1TS BIT(30)	45	#define ARM_SMMU_ID0_S1TS BIT(30)
45	#define ID0_S2TS BIT(29)	46	#define ARM_SMMU_ID0_S2TS BIT(29)
46	#define ID0_NTS BIT(28)	47	#define ARM_SMMU_ID0_NTS BIT(28)
47	#define ID0_SMS BIT(27)	48	#define ARM_SMMU_ID0_SMS BIT(27)
48	#define ID0_ATOSNS BIT(26)	49	#define ARM_SMMU_ID0_ATOSNS BIT(26)
49	#define ID0_PTFS_NO_AARCH32 BIT(25)	50	#define ARM_SMMU_ID0_PTFS_NO_AARCH32 BIT(25)
50	#define ID0_PTFS_NO_AARCH32S BIT(24)	51	#define ARM_SMMU_ID0_PTFS_NO_AARCH32S BIT(24)
51	#define ID0_NUMIRPT GENMASK(23, 16)	52	#define ARM_SMMU_ID0_NUMIRPT GENMASK(23, 16)
52	#define ID0_CTTW BIT(14)	53	#define ARM_SMMU_ID0_CTTW BIT(14)
53	#define ID0_NUMSIDB GENMASK(12, 9)	54	#define ARM_SMMU_ID0_NUMSIDB GENMASK(12, 9)
54	#define ID0_EXIDS BIT(8)	55	#define ARM_SMMU_ID0_EXIDS BIT(8)
55	#define ID0_NUMSMRG GENMASK(7, 0)	56	#define ARM_SMMU_ID0_NUMSMRG GENMASK(7, 0)
56		57
57	#define ARM_SMMU_GR0_ID1 0x24	58	#define ARM_SMMU_GR0_ID1 0x24
58	#define ID1_PAGESIZE BIT(31)	59	#define ARM_SMMU_ID1_PAGESIZE BIT(31)
59	#define ID1_NUMPAGENDXB GENMASK(30, 28)	60	#define ARM_SMMU_ID1_NUMPAGENDXB GENMASK(30, 28)
60	#define ID1_NUMS2CB GENMASK(23, 16)	61	#define ARM_SMMU_ID1_NUMS2CB GENMASK(23, 16)
61	#define ID1_NUMCB GENMASK(7, 0)	62	#define ARM_SMMU_ID1_NUMCB GENMASK(7, 0)
62		63
63	#define ARM_SMMU_GR0_ID2 0x28	64	#define ARM_SMMU_GR0_ID2 0x28
64	#define ID2_VMID16 BIT(15)	65	#define ARM_SMMU_ID2_VMID16 BIT(15)
65	#define ID2_PTFS_64K BIT(14)	66	#define ARM_SMMU_ID2_PTFS_64K BIT(14)
66	#define ID2_PTFS_16K BIT(13)	67	#define ARM_SMMU_ID2_PTFS_16K BIT(13)
67	#define ID2_PTFS_4K BIT(12)	68	#define ARM_SMMU_ID2_PTFS_4K BIT(12)
68	#define ID2_UBS GENMASK(11, 8)	69	#define ARM_SMMU_ID2_UBS GENMASK(11, 8)
69	#define ID2_OAS GENMASK(7, 4)	70	#define ARM_SMMU_ID2_OAS GENMASK(7, 4)
70	#define ID2_IAS GENMASK(3, 0)	71	#define ARM_SMMU_ID2_IAS GENMASK(3, 0)
71		72
72	#define ARM_SMMU_GR0_ID3 0x2c	73	#define ARM_SMMU_GR0_ID3 0x2c
73	#define ARM_SMMU_GR0_ID4 0x30	74	#define ARM_SMMU_GR0_ID4 0x30
@@ -75,11 +76,11 @@ · 1 unmodified line
75	#define ARM_SMMU_GR0_ID6 0x38	76	#define ARM_SMMU_GR0_ID6 0x38
76		77
77	#define ARM_SMMU_GR0_ID7 0x3c	78	#define ARM_SMMU_GR0_ID7 0x3c
78	#define ID7_MAJOR GENMASK(7, 4)	79	#define ARM_SMMU_ID7_MAJOR GENMASK(7, 4)
79	#define ID7_MINOR GENMASK(3, 0)	80	#define ARM_SMMU_ID7_MINOR GENMASK(3, 0)
80		81
81	#define ARM_SMMU_GR0_sGFSR 0x48	82	#define ARM_SMMU_GR0_sGFSR 0x48
82	#define sGFSR_USF BIT(1)	83	#define ARM_SMMU_sGFSR_USF BIT(1)
83		84
84	#define ARM_SMMU_GR0_sGFSYNR0 0x50	85	#define ARM_SMMU_GR0_sGFSYNR0 0x50
85	#define ARM_SMMU_GR0_sGFSYNR1 0x54	86	#define ARM_SMMU_GR0_sGFSYNR1 0x54
@@ -92,106 +93,132 @@ · 6 unmodified lines
92	#define ARM_SMMU_GR0_sTLBGSYNC 0x70	93	#define ARM_SMMU_GR0_sTLBGSYNC 0x70
93		94
94	#define ARM_SMMU_GR0_sTLBGSTATUS 0x74	95	#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
95	#define sTLBGSTATUS_GSACTIVE BIT(0)	96	#define ARM_SMMU_sTLBGSTATUS_GSACTIVE BIT(0)
96		97
97	/* Stream mapping registers */	98	/* Stream mapping registers */
98	#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))	99	#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
99	#define SMR_VALID BIT(31)	100	#define ARM_SMMU_SMR_VALID BIT(31)
100	#define SMR_MASK GENMASK(31, 16)	101	#define ARM_SMMU_SMR_MASK GENMASK(31, 16)
101	#define SMR_ID GENMASK(15, 0)	102	#define ARM_SMMU_SMR_ID GENMASK(15, 0)
102		103
103	#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))	104	#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
104	#define S2CR_PRIVCFG GENMASK(25, 24)	105	#define ARM_SMMU_S2CR_PRIVCFG GENMASK(25, 24)
105	enum arm_smmu_s2cr_privcfg {	106	enum arm_smmu_s2cr_privcfg {
106	S2CR_PRIVCFG_DEFAULT,	107	S2CR_PRIVCFG_DEFAULT,
107	S2CR_PRIVCFG_DIPAN,	108	S2CR_PRIVCFG_DIPAN,
108	S2CR_PRIVCFG_UNPRIV,	109	S2CR_PRIVCFG_UNPRIV,
109	S2CR_PRIVCFG_PRIV,	110	S2CR_PRIVCFG_PRIV,
110	};	111	};
111	#define S2CR_TYPE GENMASK(17, 16)	112	#define ARM_SMMU_S2CR_TYPE GENMASK(17, 16)
112	enum arm_smmu_s2cr_type {	113	enum arm_smmu_s2cr_type {
113	S2CR_TYPE_TRANS,	114	S2CR_TYPE_TRANS,
114	S2CR_TYPE_BYPASS,	115	S2CR_TYPE_BYPASS,
115	S2CR_TYPE_FAULT,	116	S2CR_TYPE_FAULT,
116	};	117	};
117	#define S2CR_EXIDVALID BIT(10)	118	#define ARM_SMMU_S2CR_EXIDVALID BIT(10)
118	#define S2CR_CBNDX GENMASK(7, 0)	119	#define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0)
119		120
120	/* Context bank attribute registers */	121	/* Context bank attribute registers */
121	#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))	122	#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
122	#define CBAR_IRPTNDX GENMASK(31, 24)	123	#define ARM_SMMU_CBAR_IRPTNDX GENMASK(31, 24)
123	#define CBAR_TYPE GENMASK(17, 16)	124	#define ARM_SMMU_CBAR_TYPE GENMASK(17, 16)
124	enum arm_smmu_cbar_type {	125	enum arm_smmu_cbar_type {
125	CBAR_TYPE_S2_TRANS,	126	CBAR_TYPE_S2_TRANS,
126	CBAR_TYPE_S1_TRANS_S2_BYPASS,	127	CBAR_TYPE_S1_TRANS_S2_BYPASS,
127	CBAR_TYPE_S1_TRANS_S2_FAULT,	128	CBAR_TYPE_S1_TRANS_S2_FAULT,
128	CBAR_TYPE_S1_TRANS_S2_TRANS,	129	CBAR_TYPE_S1_TRANS_S2_TRANS,
129	};	130	};
130	#define CBAR_S1_MEMATTR GENMASK(15, 12)	131	#define ARM_SMMU_CBAR_S1_MEMATTR GENMASK(15, 12)
131	#define CBAR_S1_MEMATTR_WB 0xf	132	#define ARM_SMMU_CBAR_S1_MEMATTR_WB 0xf
132	#define CBAR_S1_BPSHCFG GENMASK(9, 8)	133	#define ARM_SMMU_CBAR_S1_BPSHCFG GENMASK(9, 8)
133	#define CBAR_S1_BPSHCFG_NSH 3	134	#define ARM_SMMU_CBAR_S1_BPSHCFG_NSH 3
134	#define CBAR_VMID GENMASK(7, 0)	135	#define ARM_SMMU_CBAR_VMID GENMASK(7, 0)
135		136
136	#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))	137	#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
137		138
138	#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))	139	#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
139	#define CBA2R_VMID16 GENMASK(31, 16)	140	#define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16)
140	#define CBA2R_VA64 BIT(0)	141	#define ARM_SMMU_CBA2R_VA64 BIT(0)
141		142
142	#define ARM_SMMU_CB_SCTLR 0x0	143	#define ARM_SMMU_CB_SCTLR 0x0
143	#define SCTLR_S1_ASIDPNE BIT(12)	144	#define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12)
144	#define SCTLR_CFCFG BIT(7)	145	#define ARM_SMMU_SCTLR_CFCFG BIT(7)
145	#define SCTLR_CFIE BIT(6)	146	#define ARM_SMMU_SCTLR_CFIE BIT(6)
146	#define SCTLR_CFRE BIT(5)	147	#define ARM_SMMU_SCTLR_CFRE BIT(5)
147	#define SCTLR_E BIT(4)	148	#define ARM_SMMU_SCTLR_E BIT(4)
148	#define SCTLR_AFE BIT(2)	149	#define ARM_SMMU_SCTLR_AFE BIT(2)
149	#define SCTLR_TRE BIT(1)	150	#define ARM_SMMU_SCTLR_TRE BIT(1)
150	#define SCTLR_M BIT(0)	151	#define ARM_SMMU_SCTLR_M BIT(0)
151		152
152	#define ARM_SMMU_CB_ACTLR 0x4	153	#define ARM_SMMU_CB_ACTLR 0x4
153		154
154	#define ARM_SMMU_CB_RESUME 0x8	155	#define ARM_SMMU_CB_RESUME 0x8
155	#define RESUME_TERMINATE BIT(0)	156	#define ARM_SMMU_RESUME_TERMINATE BIT(0)
156		157
157	#define ARM_SMMU_CB_TCR2 0x10	158	#define ARM_SMMU_CB_TCR2 0x10
158	#define TCR2_SEP GENMASK(17, 15)	159	#define ARM_SMMU_TCR2_SEP GENMASK(17, 15)
159	#define TCR2_SEP_UPSTREAM 0x7	160	#define ARM_SMMU_TCR2_SEP_UPSTREAM 0x7
160	#define TCR2_AS BIT(4)	161	#define ARM_SMMU_TCR2_AS BIT(4)
		162	#define ARM_SMMU_TCR2_PASIZE GENMASK(3, 0)
161		163
162	#define ARM_SMMU_CB_TTBR0 0x20	164	#define ARM_SMMU_CB_TTBR0 0x20
163	#define ARM_SMMU_CB_TTBR1 0x28	165	#define ARM_SMMU_CB_TTBR1 0x28
164	#define TTBRn_ASID GENMASK_ULL(63, 48)	166	#define ARM_SMMU_TTBRn_ASID GENMASK_ULL(63, 48)
165		167
166	#define ARM_SMMU_CB_TCR 0x30	168	#define ARM_SMMU_CB_TCR 0x30
		169	#define ARM_SMMU_TCR_EAE BIT(31)
		170	#define ARM_SMMU_TCR_EPD1 BIT(23)
		171	#define ARM_SMMU_TCR_TG0 GENMASK(15, 14)
		172	#define ARM_SMMU_TCR_SH0 GENMASK(13, 12)
		173	#define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10)
		174	#define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8)
		175	#define ARM_SMMU_TCR_T0SZ GENMASK(5, 0)
		176
		177	#define ARM_SMMU_VTCR_RES1 BIT(31)
		178	#define ARM_SMMU_VTCR_PS GENMASK(18, 16)
		179	#define ARM_SMMU_VTCR_TG0 ARM_SMMU_TCR_TG0
		180	#define ARM_SMMU_VTCR_SH0 ARM_SMMU_TCR_SH0
		181	#define ARM_SMMU_VTCR_ORGN0 ARM_SMMU_TCR_ORGN0
		182	#define ARM_SMMU_VTCR_IRGN0 ARM_SMMU_TCR_IRGN0
		183	#define ARM_SMMU_VTCR_SL0 GENMASK(7, 6)
		184	#define ARM_SMMU_VTCR_T0SZ ARM_SMMU_TCR_T0SZ
		185
167	#define ARM_SMMU_CB_CONTEXTIDR 0x34	186	#define ARM_SMMU_CB_CONTEXTIDR 0x34
168	#define ARM_SMMU_CB_S1_MAIR0 0x38	187	#define ARM_SMMU_CB_S1_MAIR0 0x38
169	#define ARM_SMMU_CB_S1_MAIR1 0x3c	188	#define ARM_SMMU_CB_S1_MAIR1 0x3c
170		189
171	#define ARM_SMMU_CB_PAR 0x50	190	#define ARM_SMMU_CB_PAR 0x50
172	#define CB_PAR_F BIT(0)	191	#define ARM_SMMU_CB_PAR_F BIT(0)
173		192
174	#define ARM_SMMU_CB_FSR 0x58	193	#define ARM_SMMU_CB_FSR 0x58
175	#define FSR_MULTI BIT(31)	194	#define ARM_SMMU_FSR_MULTI BIT(31)
176	#define FSR_SS BIT(30)	195	#define ARM_SMMU_FSR_SS BIT(30)
177	#define FSR_UUT BIT(8)	196	#define ARM_SMMU_FSR_UUT BIT(8)
178	#define FSR_ASF BIT(7)	197	#define ARM_SMMU_FSR_ASF BIT(7)
179	#define FSR_TLBLKF BIT(6)	198	#define ARM_SMMU_FSR_TLBLKF BIT(6)
180	#define FSR_TLBMCF BIT(5)	199	#define ARM_SMMU_FSR_TLBMCF BIT(5)
181	#define FSR_EF BIT(4)	200	#define ARM_SMMU_FSR_EF BIT(4)
182	#define FSR_PF BIT(3)	201	#define ARM_SMMU_FSR_PF BIT(3)
183	#define FSR_AFF BIT(2)	202	#define ARM_SMMU_FSR_AFF BIT(2)
184	#define FSR_TF BIT(1)	203	#define ARM_SMMU_FSR_TF BIT(1)
185		204
186	#define FSR_IGN (FSR_AFF \| FSR_ASF \| \	205	#define ARM_SMMU_FSR_IGN (ARM_SMMU_FSR_AFF \| \
187	FSR_TLBMCF \| FSR_TLBLKF)	206	ARM_SMMU_FSR_ASF \| \
188	#define FSR_FAULT (FSR_MULTI \| FSR_SS \| FSR_UUT \| \	207	ARM_SMMU_FSR_TLBMCF \| \
189	FSR_EF \| FSR_PF \| FSR_TF \| FSR_IGN)	208	ARM_SMMU_FSR_TLBLKF)
		209
		210	#define ARM_SMMU_FSR_FAULT (ARM_SMMU_FSR_MULTI \| \
		211	ARM_SMMU_FSR_SS \| \
		212	ARM_SMMU_FSR_UUT \| \
		213	ARM_SMMU_FSR_EF \| \
		214	ARM_SMMU_FSR_PF \| \
		215	ARM_SMMU_FSR_TF \| \
		216	ARM_SMMU_FSR_IGN)
190		217
191	#define ARM_SMMU_CB_FAR 0x60	218	#define ARM_SMMU_CB_FAR 0x60
192		219
193	#define ARM_SMMU_CB_FSYNR0 0x68	220	#define ARM_SMMU_CB_FSYNR0 0x68
194	#define FSYNR0_WNR BIT(4)	221	#define ARM_SMMU_FSYNR0_WNR BIT(4)
195		222
196	#define ARM_SMMU_CB_S1_TLBIVA 0x600	223	#define ARM_SMMU_CB_S1_TLBIVA 0x600
197	#define ARM_SMMU_CB_S1_TLBIASID 0x610	224	#define ARM_SMMU_CB_S1_TLBIASID 0x610
@@ -203,7 +230,7 @@ enum arm_smmu_cbar_type { · 5 unmodified lines
203	#define ARM_SMMU_CB_ATS1PR 0x800	230	#define ARM_SMMU_CB_ATS1PR 0x800
204		231
205	#define ARM_SMMU_CB_ATSR 0x8f0	232	#define ARM_SMMU_CB_ATSR 0x8f0
206	#define ATSR_ACTIVE BIT(0)	233	#define ARM_SMMU_ATSR_ACTIVE BIT(0)
207		234
208		235
209	/* Maximum number of context banks per SMMU */	236	/* Maximum number of context banks per SMMU */
@@ -297,7 +324,7 @@ struct arm_smmu_cfg { · 87 unmodified lines
297	enum arm_smmu_cbar_type cbar;	324	enum arm_smmu_cbar_type cbar;
298	enum arm_smmu_context_fmt fmt;	325	enum arm_smmu_context_fmt fmt;
299	};	326	};
300	#define INVALID_IRPTNDX 0xff	327	#define ARM_SMMU_INVALID_IRPTNDX 0xff
301		328
302	enum arm_smmu_domain_stage {	329	enum arm_smmu_domain_stage {
303	ARM_SMMU_DOMAIN_S1 = 0,	330	ARM_SMMU_DOMAIN_S1 = 0,
@@ -318,6 +345,33 @@ struct arm_smmu_domain { · 14 unmodified lines
318	struct iommu_domain domain;	345	struct iommu_domain domain;
319	};	346	};
320		347
		348	static inline u32 arm_smmu_lpae_tcr(struct io_pgtable_cfg *cfg)
		349	{
		350	return ARM_SMMU_TCR_EPD1 \|
		351	FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) \|
		352	FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) \|
		353	FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) \|
		354	FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) \|
		355	FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
		356	}
		357
		358	static inline u32 arm_smmu_lpae_tcr2(struct io_pgtable_cfg *cfg)
		359	{
		360	return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) \|
		361	FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
		362	}
		363
		364	static inline u32 arm_smmu_lpae_vtcr(struct io_pgtable_cfg *cfg)
		365	{
		366	return ARM_SMMU_VTCR_RES1 \|
		367	FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) \|
		368	FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) \|
		369	FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) \|
		370	FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) \|
		371	FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) \|
		372	FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) \|
		373	FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
		374	}
321		375
322	/* Implementation details, yay! */	376	/* Implementation details, yay! */
323	struct arm_smmu_impl {	377	struct arm_smmu_impl {

drivers/iommu/dmar.c+42−2

@@ -244,7 +244,7 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, · 243 unmodified lines
244	info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) \|\|	244	info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) \|\|
245	(scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&	245	(scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
246	(info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&	246	(info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
247	info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {	247	info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) {
248	pr_warn("Device scope type does not match for %s\n",	248	pr_warn("Device scope type does not match for %s\n",
249	pci_name(info->dev));	249	pci_name(info->dev));
250	return -EINVAL;	250	return -EINVAL;
@@ -1354,7 +1354,6 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, · 1103 unmodified lines
1354	struct qi_desc desc;	1354	struct qi_desc desc;
1355		1355
1356	if (mask) {	1356	if (mask) {
1357	WARN_ON_ONCE(addr & ((1ULL << (VTD_PAGE_SHIFT + mask)) - 1));
1358	addr \|= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;	1357	addr \|= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1359	desc.qw1 = QI_DEV_IOTLB_ADDR(addr) \| QI_DEV_IOTLB_SIZE;	1358	desc.qw1 = QI_DEV_IOTLB_ADDR(addr) \| QI_DEV_IOTLB_SIZE;
1360	} else	1359	} else
@@ -1371,6 +1370,47 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, · 10 unmodified lines
1371	qi_submit_sync(&desc, iommu);	1370	qi_submit_sync(&desc, iommu);
1372	}	1371	}
1373		1372
		1373	/* PASID-based IOTLB invalidation */
		1374	void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
		1375	unsigned long npages, bool ih)
		1376	{
		1377	struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
		1378
		1379	/*
		1380	* npages == -1 means a PASID-selective invalidation, otherwise,
		1381	* a positive value for Page-selective-within-PASID invalidation.
		1382	* 0 is not a valid input.
		1383	*/
		1384	if (WARN_ON(!npages)) {
		1385	pr_err("Invalid input npages = %ld\n", npages);
		1386	return;
		1387	}
		1388
		1389	if (npages == -1) {
		1390	desc.qw0 = QI_EIOTLB_PASID(pasid) \|
		1391	QI_EIOTLB_DID(did) \|
		1392	QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) \|
		1393	QI_EIOTLB_TYPE;
		1394	desc.qw1 = 0;
		1395	} else {
		1396	int mask = ilog2(__roundup_pow_of_two(npages));
		1397	unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
		1398
		1399	if (WARN_ON_ONCE(!ALIGN(addr, align)))
		1400	addr &= ~(align - 1);
		1401
		1402	desc.qw0 = QI_EIOTLB_PASID(pasid) \|
		1403	QI_EIOTLB_DID(did) \|
		1404	QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) \|
		1405	QI_EIOTLB_TYPE;
		1406	desc.qw1 = QI_EIOTLB_ADDR(addr) \|
		1407	QI_EIOTLB_IH(ih) \|
		1408	QI_EIOTLB_AM(mask);
		1409	}
		1410
		1411	qi_submit_sync(&desc, iommu);
		1412	}
		1413
1374	/*	1414	/*
1375	* Disable Queued Invalidation interface.	1415	* Disable Queued Invalidation interface.
1376	*/	1416	*/

drivers/iommu/intel-iommu-debugfs.c+75−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	* Authors: Gayatri Kammela <gayatri.kammela@intel.com>	5	* Authors: Gayatri Kammela <gayatri.kammela@intel.com>
6	* Sohil Mehta <sohil.mehta@intel.com>	6	* Sohil Mehta <sohil.mehta@intel.com>
7	* Jacob Pan <jacob.jun.pan@linux.intel.com>	7	* Jacob Pan <jacob.jun.pan@linux.intel.com>
		8	* Lu Baolu <baolu.lu@linux.intel.com>
8	*/	9	*/
9		10
10	#include <linux/debugfs.h>	11	#include <linux/debugfs.h>
@@ -283,6 +284,77 @@ static int dmar_translation_struct_show(struct seq_file m, void unused) · 272 unmodified lines
283	}	284	}
284	DEFINE_SHOW_ATTRIBUTE(dmar_translation_struct);	285	DEFINE_SHOW_ATTRIBUTE(dmar_translation_struct);
285		286
		287	static inline unsigned long level_to_directory_size(int level)
		288	{
		289	return BIT_ULL(VTD_PAGE_SHIFT + VTD_STRIDE_SHIFT * (level - 1));
		290	}
		291
		292	static inline void
		293	dump_page_info(struct seq_file m, unsigned long iova, u64 path)
		294	{
		295	seq_printf(m, "0x%013lx \|\t0x%016llx\t0x%016llx\t0x%016llx\t0x%016llx\t0x%016llx\n",
		296	iova >> VTD_PAGE_SHIFT, path[5], path[4],
		297	path[3], path[2], path[1]);
		298	}
		299
		300	static void pgtable_walk_level(struct seq_file m, struct dma_pte pde,
		301	int level, unsigned long start,
		302	u64 *path)
		303	{
		304	int i;
		305
		306	if (level > 5 \|\| level < 1)
		307	return;
		308
		309	for (i = 0; i < BIT_ULL(VTD_STRIDE_SHIFT);
		310	i++, pde++, start += level_to_directory_size(level)) {
		311	if (!dma_pte_present(pde))
		312	continue;
		313
		314	path[level] = pde->val;
		315	if (dma_pte_superpage(pde) \|\| level == 1)
		316	dump_page_info(m, start, path);
		317	else
		318	pgtable_walk_level(m, phys_to_virt(dma_pte_addr(pde)),
		319	level - 1, start, path);
		320	path[level] = 0;
		321	}
		322	}
		323
		324	static int show_device_domain_translation(struct device dev, void data)
		325	{
		326	struct dmar_domain *domain = find_domain(dev);
		327	struct seq_file *m = data;
		328	u64 path[6] = { 0 };
		329
		330	if (!domain)
		331	return 0;
		332
		333	seq_printf(m, "Device %s with pasid %d @0x%llx\n",
		334	dev_name(dev), domain->default_pasid,
		335	(u64)virt_to_phys(domain->pgd));
		336	seq_puts(m, "IOVA_PFN\t\tPML5E\t\t\tPML4E\t\t\tPDPE\t\t\tPDE\t\t\tPTE\n");
		337
		338	pgtable_walk_level(m, domain->pgd, domain->agaw + 2, 0, path);
		339	seq_putc(m, '\n');
		340
		341	return 0;
		342	}
		343
		344	static int domain_translation_struct_show(struct seq_file m, void unused)
		345	{
		346	unsigned long flags;
		347	int ret;
		348
		349	spin_lock_irqsave(&device_domain_lock, flags);
		350	ret = bus_for_each_dev(&pci_bus_type, NULL, m,
		351	show_device_domain_translation);
		352	spin_unlock_irqrestore(&device_domain_lock, flags);
		353
		354	return ret;
		355	}
		356	DEFINE_SHOW_ATTRIBUTE(domain_translation_struct);
		357
286	#ifdef CONFIG_IRQ_REMAP	358	#ifdef CONFIG_IRQ_REMAP
287	static void ir_tbl_remap_entry_show(struct seq_file *m,	359	static void ir_tbl_remap_entry_show(struct seq_file *m,
288	struct intel_iommu *iommu)	360	struct intel_iommu *iommu)
@@ -396,6 +468,9 @@ void __init intel_iommu_debugfs_init(void) · 107 unmodified lines
396	&iommu_regset_fops);	468	&iommu_regset_fops);
397	debugfs_create_file("dmar_translation_struct", 0444, intel_iommu_debug,	469	debugfs_create_file("dmar_translation_struct", 0444, intel_iommu_debug,
398	NULL, &dmar_translation_struct_fops);	470	NULL, &dmar_translation_struct_fops);
		471	debugfs_create_file("domain_translation_struct", 0444,
		472	intel_iommu_debug, NULL,
		473	&domain_translation_struct_fops);
399	#ifdef CONFIG_IRQ_REMAP	474	#ifdef CONFIG_IRQ_REMAP
400	debugfs_create_file("ir_translation_struct", 0444, intel_iommu_debug,	475	debugfs_create_file("ir_translation_struct", 0444, intel_iommu_debug,
401	NULL, &ir_translation_struct_fops);	476	NULL, &ir_translation_struct_fops);

drivers/iommu/intel-iommu.c+275−91

@@ -307,6 +307,20 @@ static int hw_pass_through = 1; · 306 unmodified lines
307	*/	307	*/
308	#define DOMAIN_FLAG_LOSE_CHILDREN BIT(1)	308	#define DOMAIN_FLAG_LOSE_CHILDREN BIT(1)
309		309
		310	/*
		311	* When VT-d works in the scalable mode, it allows DMA translation to
		312	* happen through either first level or second level page table. This
		313	* bit marks that the DMA translation for the domain goes through the
		314	* first level page table, otherwise, it goes through the second level.
		315	*/
		316	#define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(2)
		317
		318	/*
		319	* Domain represents a virtual machine which demands iommu nested
		320	* translation mode support.
		321	*/
		322	#define DOMAIN_FLAG_NESTING_MODE BIT(3)
		323
310	#define for_each_domain_iommu(idx, domain) \	324	#define for_each_domain_iommu(idx, domain) \
311	for (idx = 0; idx < g_num_of_iommus; idx++) \	325	for (idx = 0; idx < g_num_of_iommus; idx++) \
312	if (domain->iommu_refcnt[idx])	326	if (domain->iommu_refcnt[idx])
@@ -355,9 +369,14 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, · 42 unmodified lines
355	int dmar_disabled = 0;	369	int dmar_disabled = 0;
356	#else	370	#else
357	int dmar_disabled = 1;	371	int dmar_disabled = 1;
358	#endif /CONFIG_INTEL_IOMMU_DEFAULT_ON/	372	#endif /* CONFIG_INTEL_IOMMU_DEFAULT_ON */
359		373
		374	#ifdef INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON
		375	int intel_iommu_sm = 1;
		376	#else
360	int intel_iommu_sm;	377	int intel_iommu_sm;
		378	#endif /* INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON */
		379
361	int intel_iommu_enabled = 0;	380	int intel_iommu_enabled = 0;
362	EXPORT_SYMBOL_GPL(intel_iommu_enabled);	381	EXPORT_SYMBOL_GPL(intel_iommu_enabled);
363		382
@@ -368,7 +387,6 @@ static int intel_iommu_superpage = 1; · 4 unmodified lines
368	static int iommu_identity_mapping;	387	static int iommu_identity_mapping;
369	static int intel_no_bounce;	388	static int intel_no_bounce;
370		389
371	#define IDENTMAP_ALL 1
372	#define IDENTMAP_GFX 2	390	#define IDENTMAP_GFX 2
373	#define IDENTMAP_AZALIA 4	391	#define IDENTMAP_AZALIA 4
374		392
@@ -377,7 +395,7 @@ EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); · 2 unmodified lines
377		395
378	#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))	396	#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
379	#define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))	397	#define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
380	static DEFINE_SPINLOCK(device_domain_lock);	398	DEFINE_SPINLOCK(device_domain_lock);
381	static LIST_HEAD(device_domain_list);	399	static LIST_HEAD(device_domain_list);
382		400
383	#define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \	401	#define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \
@@ -552,6 +570,11 @@ static inline int domain_type_is_si(struct dmar_domain *domain) · 168 unmodified lines
552	return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;	570	return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
553	}	571	}
554		572
		573	static inline bool domain_use_first_level(struct dmar_domain *domain)
		574	{
		575	return domain->flags & DOMAIN_FLAG_USE_FIRST_LEVEL;
		576	}
		577
555	static inline int domain_pfn_supported(struct dmar_domain *domain,	578	static inline int domain_pfn_supported(struct dmar_domain *domain,
556	unsigned long pfn)	579	unsigned long pfn)
557	{	580	{
@@ -661,11 +684,12 @@ static int domain_update_iommu_snooping(struct intel_iommu *skip) · 103 unmodified lines
661	return ret;	684	return ret;
662	}	685	}
663		686
664	static int domain_update_iommu_superpage(struct intel_iommu *skip)	687	static int domain_update_iommu_superpage(struct dmar_domain *domain,
		688	struct intel_iommu *skip)
665	{	689	{
666	struct dmar_drhd_unit *drhd;	690	struct dmar_drhd_unit *drhd;
667	struct intel_iommu *iommu;	691	struct intel_iommu *iommu;
668	int mask = 0xf;	692	int mask = 0x3;
669		693
670	if (!intel_iommu_superpage) {	694	if (!intel_iommu_superpage) {
671	return 0;	695	return 0;
@@ -675,7 +699,13 @@ static int domain_update_iommu_superpage(struct intel_iommu *skip) · 3 unmodified lines
675	rcu_read_lock();	699	rcu_read_lock();
676	for_each_active_iommu(iommu, drhd) {	700	for_each_active_iommu(iommu, drhd) {
677	if (iommu != skip) {	701	if (iommu != skip) {
678	mask &= cap_super_page_val(iommu->cap);	702	if (domain && domain_use_first_level(domain)) {
		703	if (!cap_fl1gp_support(iommu->cap))
		704	mask = 0x1;
		705	} else {
		706	mask &= cap_super_page_val(iommu->cap);
		707	}
		708
679	if (!mask)	709	if (!mask)
680	break;	710	break;
681	}	711	}
@@ -690,7 +720,7 @@ static void domain_update_iommu_cap(struct dmar_domain *domain) · 8 unmodified lines
690	{	720	{
691	domain_update_iommu_coherency(domain);	721	domain_update_iommu_coherency(domain);
692	domain->iommu_snooping = domain_update_iommu_snooping(NULL);	722	domain->iommu_snooping = domain_update_iommu_snooping(NULL);
693	domain->iommu_superpage = domain_update_iommu_superpage(NULL);	723	domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL);
694	}	724	}
695		725
696	struct context_entry iommu_context_addr(struct intel_iommu iommu, u8 bus,	726	struct context_entry iommu_context_addr(struct intel_iommu iommu, u8 bus,
@@ -907,6 +937,8 @@ static struct dma_pte pfn_to_dma_pte(struct dmar_domain domain, · 210 unmodified lines
907		937
908	domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);	938	domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
909	pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) \| DMA_PTE_READ \| DMA_PTE_WRITE;	939	pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) \| DMA_PTE_READ \| DMA_PTE_WRITE;
		940	if (domain_use_first_level(domain))
		941	pteval \|= DMA_FL_PTE_XD;
910	if (cmpxchg64(&pte->val, 0ULL, pteval))	942	if (cmpxchg64(&pte->val, 0ULL, pteval))
911	/* Someone else set it while we were thinking; use theirs. */	943	/* Someone else set it while we were thinking; use theirs. */
912	free_pgtable_page(tmp_page);	944	free_pgtable_page(tmp_page);
@@ -1477,6 +1509,20 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, · 564 unmodified lines
1477	spin_unlock_irqrestore(&device_domain_lock, flags);	1509	spin_unlock_irqrestore(&device_domain_lock, flags);
1478	}	1510	}
1479		1511
		1512	static void domain_flush_piotlb(struct intel_iommu *iommu,
		1513	struct dmar_domain *domain,
		1514	u64 addr, unsigned long npages, bool ih)
		1515	{
		1516	u16 did = domain->iommu_did[iommu->seq_id];
		1517
		1518	if (domain->default_pasid)
		1519	qi_flush_piotlb(iommu, did, domain->default_pasid,
		1520	addr, npages, ih);
		1521
		1522	if (!list_empty(&domain->devices))
		1523	qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, npages, ih);
		1524	}
		1525
1480	static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,	1526	static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1481	struct dmar_domain *domain,	1527	struct dmar_domain *domain,
1482	unsigned long pfn, unsigned int pages,	1528	unsigned long pfn, unsigned int pages,
@@ -1490,18 +1536,23 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, · 7 unmodified lines
1490		1536
1491	if (ih)	1537	if (ih)
1492	ih = 1 << 6;	1538	ih = 1 << 6;
1493	/*	1539
1494	* Fallback to domain selective flush if no PSI support or the size is	1540	if (domain_use_first_level(domain)) {
1495	* too big.	1541	domain_flush_piotlb(iommu, domain, addr, pages, ih);
1496	* PSI requires page size to be 2 ^ x, and the base address is naturally	1542	} else {
1497	* aligned to the size	1543	/*
1498	*/	1544	* Fallback to domain selective flush if no PSI support or
1499	if (!cap_pgsel_inv(iommu->cap) \|\| mask > cap_max_amask_val(iommu->cap))	1545	* the size is too big. PSI requires page size to be 2 ^ x,
1500	iommu->flush.flush_iotlb(iommu, did, 0, 0,	1546	* and the base address is naturally aligned to the size.
1501	DMA_TLB_DSI_FLUSH);	1547	*/
1502	else	1548	if (!cap_pgsel_inv(iommu->cap) \|\|
1503	iommu->flush.flush_iotlb(iommu, did, addr \| ih, mask,	1549	mask > cap_max_amask_val(iommu->cap))
1504	DMA_TLB_PSI_FLUSH);	1550	iommu->flush.flush_iotlb(iommu, did, 0, 0,
		1551	DMA_TLB_DSI_FLUSH);
		1552	else
		1553	iommu->flush.flush_iotlb(iommu, did, addr \| ih, mask,
		1554	DMA_TLB_PSI_FLUSH);
		1555	}
1505		1556
1506	/*	1557	/*
1507	* In caching mode, changes of pages from non-present to present require	1558	* In caching mode, changes of pages from non-present to present require
@@ -1516,8 +1567,11 @@ static inline void __mapping_notify_one(struct intel_iommu *iommu, · 8 unmodified lines
1516	struct dmar_domain *domain,	1567	struct dmar_domain *domain,
1517	unsigned long pfn, unsigned int pages)	1568	unsigned long pfn, unsigned int pages)
1518	{	1569	{
1519	/* It's a non-present to present mapping. Only flush if caching mode */	1570	/*
1520	if (cap_caching_mode(iommu->cap))	1571	* It's a non-present to present mapping. Only flush if caching mode
		1572	* and second level.
		1573	*/
		1574	if (cap_caching_mode(iommu->cap) && !domain_use_first_level(domain))
1521	iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);	1575	iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1522	else	1576	else
1523	iommu_flush_write_buffer(iommu);	1577	iommu_flush_write_buffer(iommu);
@@ -1534,7 +1588,11 @@ static void iommu_flush_iova(struct iova_domain *iovad) · 10 unmodified lines
1534	struct intel_iommu *iommu = g_iommus[idx];	1588	struct intel_iommu *iommu = g_iommus[idx];
1535	u16 did = domain->iommu_did[iommu->seq_id];	1589	u16 did = domain->iommu_did[iommu->seq_id];
1536		1590
1537	iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);	1591	if (domain_use_first_level(domain))
		1592	domain_flush_piotlb(iommu, domain, 0, -1, 0);
		1593	else
		1594	iommu->flush.flush_iotlb(iommu, did, 0, 0,
		1595	DMA_TLB_DSI_FLUSH);
1538		1596
1539	if (!cap_caching_mode(iommu->cap))	1597	if (!cap_caching_mode(iommu->cap))
1540	iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),	1598	iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
@@ -1703,6 +1761,33 @@ static void free_dmar_iommu(struct intel_iommu *iommu) · 162 unmodified lines
1703	#endif	1761	#endif
1704	}	1762	}
1705		1763
		1764	/*
		1765	* Check and return whether first level is used by default for
		1766	* DMA translation.
		1767	*/
		1768	static bool first_level_by_default(void)
		1769	{
		1770	struct dmar_drhd_unit *drhd;
		1771	struct intel_iommu *iommu;
		1772	static int first_level_support = -1;
		1773
		1774	if (likely(first_level_support != -1))
		1775	return first_level_support;
		1776
		1777	first_level_support = 1;
		1778
		1779	rcu_read_lock();
		1780	for_each_active_iommu(iommu, drhd) {
		1781	if (!sm_supported(iommu) \|\| !ecap_flts(iommu->ecap)) {
		1782	first_level_support = 0;
		1783	break;
		1784	}
		1785	}
		1786	rcu_read_unlock();
		1787
		1788	return first_level_support;
		1789	}
		1790
1706	static struct dmar_domain *alloc_domain(int flags)	1791	static struct dmar_domain *alloc_domain(int flags)
1707	{	1792	{
1708	struct dmar_domain *domain;	1793	struct dmar_domain *domain;
@@ -1714,6 +1799,8 @@ static struct dmar_domain *alloc_domain(int flags) · 5 unmodified lines
1714	memset(domain, 0, sizeof(*domain));	1799	memset(domain, 0, sizeof(*domain));
1715	domain->nid = NUMA_NO_NODE;	1800	domain->nid = NUMA_NO_NODE;
1716	domain->flags = flags;	1801	domain->flags = flags;
		1802	if (first_level_by_default())
		1803	domain->flags \|= DOMAIN_FLAG_USE_FIRST_LEVEL;
1717	domain->has_iotlb_device = false;	1804	domain->has_iotlb_device = false;
1718	INIT_LIST_HEAD(&domain->devices);	1805	INIT_LIST_HEAD(&domain->devices);
1719		1806
@@ -1843,14 +1930,16 @@ static int domain_init(struct dmar_domain domain, struct intel_iommu iommu, · 123 unmodified lines
1843	{	1930	{
1844	int adjust_width, agaw;	1931	int adjust_width, agaw;
1845	unsigned long sagaw;	1932	unsigned long sagaw;
1846	int err;	1933	int ret;
1847		1934
1848	init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);	1935	init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1849		1936
1850	err = init_iova_flush_queue(&domain->iovad,	1937	if (!intel_iommu_strict) {
1851	iommu_flush_iova, iova_entry_free);	1938	ret = init_iova_flush_queue(&domain->iovad,
1852	if (err)	1939	iommu_flush_iova, iova_entry_free);
1853	return err;	1940	if (ret)
		1941	pr_info("iova flush queue initialization failed\n");
		1942	}
1854		1943
1855	domain_reserve_special_ranges(domain);	1944	domain_reserve_special_ranges(domain);
1856		1945
@@ -2223,17 +2312,20 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, · 366 unmodified lines
2223	unsigned long sg_res = 0;	2312	unsigned long sg_res = 0;
2224	unsigned int largepage_lvl = 0;	2313	unsigned int largepage_lvl = 0;
2225	unsigned long lvl_pages = 0;	2314	unsigned long lvl_pages = 0;
		2315	u64 attr;
2226		2316
2227	BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));	2317	BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2228		2318
2229	if ((prot & (DMA_PTE_READ\|DMA_PTE_WRITE)) == 0)	2319	if ((prot & (DMA_PTE_READ\|DMA_PTE_WRITE)) == 0)
2230	return -EINVAL;	2320	return -EINVAL;
2231		2321
2232	prot &= DMA_PTE_READ \| DMA_PTE_WRITE \| DMA_PTE_SNP;	2322	attr = prot & (DMA_PTE_READ \| DMA_PTE_WRITE \| DMA_PTE_SNP);
		2323	if (domain_use_first_level(domain))
		2324	attr \|= DMA_FL_PTE_PRESENT \| DMA_FL_PTE_XD;
2233		2325
2234	if (!sg) {	2326	if (!sg) {
2235	sg_res = nr_pages;	2327	sg_res = nr_pages;
2236	pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) \| prot;	2328	pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) \| attr;
2237	}	2329	}
2238		2330
2239	while (nr_pages > 0) {	2331	while (nr_pages > 0) {
@@ -2245,7 +2337,7 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, · 5 unmodified lines
2245	sg_res = aligned_nrpages(sg->offset, sg->length);	2337	sg_res = aligned_nrpages(sg->offset, sg->length);
2246	sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;	2338	sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2247	sg->dma_length = sg->length;	2339	sg->dma_length = sg->length;
2248	pteval = (sg_phys(sg) - pgoff) \| prot;	2340	pteval = (sg_phys(sg) - pgoff) \| attr;
2249	phys_pfn = pteval >> VTD_PAGE_SHIFT;	2341	phys_pfn = pteval >> VTD_PAGE_SHIFT;
2250	}	2342	}
2251		2343
@@ -2414,7 +2506,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain) · 162 unmodified lines
2414	spin_unlock_irqrestore(&device_domain_lock, flags);	2506	spin_unlock_irqrestore(&device_domain_lock, flags);
2415	}	2507	}
2416		2508
2417	static struct dmar_domain find_domain(struct device dev)	2509	struct dmar_domain find_domain(struct device dev)
2418	{	2510	{
2419	struct device_domain_info *info;	2511	struct device_domain_info *info;
2420		2512
@@ -2460,6 +2552,36 @@ dmar_search_domain_by_dev_info(int segment, int bus, int devfn) · 39 unmodified lines
2460	return NULL;	2552	return NULL;
2461	}	2553	}
2462		2554
		2555	static int domain_setup_first_level(struct intel_iommu *iommu,
		2556	struct dmar_domain *domain,
		2557	struct device *dev,
		2558	int pasid)
		2559	{
		2560	int flags = PASID_FLAG_SUPERVISOR_MODE;
		2561	struct dma_pte *pgd = domain->pgd;
		2562	int agaw, level;
		2563
		2564	/*
		2565	* Skip top levels of page tables for iommu which has
		2566	* less agaw than default. Unnecessary for PT mode.
		2567	*/
		2568	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
		2569	pgd = phys_to_virt(dma_pte_addr(pgd));
		2570	if (!dma_pte_present(pgd))
		2571	return -ENOMEM;
		2572	}
		2573
		2574	level = agaw_to_level(agaw);
		2575	if (level != 4 && level != 5)
		2576	return -EINVAL;
		2577
		2578	flags \|= (level == 5) ? PASID_FLAG_FL5LP : 0;
		2579
		2580	return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid,
		2581	domain->iommu_did[iommu->seq_id],
		2582	flags);
		2583	}
		2584
2463	static struct dmar_domain dmar_insert_one_dev_info(struct intel_iommu iommu,	2585	static struct dmar_domain dmar_insert_one_dev_info(struct intel_iommu iommu,
2464	int bus, int devfn,	2586	int bus, int devfn,
2465	struct device *dev,	2587	struct device *dev,
@@ -2559,6 +2681,9 @@ static struct dmar_domain dmar_insert_one_dev_info(struct intel_iommu iommu, · 93 unmodified lines
2559	if (hw_pass_through && domain_type_is_si(domain))	2681	if (hw_pass_through && domain_type_is_si(domain))
2560	ret = intel_pasid_setup_pass_through(iommu, domain,	2682	ret = intel_pasid_setup_pass_through(iommu, domain,
2561	dev, PASID_RID2PASID);	2683	dev, PASID_RID2PASID);
		2684	else if (domain_use_first_level(domain))
		2685	ret = domain_setup_first_level(iommu, domain, dev,
		2686	PASID_RID2PASID);
2562	else	2687	else
2563	ret = intel_pasid_setup_second_level(iommu, domain,	2688	ret = intel_pasid_setup_second_level(iommu, domain,
2564	dev, PASID_RID2PASID);	2689	dev, PASID_RID2PASID);
@@ -2764,10 +2889,8 @@ static int __init si_domain_init(int hw) · 199 unmodified lines
2764	}	2889	}
2765		2890
2766	/*	2891	/*
2767	* Normally we use DMA domains for devices which have RMRRs. But we	2892	* Identity map the RMRRs so that devices with RMRRs could also use
2768	* loose this requirement for graphic and usb devices. Identity map	2893	* the si_domain.
2769	* the RMRRs for graphic and USB devices so that they could use the
2770	* si_domain.
2771	*/	2894	*/
2772	for_each_rmrr_units(rmrr) {	2895	for_each_rmrr_units(rmrr) {
2773	for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,	2896	for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
@@ -2775,9 +2898,6 @@ static int __init si_domain_init(int hw) · 1 unmodified line
2775	unsigned long long start = rmrr->base_address;	2898	unsigned long long start = rmrr->base_address;
2776	unsigned long long end = rmrr->end_address;	2899	unsigned long long end = rmrr->end_address;
2777		2900
2778	if (device_is_rmrr_locked(dev))
2779	continue;
2780
2781	if (WARN_ON(end < start \|\|	2901	if (WARN_ON(end < start \|\|
2782	end >> agaw_to_width(si_domain->agaw)))	2902	end >> agaw_to_width(si_domain->agaw)))
2783	continue;	2903	continue;
@@ -2916,9 +3036,6 @@ static int device_def_domain_type(struct device *dev) · 132 unmodified lines
2916	if (dev_is_pci(dev)) {	3036	if (dev_is_pci(dev)) {
2917	struct pci_dev *pdev = to_pci_dev(dev);	3037	struct pci_dev *pdev = to_pci_dev(dev);
2918		3038
2919	if (device_is_rmrr_locked(dev))
2920	return IOMMU_DOMAIN_DMA;
2921
2922	/*	3039	/*
2923	* Prevent any device marked as untrusted from getting	3040	* Prevent any device marked as untrusted from getting
2924	* placed into the statically identity mapping domain.	3041	* placed into the statically identity mapping domain.
@@ -2956,13 +3073,9 @@ static int device_def_domain_type(struct device *dev) · 31 unmodified lines
2956	return IOMMU_DOMAIN_DMA;	3073	return IOMMU_DOMAIN_DMA;
2957	} else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)	3074	} else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2958	return IOMMU_DOMAIN_DMA;	3075	return IOMMU_DOMAIN_DMA;
2959	} else {
2960	if (device_has_rmrr(dev))
2961	return IOMMU_DOMAIN_DMA;
2962	}	3076	}
2963		3077
2964	return (iommu_identity_mapping & IDENTMAP_ALL) ?	3078	return 0;
2965	IOMMU_DOMAIN_IDENTITY : 0;
2966	}	3079	}
2967		3080
2968	static void intel_iommu_init_qi(struct intel_iommu *iommu)	3081	static void intel_iommu_init_qi(struct intel_iommu *iommu)
@@ -3291,10 +3404,7 @@ static int __init init_dmars(void) · 322 unmodified lines
3291		3404
3292	if (!ecap_pass_through(iommu->ecap))	3405	if (!ecap_pass_through(iommu->ecap))
3293	hw_pass_through = 0;	3406	hw_pass_through = 0;
3294	#ifdef CONFIG_INTEL_IOMMU_SVM	3407	intel_svm_check(iommu);
3295	if (pasid_supported(iommu))
3296	intel_svm_init(iommu);
3297	#endif
3298	}	3408	}
3299		3409
3300	/*	3410	/*
@@ -3309,9 +3419,6 @@ static int __init init_dmars(void) · 8 unmodified lines
3309	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);	3419	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3310	}	3420	}
3311		3421
3312	if (iommu_default_passthrough())
3313	iommu_identity_mapping \|= IDENTMAP_ALL;
3314
3315	#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA	3422	#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3316	dmar_map_gfx = 0;	3423	dmar_map_gfx = 0;
3317	#endif	3424	#endif
@@ -3384,8 +3491,21 @@ static unsigned long intel_alloc_iova(struct device *dev, · 66 unmodified lines
3384	{	3491	{
3385	unsigned long iova_pfn;	3492	unsigned long iova_pfn;
3386		3493
3387	/* Restrict dma_mask to the width that the iommu can handle */	3494	/*
3388	dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);	3495	* Restrict dma_mask to the width that the iommu can handle.
		3496	* First-level translation restricts the input-address to a
		3497	* canonical address (i.e., address bits 63:N have the same
		3498	* value as address bit [N-1], where N is 48-bits with 4-level
		3499	* paging and 57-bits with 5-level paging). Hence, skip bit
		3500	* [N-1].
		3501	*/
		3502	if (domain_use_first_level(domain))
		3503	dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw - 1),
		3504	dma_mask);
		3505	else
		3506	dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw),
		3507	dma_mask);
		3508
3389	/* Ensure we reserve the whole size-aligned region */	3509	/* Ensure we reserve the whole size-aligned region */
3390	nrpages = __roundup_pow_of_two(nrpages);	3510	nrpages = __roundup_pow_of_two(nrpages);
3391		3511
@@ -3403,7 +3523,8 @@ static unsigned long intel_alloc_iova(struct device *dev, · 11 unmodified lines
3403	iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,	3523	iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3404	IOVA_PFN(dma_mask), true);	3524	IOVA_PFN(dma_mask), true);
3405	if (unlikely(!iova_pfn)) {	3525	if (unlikely(!iova_pfn)) {
3406	dev_err(dev, "Allocating %ld-page iova failed", nrpages);	3526	dev_err_once(dev, "Allocating %ld-page iova failed\n",
		3527	nrpages);
3407	return 0;	3528	return 0;
3408	}	3529	}
3409		3530
@@ -3771,8 +3892,8 @@ static int intel_map_sg(struct device dev, struct scatterlist sglist, int nele · 361 unmodified lines
3771	return 0;	3892	return 0;
3772	}	3893	}
3773		3894
3774	trace_map_sg(dev, iova_pfn << PAGE_SHIFT,	3895	for_each_sg(sglist, sg, nelems, i)
3775	sg_phys(sglist), size << VTD_PAGE_SHIFT);	3896	trace_map_sg(dev, i + 1, nelems, sg);
3776		3897
3777	return nelems;	3898	return nelems;
3778	}	3899	}
@@ -3984,6 +4105,9 @@ bounce_map_sg(struct device dev, struct scatterlist sglist, int nelems, · 205 unmodified lines
3984	sg_dma_len(sg) = sg->length;	4105	sg_dma_len(sg) = sg->length;
3985	}	4106	}
3986		4107
		4108	for_each_sg(sglist, sg, nelems, i)
		4109	trace_bounce_map_sg(dev, i + 1, nelems, sg);
		4110
3987	return nelems;	4111	return nelems;
3988		4112
3989	out_unmap:	4113	out_unmap:
@@ -4312,16 +4436,31 @@ static void __init init_iommu_pm_ops(void) · 322 unmodified lines
4312	static inline void init_iommu_pm_ops(void) {}	4436	static inline void init_iommu_pm_ops(void) {}
4313	#endif /* CONFIG_PM */	4437	#endif /* CONFIG_PM */
4314		4438
		4439	static int rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
		4440	{
		4441	if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) \|\|
		4442	!IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) \|\|
		4443	rmrr->end_address <= rmrr->base_address \|\|
		4444	arch_rmrr_sanity_check(rmrr))
		4445	return -EINVAL;
		4446
		4447	return 0;
		4448	}
		4449
4315	int __init dmar_parse_one_rmrr(struct acpi_dmar_header header, void arg)	4450	int __init dmar_parse_one_rmrr(struct acpi_dmar_header header, void arg)
4316	{	4451	{
4317	struct acpi_dmar_reserved_memory *rmrr;	4452	struct acpi_dmar_reserved_memory *rmrr;
4318	struct dmar_rmrr_unit *rmrru;	4453	struct dmar_rmrr_unit *rmrru;
4319	int ret;
4320		4454
4321	rmrr = (struct acpi_dmar_reserved_memory *)header;	4455	rmrr = (struct acpi_dmar_reserved_memory *)header;
4322	ret = arch_rmrr_sanity_check(rmrr);	4456	if (rmrr_sanity_check(rmrr))
4323	if (ret)	4457	WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
4324	return ret;	4458	"Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
		4459	"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
		4460	rmrr->base_address, rmrr->end_address,
		4461	dmi_get_system_info(DMI_BIOS_VENDOR),
		4462	dmi_get_system_info(DMI_BIOS_VERSION),
		4463	dmi_get_system_info(DMI_PRODUCT_VERSION));
4325		4464
4326	rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);	4465	rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4327	if (!rmrru)	4466	if (!rmrru)
@@ -4467,7 +4606,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) · 139 unmodified lines
4467	iommu->name);	4606	iommu->name);
4468	return -ENXIO;	4607	return -ENXIO;
4469	}	4608	}
4470	sp = domain_update_iommu_superpage(iommu) - 1;	4609	sp = domain_update_iommu_superpage(NULL, iommu) - 1;
4471	if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {	4610	if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4472	pr_warn("%s: Doesn't support large page.\n",	4611	pr_warn("%s: Doesn't support large page.\n",
4473	iommu->name);	4612	iommu->name);
@@ -4487,10 +4626,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) · 13 unmodified lines
4487	if (ret)	4626	if (ret)
4488	goto out;	4627	goto out;
4489		4628
4490	#ifdef CONFIG_INTEL_IOMMU_SVM	4629	intel_svm_check(iommu);
4491	if (pasid_supported(iommu))
4492	intel_svm_init(iommu);
4493	#endif
4494		4630
4495	if (dmaru->ignored) {	4631	if (dmaru->ignored) {
4496	/*	4632	/*
@@ -4895,7 +5031,7 @@ static int __init platform_optin_force_iommu(void) · 398 unmodified lines
4895	* map for all devices except those marked as being untrusted.	5031	* map for all devices except those marked as being untrusted.
4896	*/	5032	*/
4897	if (dmar_disabled)	5033	if (dmar_disabled)
4898	iommu_identity_mapping \|= IDENTMAP_ALL;	5034	iommu_set_default_passthrough(false);
4899		5035
4900	dmar_disabled = 0;	5036	dmar_disabled = 0;
4901	no_iommu = 0;	5037	no_iommu = 0;
@@ -5195,6 +5331,7 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) · 293 unmodified lines
5195	{	5331	{
5196	struct dmar_domain *dmar_domain;	5332	struct dmar_domain *dmar_domain;
5197	struct iommu_domain *domain;	5333	struct iommu_domain *domain;
		5334	int ret;
5198		5335
5199	switch (type) {	5336	switch (type) {
5200	case IOMMU_DOMAIN_DMA:	5337	case IOMMU_DOMAIN_DMA:
@@ -5211,11 +5348,12 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) · 10 unmodified lines
5211	return NULL;	5348	return NULL;
5212	}	5349	}
5213		5350
5214	if (type == IOMMU_DOMAIN_DMA &&	5351	if (!intel_iommu_strict && type == IOMMU_DOMAIN_DMA) {
5215	init_iova_flush_queue(&dmar_domain->iovad,	5352	ret = init_iova_flush_queue(&dmar_domain->iovad,
5216	iommu_flush_iova, iova_entry_free)) {	5353	iommu_flush_iova,
5217	pr_warn("iova flush queue initialization failed\n");	5354	iova_entry_free);
5218	intel_iommu_strict = 1;	5355	if (ret)
		5356	pr_info("iova flush queue initialization failed\n");
5219	}	5357	}
5220		5358
5221	domain_update_iommu_cap(dmar_domain);	5359	domain_update_iommu_cap(dmar_domain);
@@ -5281,7 +5419,7 @@ static void auxiliary_unlink_device(struct dmar_domain *domain, · 59 unmodified lines
5281	domain->auxd_refcnt--;	5419	domain->auxd_refcnt--;
5282		5420
5283	if (!domain->auxd_refcnt && domain->default_pasid > 0)	5421	if (!domain->auxd_refcnt && domain->default_pasid > 0)
5284	intel_pasid_free_id(domain->default_pasid);	5422	ioasid_free(domain->default_pasid);
5285	}	5423	}
5286		5424
5287	static int aux_domain_add_dev(struct dmar_domain *domain,	5425	static int aux_domain_add_dev(struct dmar_domain *domain,
@@ -5299,10 +5437,11 @@ static int aux_domain_add_dev(struct dmar_domain *domain, · 11 unmodified lines
5299	if (domain->default_pasid <= 0) {	5437	if (domain->default_pasid <= 0) {
5300	int pasid;	5438	int pasid;
5301		5439
5302	pasid = intel_pasid_alloc_id(domain, PASID_MIN,	5440	/* No private data needed for the default pasid */
5303	pci_max_pasids(to_pci_dev(dev)),	5441	pasid = ioasid_alloc(NULL, PASID_MIN,
5304	GFP_KERNEL);	5442	pci_max_pasids(to_pci_dev(dev)) - 1,
5305	if (pasid <= 0) {	5443	NULL);
		5444	if (pasid == INVALID_IOASID) {
5306	pr_err("Can't allocate default pasid\n");	5445	pr_err("Can't allocate default pasid\n");
5307	return -ENODEV;	5446	return -ENODEV;
5308	}	5447	}
@@ -5320,8 +5459,12 @@ static int aux_domain_add_dev(struct dmar_domain *domain, · 11 unmodified lines
5320	goto attach_failed;	5459	goto attach_failed;
5321		5460
5322	/* Setup the PASID entry for mediated devices: */	5461	/* Setup the PASID entry for mediated devices: */
5323	ret = intel_pasid_setup_second_level(iommu, domain, dev,	5462	if (domain_use_first_level(domain))
5324	domain->default_pasid);	5463	ret = domain_setup_first_level(iommu, domain, dev,
		5464	domain->default_pasid);
		5465	else
		5466	ret = intel_pasid_setup_second_level(iommu, domain, dev,
		5467	domain->default_pasid);
5325	if (ret)	5468	if (ret)
5326	goto table_failed;	5469	goto table_failed;
5327	spin_unlock(&iommu->lock);	5470	spin_unlock(&iommu->lock);
@@ -5338,7 +5481,7 @@ attach_failed: · 10 unmodified lines
5338	spin_unlock(&iommu->lock);	5481	spin_unlock(&iommu->lock);
5339	spin_unlock_irqrestore(&device_domain_lock, flags);	5482	spin_unlock_irqrestore(&device_domain_lock, flags);
5340	if (!domain->auxd_refcnt && domain->default_pasid > 0)	5483	if (!domain->auxd_refcnt && domain->default_pasid > 0)
5341	intel_pasid_free_id(domain->default_pasid);	5484	ioasid_free(domain->default_pasid);
5342		5485
5343	return ret;	5486	return ret;
5344	}	5487	}
@@ -5592,6 +5735,24 @@ static inline bool iommu_pasid_support(void) · 247 unmodified lines
5592	return ret;	5735	return ret;
5593	}	5736	}
5594		5737
		5738	static inline bool nested_mode_support(void)
		5739	{
		5740	struct dmar_drhd_unit *drhd;
		5741	struct intel_iommu *iommu;
		5742	bool ret = true;
		5743
		5744	rcu_read_lock();
		5745	for_each_active_iommu(iommu, drhd) {
		5746	if (!sm_supported(iommu) \|\| !ecap_nest(iommu->ecap)) {
		5747	ret = false;
		5748	break;
		5749	}
		5750	}
		5751	rcu_read_unlock();
		5752
		5753	return ret;
		5754	}
		5755
5595	static bool intel_iommu_capable(enum iommu_cap cap)	5756	static bool intel_iommu_capable(enum iommu_cap cap)
5596	{	5757	{
5597	if (cap == IOMMU_CAP_CACHE_COHERENCY)	5758	if (cap == IOMMU_CAP_CACHE_COHERENCY)
@@ -5749,15 +5910,6 @@ static void intel_iommu_get_resv_regions(struct device *device, · 151 unmodified lines
5749	list_add_tail(&reg->list, head);	5910	list_add_tail(&reg->list, head);
5750	}	5911	}
5751		5912
5752	static void intel_iommu_put_resv_regions(struct device *dev,
5753	struct list_head *head)
5754	{
5755	struct iommu_resv_region entry, next;
5756
5757	list_for_each_entry_safe(entry, next, head, list)
5758	kfree(entry);
5759	}
5760
5761	int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev)	5913	int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev)
5762	{	5914	{
5763	struct device_domain_info *info;	5915	struct device_domain_info *info;
@@ -5984,10 +6136,42 @@ static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain, · 220 unmodified lines
5984	return dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO;	6136	return dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO;
5985	}	6137	}
5986		6138
		6139	static int
		6140	intel_iommu_domain_set_attr(struct iommu_domain *domain,
		6141	enum iommu_attr attr, void *data)
		6142	{
		6143	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
		6144	unsigned long flags;
		6145	int ret = 0;
		6146
		6147	if (domain->type != IOMMU_DOMAIN_UNMANAGED)
		6148	return -EINVAL;
		6149
		6150	switch (attr) {
		6151	case DOMAIN_ATTR_NESTING:
		6152	spin_lock_irqsave(&device_domain_lock, flags);
		6153	if (nested_mode_support() &&
		6154	list_empty(&dmar_domain->devices)) {
		6155	dmar_domain->flags \|= DOMAIN_FLAG_NESTING_MODE;
		6156	dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL;
		6157	} else {
		6158	ret = -ENODEV;
		6159	}
		6160	spin_unlock_irqrestore(&device_domain_lock, flags);
		6161	break;
		6162	default:
		6163	ret = -EINVAL;
		6164	break;
		6165	}
		6166
		6167	return ret;
		6168	}
		6169
5987	const struct iommu_ops intel_iommu_ops = {	6170	const struct iommu_ops intel_iommu_ops = {
5988	.capable = intel_iommu_capable,	6171	.capable = intel_iommu_capable,
5989	.domain_alloc = intel_iommu_domain_alloc,	6172	.domain_alloc = intel_iommu_domain_alloc,
5990	.domain_free = intel_iommu_domain_free,	6173	.domain_free = intel_iommu_domain_free,
		6174	.domain_set_attr = intel_iommu_domain_set_attr,
5991	.attach_dev = intel_iommu_attach_device,	6175	.attach_dev = intel_iommu_attach_device,
5992	.detach_dev = intel_iommu_detach_device,	6176	.detach_dev = intel_iommu_detach_device,
5993	.aux_attach_dev = intel_iommu_aux_attach_device,	6177	.aux_attach_dev = intel_iommu_aux_attach_device,
@@ -5999,7 +6183,7 @@ const struct iommu_ops intel_iommu_ops = { · 5 unmodified lines
5999	.add_device = intel_iommu_add_device,	6183	.add_device = intel_iommu_add_device,
6000	.remove_device = intel_iommu_remove_device,	6184	.remove_device = intel_iommu_remove_device,
6001	.get_resv_regions = intel_iommu_get_resv_regions,	6185	.get_resv_regions = intel_iommu_get_resv_regions,
6002	.put_resv_regions = intel_iommu_put_resv_regions,	6186	.put_resv_regions = generic_iommu_put_resv_regions,
6003	.apply_resv_region = intel_iommu_apply_resv_region,	6187	.apply_resv_region = intel_iommu_apply_resv_region,
6004	.device_group = intel_iommu_device_group,	6188	.device_group = intel_iommu_device_group,
6005	.dev_has_feat = intel_iommu_dev_has_feat,	6189	.dev_has_feat = intel_iommu_dev_has_feat,

drivers/iommu/intel-pasid.c+27−70

@@ -26,42 +26,6 @@ · 25 unmodified lines
26	*/	26	*/
27	static DEFINE_SPINLOCK(pasid_lock);	27	static DEFINE_SPINLOCK(pasid_lock);
28	u32 intel_pasid_max_id = PASID_MAX;	28	u32 intel_pasid_max_id = PASID_MAX;
29	static DEFINE_IDR(pasid_idr);
30
31	int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp)
32	{
33	int ret, min, max;
34
35	min = max_t(int, start, PASID_MIN);
36	max = min_t(int, end, intel_pasid_max_id);
37
38	WARN_ON(in_interrupt());
39	idr_preload(gfp);
40	spin_lock(&pasid_lock);
41	ret = idr_alloc(&pasid_idr, ptr, min, max, GFP_ATOMIC);
42	spin_unlock(&pasid_lock);
43	idr_preload_end();
44
45	return ret;
46	}
47
48	void intel_pasid_free_id(int pasid)
49	{
50	spin_lock(&pasid_lock);
51	idr_remove(&pasid_idr, pasid);
52	spin_unlock(&pasid_lock);
53	}
54
55	void *intel_pasid_lookup_id(int pasid)
56	{
57	void *p;
58
59	spin_lock(&pasid_lock);
60	p = idr_find(&pasid_idr, pasid);
61	spin_unlock(&pasid_lock);
62
63	return p;
64	}
65		29
66	/*	30	/*
67	* Per device pasid table management:	31	* Per device pasid table management:
@@ -465,6 +429,21 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, · 397 unmodified lines
465	devtlb_invalidation_with_pasid(iommu, dev, pasid);	429	devtlb_invalidation_with_pasid(iommu, dev, pasid);
466	}	430	}
467		431
		432	static void pasid_flush_caches(struct intel_iommu *iommu,
		433	struct pasid_entry *pte,
		434	int pasid, u16 did)
		435	{
		436	if (!ecap_coherent(iommu->ecap))
		437	clflush_cache_range(pte, sizeof(*pte));
		438
		439	if (cap_caching_mode(iommu->cap)) {
		440	pasid_cache_invalidation_with_pasid(iommu, did, pasid);
		441	iotlb_invalidation_with_pasid(iommu, did, pasid);
		442	} else {
		443	iommu_flush_write_buffer(iommu);
		444	}
		445	}
		446
468	/*	447	/*
469	* Set up the scalable mode pasid table entry for first only	448	* Set up the scalable mode pasid table entry for first only
470	* translation type.	449	* translation type.
@@ -498,10 +477,15 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, · 27 unmodified lines
498	pasid_set_sre(pte);	477	pasid_set_sre(pte);
499	}	478	}
500		479
501	#ifdef CONFIG_X86	480	if (flags & PASID_FLAG_FL5LP) {
502	if (cpu_feature_enabled(X86_FEATURE_LA57))	481	if (cap_5lp_support(iommu->cap)) {
503	pasid_set_flpm(pte, 1);	482	pasid_set_flpm(pte, 1);
504	#endif /* CONFIG_X86 */	483	} else {
		484	pr_err("No 5-level paging support for first-level\n");
		485	pasid_clear_entry(pte);
		486	return -EINVAL;
		487	}
		488	}
505		489
506	pasid_set_domain_id(pte, did);	490	pasid_set_domain_id(pte, did);
507	pasid_set_address_width(pte, iommu->agaw);	491	pasid_set_address_width(pte, iommu->agaw);
@@ -510,16 +494,7 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, · 2 unmodified lines
510	/* Setup Present and PASID Granular Transfer Type: */	494	/* Setup Present and PASID Granular Transfer Type: */
511	pasid_set_translation_type(pte, 1);	495	pasid_set_translation_type(pte, 1);
512	pasid_set_present(pte);	496	pasid_set_present(pte);
513		497	pasid_flush_caches(iommu, pte, pasid, did);
514	if (!ecap_coherent(iommu->ecap))
515	clflush_cache_range(pte, sizeof(*pte));
516
517	if (cap_caching_mode(iommu->cap)) {
518	pasid_cache_invalidation_with_pasid(iommu, did, pasid);
519	iotlb_invalidation_with_pasid(iommu, did, pasid);
520	} else {
521	iommu_flush_write_buffer(iommu);
522	}
523		498
524	return 0;	499	return 0;
525	}	500	}
@@ -583,16 +558,7 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, · 57 unmodified lines
583	*/	558	*/
584	pasid_set_sre(pte);	559	pasid_set_sre(pte);
585	pasid_set_present(pte);	560	pasid_set_present(pte);
586		561	pasid_flush_caches(iommu, pte, pasid, did);
587	if (!ecap_coherent(iommu->ecap))
588	clflush_cache_range(pte, sizeof(*pte));
589
590	if (cap_caching_mode(iommu->cap)) {
591	pasid_cache_invalidation_with_pasid(iommu, did, pasid);
592	iotlb_invalidation_with_pasid(iommu, did, pasid);
593	} else {
594	iommu_flush_write_buffer(iommu);
595	}
596		562
597	return 0;	563	return 0;
598	}	564	}
@@ -626,16 +592,7 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, · 27 unmodified lines
626	*/	592	*/
627	pasid_set_sre(pte);	593	pasid_set_sre(pte);
628	pasid_set_present(pte);	594	pasid_set_present(pte);
629		595	pasid_flush_caches(iommu, pte, pasid, did);
630	if (!ecap_coherent(iommu->ecap))
631	clflush_cache_range(pte, sizeof(*pte));
632
633	if (cap_caching_mode(iommu->cap)) {
634	pasid_cache_invalidation_with_pasid(iommu, did, pasid);
635	iotlb_invalidation_with_pasid(iommu, did, pasid);
636	} else {
637	iommu_flush_write_buffer(iommu);
638	}
639		596
640	return 0;	597	return 0;
641	}	598	}

drivers/iommu/intel-pasid.h+6−0

  */
 #define PASID_FLAG_SUPERVISOR_MODE	BIT(0)
+/*
+ * The PASID_FLAG_FL5LP flag Indicates using 5-level paging for first-
+ * level translation, otherwise, 4-level paging will be used.
+ */
+#define PASID_FLAG_FL5LP		BIT(1)
 struct pasid_dir_entry {
 	u64 val;
 };

drivers/iommu/intel-svm.c+99−72

@@ -17,25 +17,13 @@ · 16 unmodified lines
17	#include <linux/dmar.h>	17	#include <linux/dmar.h>
18	#include <linux/interrupt.h>	18	#include <linux/interrupt.h>
19	#include <linux/mm_types.h>	19	#include <linux/mm_types.h>
		20	#include <linux/ioasid.h>
20	#include <asm/page.h>	21	#include <asm/page.h>
21		22
22	#include "intel-pasid.h"	23	#include "intel-pasid.h"
23		24
24	static irqreturn_t prq_event_thread(int irq, void *d);	25	static irqreturn_t prq_event_thread(int irq, void *d);
25		26
26	int intel_svm_init(struct intel_iommu *iommu)
27	{
28	if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
29	!cap_fl1gp_support(iommu->cap))
30	return -EINVAL;
31
32	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
33	!cap_5lp_support(iommu->cap))
34	return -EINVAL;
35
36	return 0;
37	}
38
39	#define PRQ_ORDER 0	27	#define PRQ_ORDER 0
40		28
41	int intel_svm_enable_prq(struct intel_iommu *iommu)	29	int intel_svm_enable_prq(struct intel_iommu *iommu)
@@ -99,6 +87,33 @@ int intel_svm_finish_prq(struct intel_iommu *iommu) · 57 unmodified lines
99	return 0;	87	return 0;
100	}	88	}
101		89
		90	static inline bool intel_svm_capable(struct intel_iommu *iommu)
		91	{
		92	return iommu->flags & VTD_FLAG_SVM_CAPABLE;
		93	}
		94
		95	void intel_svm_check(struct intel_iommu *iommu)
		96	{
		97	if (!pasid_supported(iommu))
		98	return;
		99
		100	if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
		101	!cap_fl1gp_support(iommu->cap)) {
		102	pr_err("%s SVM disabled, incompatible 1GB page capability\n",
		103	iommu->name);
		104	return;
		105	}
		106
		107	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
		108	!cap_5lp_support(iommu->cap)) {
		109	pr_err("%s SVM disabled, incompatible paging mode\n",
		110	iommu->name);
		111	return;
		112	}
		113
		114	iommu->flags \|= VTD_FLAG_SVM_CAPABLE;
		115	}
		116
102	static void intel_flush_svm_range_dev (struct intel_svm svm, struct intel_svm_dev sdev,	117	static void intel_flush_svm_range_dev (struct intel_svm svm, struct intel_svm_dev sdev,
103	unsigned long address, unsigned long pages, int ih)	118	unsigned long address, unsigned long pages, int ih)
104	{	119	{
@@ -207,6 +222,10 @@ static const struct mmu_notifier_ops intel_mmuops = { · 102 unmodified lines
207	static DEFINE_MUTEX(pasid_mutex);	222	static DEFINE_MUTEX(pasid_mutex);
208	static LIST_HEAD(global_svm_list);	223	static LIST_HEAD(global_svm_list);
209		224
		225	#define for_each_svm_dev(sdev, svm, d) \
		226	list_for_each_entry((sdev), &(svm)->devs, list) \
		227	if ((d) != (sdev)->dev) {} else
		228
210	int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ops *ops)	229	int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ops *ops)
211	{	230	{
212	struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);	231	struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
@@ -220,6 +239,9 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 7 unmodified lines
220	if (!iommu \|\| dmar_disabled)	239	if (!iommu \|\| dmar_disabled)
221	return -EINVAL;	240	return -EINVAL;
222		241
		242	if (!intel_svm_capable(iommu))
		243	return -ENOTSUPP;
		244
223	if (dev_is_pci(dev)) {	245	if (dev_is_pci(dev)) {
224	pasid_max = pci_max_pasids(to_pci_dev(dev));	246	pasid_max = pci_max_pasids(to_pci_dev(dev));
225	if (pasid_max < 0)	247	if (pasid_max < 0)
@@ -252,15 +274,14 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 26 unmodified lines
252	goto out;	274	goto out;
253	}	275	}
254		276
255	list_for_each_entry(sdev, &svm->devs, list) {	277	/* Find the matching device in svm list */
256	if (dev == sdev->dev) {	278	for_each_svm_dev(sdev, svm, dev) {
257	if (sdev->ops != ops) {	279	if (sdev->ops != ops) {
258	ret = -EBUSY;	280	ret = -EBUSY;
259	goto out;	281	goto out;
260	}
261	sdev->users++;
262	goto success;
263	}	282	}
		283	sdev->users++;
		284	goto success;
264	}	285	}
265		286
266	break;	287	break;
@@ -314,16 +335,15 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 47 unmodified lines
314	if (pasid_max > intel_pasid_max_id)	335	if (pasid_max > intel_pasid_max_id)
315	pasid_max = intel_pasid_max_id;	336	pasid_max = intel_pasid_max_id;
316		337
317	/* Do not use PASID 0 in caching mode (virtualised IOMMU) */	338	/* Do not use PASID 0, reserved for RID to PASID */
318	ret = intel_pasid_alloc_id(svm,	339	svm->pasid = ioasid_alloc(NULL, PASID_MIN,
319	!!cap_caching_mode(iommu->cap),	340	pasid_max - 1, svm);
320	pasid_max - 1, GFP_KERNEL);	341	if (svm->pasid == INVALID_IOASID) {
321	if (ret < 0) {
322	kfree(svm);	342	kfree(svm);
323	kfree(sdev);	343	kfree(sdev);
		344	ret = -ENOSPC;
324	goto out;	345	goto out;
325	}	346	}
326	svm->pasid = ret;
327	svm->notifier.ops = &intel_mmuops;	347	svm->notifier.ops = &intel_mmuops;
328	svm->mm = mm;	348	svm->mm = mm;
329	svm->flags = flags;	349	svm->flags = flags;
@@ -333,7 +353,7 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 3 unmodified lines
333	if (mm) {	353	if (mm) {
334	ret = mmu_notifier_register(&svm->notifier, mm);	354	ret = mmu_notifier_register(&svm->notifier, mm);
335	if (ret) {	355	if (ret) {
336	intel_pasid_free_id(svm->pasid);	356	ioasid_free(svm->pasid);
337	kfree(svm);	357	kfree(svm);
338	kfree(sdev);	358	kfree(sdev);
339	goto out;	359	goto out;
@@ -344,12 +364,14 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 4 unmodified lines
344	ret = intel_pasid_setup_first_level(iommu, dev,	364	ret = intel_pasid_setup_first_level(iommu, dev,
345	mm ? mm->pgd : init_mm.pgd,	365	mm ? mm->pgd : init_mm.pgd,
346	svm->pasid, FLPT_DEFAULT_DID,	366	svm->pasid, FLPT_DEFAULT_DID,
347	mm ? 0 : PASID_FLAG_SUPERVISOR_MODE);	367	(mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) \|
		368	(cpu_feature_enabled(X86_FEATURE_LA57) ?
		369	PASID_FLAG_FL5LP : 0));
348	spin_unlock(&iommu->lock);	370	spin_unlock(&iommu->lock);
349	if (ret) {	371	if (ret) {
350	if (mm)	372	if (mm)
351	mmu_notifier_unregister(&svm->notifier, mm);	373	mmu_notifier_unregister(&svm->notifier, mm);
352	intel_pasid_free_id(svm->pasid);	374	ioasid_free(svm->pasid);
353	kfree(svm);	375	kfree(svm);
354	kfree(sdev);	376	kfree(sdev);
355	goto out;	377	goto out;
@@ -365,7 +387,9 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 9 unmodified lines
365	ret = intel_pasid_setup_first_level(iommu, dev,	387	ret = intel_pasid_setup_first_level(iommu, dev,
366	mm ? mm->pgd : init_mm.pgd,	388	mm ? mm->pgd : init_mm.pgd,
367	svm->pasid, FLPT_DEFAULT_DID,	389	svm->pasid, FLPT_DEFAULT_DID,
368	mm ? 0 : PASID_FLAG_SUPERVISOR_MODE);	390	(mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) \|
		391	(cpu_feature_enabled(X86_FEATURE_LA57) ?
		392	PASID_FLAG_FL5LP : 0));
369	spin_unlock(&iommu->lock);	393	spin_unlock(&iommu->lock);
370	if (ret) {	394	if (ret) {
371	kfree(sdev);	395	kfree(sdev);
@@ -397,44 +421,45 @@ int intel_svm_unbind_mm(struct device *dev, int pasid) · 25 unmodified lines
397	if (!iommu)	421	if (!iommu)
398	goto out;	422	goto out;
399		423
400	svm = intel_pasid_lookup_id(pasid);	424	svm = ioasid_find(NULL, pasid, NULL);
401	if (!svm)	425	if (!svm)
402	goto out;	426	goto out;
403		427
404	list_for_each_entry(sdev, &svm->devs, list) {	428	if (IS_ERR(svm)) {
405	if (dev == sdev->dev) {	429	ret = PTR_ERR(svm);
406	ret = 0;	430	goto out;
407	sdev->users--;	431	}
408	if (!sdev->users) {	432
409	list_del_rcu(&sdev->list);	433	for_each_svm_dev(sdev, svm, dev) {
410	/* Flush the PASID cache and IOTLB for this device.	434	ret = 0;
411	* Note that we do depend on the hardware not using	435	sdev->users--;
412	* the PASID any more. Just as we depend on other	436	if (!sdev->users) {
413	* devices never using PASIDs that they have no right	437	list_del_rcu(&sdev->list);
414	* to use. We have a shared PASID table, because it's	438	/* Flush the PASID cache and IOTLB for this device.
415	* large and has to be physically contiguous. So it's	439	* Note that we do depend on the hardware not using
416	* hard to be as defensive as we might like. */	440	* the PASID any more. Just as we depend on other
417	intel_pasid_tear_down_entry(iommu, dev, svm->pasid);	441	* devices never using PASIDs that they have no right
418	intel_flush_svm_range_dev(svm, sdev, 0, -1, 0);	442	* to use. We have a shared PASID table, because it's
419	kfree_rcu(sdev, rcu);	443	* large and has to be physically contiguous. So it's
420		444	* hard to be as defensive as we might like. */
421	if (list_empty(&svm->devs)) {	445	intel_pasid_tear_down_entry(iommu, dev, svm->pasid);
422	intel_pasid_free_id(svm->pasid);	446	intel_flush_svm_range_dev(svm, sdev, 0, -1, 0);
423	if (svm->mm)	447	kfree_rcu(sdev, rcu);
424	mmu_notifier_unregister(&svm->notifier, svm->mm);	448
425		449	if (list_empty(&svm->devs)) {
426	list_del(&svm->list);	450	ioasid_free(svm->pasid);
427		451	if (svm->mm)
428	/* We mandate that no page faults may be outstanding	452	mmu_notifier_unregister(&svm->notifier, svm->mm);
429	* for the PASID when intel_svm_unbind_mm() is called.	453	list_del(&svm->list);
430	* If that is not obeyed, subtle errors will happen.	454	/* We mandate that no page faults may be outstanding
431	* Let's make them less subtle... */	455	* for the PASID when intel_svm_unbind_mm() is called.
432	memset(svm, 0x6b, sizeof(*svm));	456	* If that is not obeyed, subtle errors will happen.
433	kfree(svm);	457	* Let's make them less subtle... */
434	}	458	memset(svm, 0x6b, sizeof(*svm));
		459	kfree(svm);
435	}	460	}
436	break;
437	}	461	}
		462	break;
438	}	463	}
439	out:	464	out:
440	mutex_unlock(&pasid_mutex);	465	mutex_unlock(&pasid_mutex);
@@ -454,10 +479,14 @@ int intel_svm_is_pasid_valid(struct device *dev, int pasid) · 13 unmodified lines
454	if (!iommu)	479	if (!iommu)
455	goto out;	480	goto out;
456		481
457	svm = intel_pasid_lookup_id(pasid);	482	svm = ioasid_find(NULL, pasid, NULL);
458	if (!svm)	483	if (!svm)
459	goto out;	484	goto out;
460		485
		486	if (IS_ERR(svm)) {
		487	ret = PTR_ERR(svm);
		488	goto out;
		489	}
461	/* init_mm is used in this case */	490	/* init_mm is used in this case */
462	if (!svm->mm)	491	if (!svm->mm)
463	ret = 1;	492	ret = 1;
@@ -564,13 +593,12 @@ static irqreturn_t prq_event_thread(int irq, void *d) · 100 unmodified lines
564		593
565	if (!svm \|\| svm->pasid != req->pasid) {	594	if (!svm \|\| svm->pasid != req->pasid) {
566	rcu_read_lock();	595	rcu_read_lock();
567	svm = intel_pasid_lookup_id(req->pasid);	596	svm = ioasid_find(NULL, req->pasid, NULL);
568	/* It can't go away, because the driver is not permitted	597	/* It can't go away, because the driver is not permitted
569	* to unbind the mm while any page faults are outstanding.	598	* to unbind the mm while any page faults are outstanding.
570	* So we only need RCU to protect the internal idr code. */	599	* So we only need RCU to protect the internal idr code. */
571	rcu_read_unlock();	600	rcu_read_unlock();
572		601	if (IS_ERR_OR_NULL(svm)) {
573	if (!svm) {
574	pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n",	602	pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n",
575	iommu->name, req->pasid, ((unsigned long long *)req)[0],	603	iommu->name, req->pasid, ((unsigned long long *)req)[0],
576	((unsigned long long *)req)[1]);	604	((unsigned long long *)req)[1]);
@@ -654,11 +682,10 @@ static irqreturn_t prq_event_thread(int irq, void *d) · 77 unmodified lines
654	if (req->priv_data_present)	682	if (req->priv_data_present)
655	memcpy(&resp.qw2, req->priv_data,	683	memcpy(&resp.qw2, req->priv_data,
656	sizeof(req->priv_data));	684	sizeof(req->priv_data));
		685	resp.qw2 = 0;
		686	resp.qw3 = 0;
		687	qi_submit_sync(&resp, iommu);
657	}	688	}
658	resp.qw2 = 0;
659	resp.qw3 = 0;
660	qi_submit_sync(&resp, iommu);
661
662	head = (head + sizeof(*req)) & PRQ_RING_MASK;	689	head = (head + sizeof(*req)) & PRQ_RING_MASK;
663	}	690	}
664		691

drivers/iommu/io-pgtable-arm-v7s.c+9−13

 #define ARM_V7S_TTBR_IRGN_ATTR(attr)					\
 	((((attr) & 0x1) << 6) | (((attr) & 0x2) >> 1))
-#define ARM_V7S_TCR_PD1			BIT(5)
 #ifdef CONFIG_ZONE_DMA32
 #define ARM_V7S_TABLE_GFP_DMA GFP_DMA32
 #define ARM_V7S_TABLE_SLAB_FLAGS SLAB_CACHE_DMA32
 	 */
 	cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
-	/* TCR: T0SZ=0, disable TTBR1 */
+	/* TCR: T0SZ=0, EAE=0 (if applicable) */
-	cfg->arm_v7s_cfg.tcr = ARM_V7S_TCR_PD1;
+	cfg->arm_v7s_cfg.tcr = 0;
 	/*
 	 * TEX remap: the indices used map to the closest equivalent types
 	/* Ensure the empty pgd is visible before any actual TTBR write */
 	wmb();
-	/* TTBRs */
+	/* TTBR */
-	cfg->arm_v7s_cfg.ttbr[0] = virt_to_phys(data->pgd) |
+	cfg->arm_v7s_cfg.ttbr = virt_to_phys(data->pgd) | ARM_V7S_TTBR_S |
-				   ARM_V7S_TTBR_S | ARM_V7S_TTBR_NOS |
+				(cfg->coherent_walk ? (ARM_V7S_TTBR_NOS |
-				   (cfg->coherent_walk ?
+				 ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) |
-				   (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) |
+				 ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) :
-				    ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) :
+				(ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) |
-				   (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) |
+				 ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC)));
-				    ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC)));
-	cfg->arm_v7s_cfg.ttbr[1] = 0;
 	return &data->iop;
 out_free_data:

drivers/iommu/io-pgtable-arm.c+77−87

 #define ARM_LPAE_PTE_MEMATTR_DEV	(((arm_lpae_iopte)0x1) << 2)
 /* Register bits */
-#define ARM_32_LPAE_TCR_EAE		(1 << 31)
+#define ARM_LPAE_TCR_TG0_4K		0
-#define ARM_64_LPAE_S2_TCR_RES1		(1 << 31)
+#define ARM_LPAE_TCR_TG0_64K		1
+#define ARM_LPAE_TCR_TG0_16K		2
-#define ARM_LPAE_TCR_EPD1		(1 << 23)
+#define ARM_LPAE_TCR_TG1_16K		1
+#define ARM_LPAE_TCR_TG1_4K		2
+#define ARM_LPAE_TCR_TG1_64K		3
-#define ARM_LPAE_TCR_TG0_4K		(0 << 14)
-#define ARM_LPAE_TCR_TG0_64K		(1 << 14)
-#define ARM_LPAE_TCR_TG0_16K		(2 << 14)
-#define ARM_LPAE_TCR_SH0_SHIFT		12
-#define ARM_LPAE_TCR_SH0_MASK		0x3
 #define ARM_LPAE_TCR_SH_NS		0
 #define ARM_LPAE_TCR_SH_OS		2
 #define ARM_LPAE_TCR_SH_IS		3
-#define ARM_LPAE_TCR_ORGN0_SHIFT	10
-#define ARM_LPAE_TCR_IRGN0_SHIFT	8
-#define ARM_LPAE_TCR_RGN_MASK		0x3
 #define ARM_LPAE_TCR_RGN_NC		0
 #define ARM_LPAE_TCR_RGN_WBWA		1
 #define ARM_LPAE_TCR_RGN_WT		2
 #define ARM_LPAE_TCR_RGN_WB		3
-#define ARM_LPAE_TCR_SL0_SHIFT		6
+#define ARM_LPAE_VTCR_SL0_MASK		0x3
-#define ARM_LPAE_TCR_SL0_MASK		0x3
 #define ARM_LPAE_TCR_T0SZ_SHIFT		0
-#define ARM_LPAE_TCR_SZ_MASK		0xf
-#define ARM_LPAE_TCR_PS_SHIFT		16
-#define ARM_LPAE_TCR_PS_MASK		0x7
-#define ARM_LPAE_TCR_IPS_SHIFT		32
+#define ARM_LPAE_VTCR_PS_SHIFT		16
-#define ARM_LPAE_TCR_IPS_MASK		0x7
+#define ARM_LPAE_VTCR_PS_MASK		0x7
 #define ARM_LPAE_TCR_PS_32_BIT		0x0ULL
 #define ARM_LPAE_TCR_PS_36_BIT		0x1ULL
 {
 	arm_lpae_iopte pte = prot;
-	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
-		pte |= ARM_LPAE_PTE_NS;
 	if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
 		pte |= ARM_LPAE_PTE_TYPE_PAGE;
 	else
 		pte |= ARM_LPAE_PTE_TYPE_BLOCK;
-	if (data->iop.fmt != ARM_MALI_LPAE)
-		pte |= ARM_LPAE_PTE_AF;
-	pte |= ARM_LPAE_PTE_SH_IS;
 	pte |= paddr_to_iopte(paddr, data);
 	__arm_lpae_set_pte(ptep, pte, &data->iop.cfg);
 				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
 	}
+	if (prot & IOMMU_CACHE)
+		pte |= ARM_LPAE_PTE_SH_IS;
+	else
+		pte |= ARM_LPAE_PTE_SH_OS;
 	if (prot & IOMMU_NOEXEC)
 		pte |= ARM_LPAE_PTE_XN;
+	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
+		pte |= ARM_LPAE_PTE_NS;
+	if (data->iop.fmt != ARM_MALI_LPAE)
+		pte |= ARM_LPAE_PTE_AF;
 	return pte;
 }
 	arm_lpae_iopte *ptep = data->pgd;
 	int ret, lvl = data->start_level;
 	arm_lpae_iopte prot;
+	long iaext = (long)iova >> cfg->ias;
 	/* If no access, then nothing to do */
 	if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
 	if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size))
 		return -EINVAL;
-	if (WARN_ON(iova >> data->iop.cfg.ias || paddr >> data->iop.cfg.oas))
+	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
+		iaext = ~iaext;
+	if (WARN_ON(iaext || paddr >> cfg->oas))
 		return -ERANGE;
 	prot = arm_lpae_prot_to_pte(data, iommu_prot);
 	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
 	struct io_pgtable_cfg *cfg = &data->iop.cfg;
 	arm_lpae_iopte *ptep = data->pgd;
+	long iaext = (long)iova >> cfg->ias;
 	if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size))
 		return 0;
-	if (WARN_ON(iova >> data->iop.cfg.ias))
+	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
+		iaext = ~iaext;
+	if (WARN_ON(iaext))
 		return 0;
 	return __arm_lpae_unmap(data, gather, iova, size, data->start_level, ptep);
 {
 	u64 reg;
 	struct arm_lpae_io_pgtable *data;
+	typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;
+	bool tg1;
 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |
-			    IO_PGTABLE_QUIRK_NON_STRICT))
+			    IO_PGTABLE_QUIRK_NON_STRICT |
+			    IO_PGTABLE_QUIRK_ARM_TTBR1))
 		return NULL;
 	data = arm_lpae_alloc_pgtable(cfg);
 	/* TCR */
 	if (cfg->coherent_walk) {
-		reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) |
+		tcr->sh = ARM_LPAE_TCR_SH_IS;
-		      (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
+		tcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
-		      (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
+		tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
 	} else {
-		reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) |
+		tcr->sh = ARM_LPAE_TCR_SH_OS;
-		      (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) |
+		tcr->irgn = ARM_LPAE_TCR_RGN_NC;
-		      (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT);
+		tcr->orgn = ARM_LPAE_TCR_RGN_NC;
 	}
+	tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;
 	switch (ARM_LPAE_GRANULE(data)) {
 	case SZ_4K:
-		reg |= ARM_LPAE_TCR_TG0_4K;
+		tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K;
 		break;
 	case SZ_16K:
-		reg |= ARM_LPAE_TCR_TG0_16K;
+		tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K;
 		break;
 	case SZ_64K:
-		reg |= ARM_LPAE_TCR_TG0_64K;
+		tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K;
 		break;
 	}
 	switch (cfg->oas) {
 	case 32:
-		reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT);
+		tcr->ips = ARM_LPAE_TCR_PS_32_BIT;
 		break;
 	case 36:
-		reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT);
+		tcr->ips = ARM_LPAE_TCR_PS_36_BIT;
 		break;
 	case 40:
-		reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT);
+		tcr->ips = ARM_LPAE_TCR_PS_40_BIT;
 		break;
 	case 42:
-		reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT);
+		tcr->ips = ARM_LPAE_TCR_PS_42_BIT;
 		break;
 	case 44:
-		reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT);
+		tcr->ips = ARM_LPAE_TCR_PS_44_BIT;
 		break;
 	case 48:
-		reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT);
+		tcr->ips = ARM_LPAE_TCR_PS_48_BIT;
 		break;
 	case 52:
-		reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_IPS_SHIFT);
+		tcr->ips = ARM_LPAE_TCR_PS_52_BIT;
 		break;
 	default:
 		goto out_free_data;
 	}
-	reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
+	tcr->tsz = 64ULL - cfg->ias;
-	/* Disable speculative walks through TTBR1 */
-	reg |= ARM_LPAE_TCR_EPD1;
-	cfg->arm_lpae_s1_cfg.tcr = reg;
 	/* MAIRs */
 	reg = (ARM_LPAE_MAIR_ATTR_NC
 	/* Ensure the empty pgd is visible before any actual TTBR write */
 	wmb();
-	/* TTBRs */
+	/* TTBR */
-	cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
+	cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
-	cfg->arm_lpae_s1_cfg.ttbr[1] = 0;
 	return &data->iop;
 out_free_data:
 static struct io_pgtable *
 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
 {
-	u64 reg, sl;
+	u64 sl;
 	struct arm_lpae_io_pgtable *data;
+	typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;
 	/* The NS quirk doesn't apply at stage 2 */
 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NON_STRICT))
 	}
 	/* VTCR */
-	reg = ARM_64_LPAE_S2_TCR_RES1 |
+	if (cfg->coherent_walk) {
-	     (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) |
+		vtcr->sh = ARM_LPAE_TCR_SH_IS;
-	     (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) |
+		vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
-	     (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
+		vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
+	} else {
+		vtcr->sh = ARM_LPAE_TCR_SH_OS;
+		vtcr->irgn = ARM_LPAE_TCR_RGN_NC;
+		vtcr->orgn = ARM_LPAE_TCR_RGN_NC;
+	}
 	sl = data->start_level;
 	switch (ARM_LPAE_GRANULE(data)) {
 	case SZ_4K:
-		reg |= ARM_LPAE_TCR_TG0_4K;
+		vtcr->tg = ARM_LPAE_TCR_TG0_4K;
 		sl++; /* SL0 format is different for 4K granule size */
 		break;
 	case SZ_16K:
-		reg |= ARM_LPAE_TCR_TG0_16K;
+		vtcr->tg = ARM_LPAE_TCR_TG0_16K;
 		break;
 	case SZ_64K:
-		reg |= ARM_LPAE_TCR_TG0_64K;
+		vtcr->tg = ARM_LPAE_TCR_TG0_64K;
 		break;
 	}
 	switch (cfg->oas) {
 	case 32:
-		reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_PS_SHIFT);
+		vtcr->ps = ARM_LPAE_TCR_PS_32_BIT;
 		break;
 	case 36:
-		reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_PS_SHIFT);
+		vtcr->ps = ARM_LPAE_TCR_PS_36_BIT;
 		break;
 	case 40:
-		reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_PS_SHIFT);
+		vtcr->ps = ARM_LPAE_TCR_PS_40_BIT;
 		break;
 	case 42:
-		reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_PS_SHIFT);
+		vtcr->ps = ARM_LPAE_TCR_PS_42_BIT;
 		break;
 	case 44:
-		reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_PS_SHIFT);
+		vtcr->ps = ARM_LPAE_TCR_PS_44_BIT;
 		break;
 	case 48:
-		reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT);
+		vtcr->ps = ARM_LPAE_TCR_PS_48_BIT;
 		break;
 	case 52:
-		reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_PS_SHIFT);
+		vtcr->ps = ARM_LPAE_TCR_PS_52_BIT;
 		break;
 	default:
 		goto out_free_data;
 	}
-	reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
+	vtcr->tsz = 64ULL - cfg->ias;
-	reg |= (~sl & ARM_LPAE_TCR_SL0_MASK) << ARM_LPAE_TCR_SL0_SHIFT;
+	vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK;
-	cfg->arm_lpae_s2_cfg.vtcr = reg;
 	/* Allocate pgd pages */
 	data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
 static struct io_pgtable *
 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
 {
-	struct io_pgtable *iop;
 	if (cfg->ias > 32 || cfg->oas > 40)
 		return NULL;
 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
-	iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
+	return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
-	if (iop) {
-		cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE;
-		cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff;
-	}
-	return iop;
 }
 static struct io_pgtable *
 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
 {
-	struct io_pgtable *iop;
 	if (cfg->ias > 40 || cfg->oas > 40)
 		return NULL;
 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
-	iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
+	return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
-	if (iop)
-		cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff;
-	return iop;
 }
 static struct io_pgtable *

drivers/iommu/io-pgtable.c+1−1

@@ -63,7 +63,7 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) · 62 unmodified lines
63	if (!ops)	63	if (!ops)
64	return;	64	return;
65		65
66	iop = container_of(ops, struct io_pgtable, ops);	66	iop = io_pgtable_ops_to_pgtable(ops);
67	io_pgtable_tlb_flush_all(iop);	67	io_pgtable_tlb_flush_all(iop);
68	io_pgtable_init_table[iop->fmt]->free(iop);	68	io_pgtable_init_table[iop->fmt]->free(iop);
69	}	69	}

drivers/iommu/iommu-sysfs.c+5−0

 	put_device(iommu->dev);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(iommu_device_sysfs_add);
 void iommu_device_sysfs_remove(struct iommu_device *iommu)
 {
 	device_unregister(iommu->dev);
 	iommu->dev = NULL;
 }
+EXPORT_SYMBOL_GPL(iommu_device_sysfs_remove);
 /*
  * IOMMU drivers can indicate a device is managed by a given IOMMU using
  * this interface.  A link to the device will be created in the "devices"
 	return ret;
 }
+EXPORT_SYMBOL_GPL(iommu_device_link);
 void iommu_device_unlink(struct iommu_device *iommu, struct device *link)
 {
 	sysfs_remove_link(&link->kobj, "iommu");
 	sysfs_remove_link_from_group(&iommu->dev->kobj, "devices", dev_name(link));
 }
+EXPORT_SYMBOL_GPL(iommu_device_unlink);

drivers/iommu/iommu.c+49−2

 #include <linux/bitops.h>
 #include <linux/property.h>
 #include <linux/fsl/mc.h>
+#include <linux/module.h>
 #include <trace/events/iommu.h>
 static struct kset *iommu_group_kset;
 	spin_unlock(&iommu_device_lock);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(iommu_device_register);
 void iommu_device_unregister(struct iommu_device *iommu)
 {
 	list_del(&iommu->list);
 	spin_unlock(&iommu_device_lock);
 }
+EXPORT_SYMBOL_GPL(iommu_device_unregister);
 static struct iommu_param *iommu_get_dev_param(struct device *dev)
 {
 	if (!iommu_get_dev_param(dev))
 		return -ENOMEM;
+	if (!try_module_get(ops->owner)) {
+		ret = -EINVAL;
+		goto err_free_dev_param;
+	}
 	ret = ops->add_device(dev);
 	if (ret)
-		iommu_free_dev_param(dev);
+		goto err_module_put;
+	return 0;
+err_module_put:
+	module_put(ops->owner);
+err_free_dev_param:
+	iommu_free_dev_param(dev);
 	return ret;
 }
 	if (dev->iommu_group)
 		ops->remove_device(dev);
-	iommu_free_dev_param(dev);
+	if (dev->iommu_param) {
+		module_put(ops->owner);
+		iommu_free_dev_param(dev);
+	}
 }
 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
 	kobject_get(group->devices_kobj);
 	return group;
 }
+EXPORT_SYMBOL_GPL(iommu_group_ref_get);
 /**
  * iommu_group_put - Decrement group reference
 {
 	return iommu_group_alloc();
 }
+EXPORT_SYMBOL_GPL(generic_device_group);
 /*
  * Use standard PCI bus topology, isolation features, and DMA alias quirks
 	/* No shared group found, allocate new */
 	return iommu_group_alloc();
 }
+EXPORT_SYMBOL_GPL(pci_device_group);
 /* Get the IOMMU group for device on fsl-mc bus */
 struct iommu_group *fsl_mc_device_group(struct device *dev)
 		group = iommu_group_alloc();
 	return group;
 }
+EXPORT_SYMBOL_GPL(fsl_mc_device_group);
 /**
  * iommu_group_get_for_dev - Find or create the IOMMU group for a device
 	return group;
 }
+EXPORT_SYMBOL(iommu_group_get_for_dev);
 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
 {
 {
 	int err;
+	if (ops == NULL) {
+		bus->iommu_ops = NULL;
+		return 0;
+	}
 	if (bus->iommu_ops != NULL)
 		return -EBUSY;
 		ops->put_resv_regions(dev, list);
 }
+/**
+ * generic_iommu_put_resv_regions - Reserved region driver helper
+ * @dev: device for which to free reserved regions
+ * @list: reserved region list for device
+ *
+ * IOMMU drivers can use this to implement their .put_resv_regions() callback
+ * for simple reservations. Memory allocated for each reserved region will be
+ * freed. If an IOMMU driver allocates additional resources per region, it is
+ * going to have to implement a custom callback.
+ */
+void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list)
+{
+	struct iommu_resv_region *entry, *next;
+	list_for_each_entry_safe(entry, next, list, list)
+		kfree(entry);
+}
+EXPORT_SYMBOL(generic_iommu_put_resv_regions);
 struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
 						  size_t length, int prot,
 						  enum iommu_resv_type type)
 	region->type = type;
 	return region;
 }
+EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
 static int
 request_default_domain_for_dev(struct device *dev, unsigned long type)

drivers/iommu/iova.c+1−1

@@ -233,7 +233,7 @@ static DEFINE_MUTEX(iova_cache_mutex); · 232 unmodified lines
233		233
234	struct iova *alloc_iova_mem(void)	234	struct iova *alloc_iova_mem(void)
235	{	235	{
236	return kmem_cache_zalloc(iova_cache, GFP_ATOMIC);	236	return kmem_cache_zalloc(iova_cache, GFP_ATOMIC \| __GFP_NOWARN);
237	}	237	}
238	EXPORT_SYMBOL(alloc_iova_mem);	238	EXPORT_SYMBOL(alloc_iova_mem);
239		239

drivers/iommu/ipmmu-vmsa.c+1−1

@@ -374,7 +374,7 @@ static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain) · 373 unmodified lines
374	u32 tmp;	374	u32 tmp;
375		375
376	/* TTBR0 */	376	/* TTBR0 */
377	ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0];	377	ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr;
378	ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr);	378	ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr);
379	ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32);	379	ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32);
380		380

drivers/iommu/msm_iommu.c+2−2

@@ -279,8 +279,8 @@ static void __program_context(void __iomem *base, int ctx, · 278 unmodified lines
279	SET_V2PCFG(base, ctx, 0x3);	279	SET_V2PCFG(base, ctx, 0x3);
280		280
281	SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr);	281	SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr);
282	SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]);	282	SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr);
283	SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]);	283	SET_TTBR1(base, ctx, 0);
284		284
285	/* Set prrr and nmrr */	285	/* Set prrr and nmrr */
286	SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr);	286	SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr);

drivers/iommu/mtk_iommu.c+2−2

@@ -367,7 +367,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, · 366 unmodified lines
367	/* Update the pgtable base address register of the M4U HW */	367	/* Update the pgtable base address register of the M4U HW */
368	if (!data->m4u_dom) {	368	if (!data->m4u_dom) {
369	data->m4u_dom = dom;	369	data->m4u_dom = dom;
370	writel(dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK,	370	writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK,
371	data->base + REG_MMU_PT_BASE_ADDR);	371	data->base + REG_MMU_PT_BASE_ADDR);
372	}	372	}
373		373
@@ -765,7 +765,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) · 391 unmodified lines
765	writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);	765	writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
766	writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);	766	writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);
767	if (m4u_dom)	767	if (m4u_dom)
768	writel(m4u_dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK,	768	writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK,
769	base + REG_MMU_PT_BASE_ADDR);	769	base + REG_MMU_PT_BASE_ADDR);
770	return 0;	770	return 0;
771	}	771	}

drivers/iommu/of_iommu.c+18−7

 #include <linux/export.h>
 #include <linux/iommu.h>
 #include <linux/limits.h>
-#include <linux/pci.h>
+#include <linux/module.h>
 #include <linux/msi.h>
 #include <linux/of.h>
 #include <linux/of_iommu.h>
 #include <linux/of_pci.h>
+#include <linux/pci.h>
 #include <linux/slab.h>
 #include <linux/fsl/mc.h>
 {
 	const struct iommu_ops *ops;
 	struct fwnode_handle *fwnode = &iommu_spec->np->fwnode;
-	int err;
+	int ret;
 	ops = iommu_ops_from_fwnode(fwnode);
 	if ((ops && !ops->of_xlate) ||
 	    !of_device_is_available(iommu_spec->np))
 		return NO_IOMMU;
-	err = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops);
+	ret = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops);
-	if (err)
+	if (ret)
-		return err;
+		return ret;
 	/*
 	 * The otherwise-empty fwspec handily serves to indicate the specific
 	 * IOMMU device we're waiting for, which will be useful if we ever get
 	if (!ops)
 		return driver_deferred_probe_check_state(dev);
-	return ops->of_xlate(dev, iommu_spec);
+	if (!try_module_get(ops->owner))
+		return -ENODEV;
+	ret = ops->of_xlate(dev, iommu_spec);
+	module_put(ops->owner);
+	return ret;
 }
 struct of_pci_iommu_alias_info {
 			.np = master_np,
 		};
+		pci_request_acs();
 		err = pci_for_each_dma_alias(to_pci_dev(dev),
 					     of_pci_iommu_init, &info);
 	} else if (dev_is_fsl_mc(dev)) {
 			if (err)
 				break;
 		}
-	}
+		fwspec = dev_iommu_fwspec_get(dev);
+		if (!err && fwspec)
+			of_property_read_u32(master_np, "pasid-num-bits",
+					     &fwspec->num_pasid_bits);
+	}
 	/*
 	 * Two success conditions can be represented by non-negative err here:

drivers/iommu/qcom_iommu.c+12−13

 	fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
-	if (!(fsr & FSR_FAULT))
+	if (!(fsr & ARM_SMMU_FSR_FAULT))
 		return IRQ_NONE;
 	fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
 	}
 	iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);
-	iommu_writel(ctx, ARM_SMMU_CB_RESUME, RESUME_TERMINATE);
+	iommu_writel(ctx, ARM_SMMU_CB_RESUME, ARM_SMMU_RESUME_TERMINATE);
 	return IRQ_HANDLED;
 }
 		/* TTBRs */
 		iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
-				pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
+				pgtbl_cfg.arm_lpae_s1_cfg.ttbr |
-				FIELD_PREP(TTBRn_ASID, ctx->asid));
+				FIELD_PREP(ARM_SMMU_TTBRn_ASID, ctx->asid));
-		iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
+		iommu_writeq(ctx, ARM_SMMU_CB_TTBR1, 0);
-				pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
-				FIELD_PREP(TTBRn_ASID, ctx->asid));
 		/* TCR */
 		iommu_writel(ctx, ARM_SMMU_CB_TCR2,
-				(pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
+				arm_smmu_lpae_tcr2(&pgtbl_cfg));
-				FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM));
 		iommu_writel(ctx, ARM_SMMU_CB_TCR,
-				pgtbl_cfg.arm_lpae_s1_cfg.tcr);
+			     arm_smmu_lpae_tcr(&pgtbl_cfg) | ARM_SMMU_TCR_EAE);
 		/* MAIRs (stage-1 only) */
 		iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
 				pgtbl_cfg.arm_lpae_s1_cfg.mair >> 32);
 		/* SCTLR */
-		reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE |
+		reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE |
-			SCTLR_M | SCTLR_S1_ASIDPNE | SCTLR_CFCFG;
+		      ARM_SMMU_SCTLR_AFE | ARM_SMMU_SCTLR_TRE |
+		      ARM_SMMU_SCTLR_M | ARM_SMMU_SCTLR_S1_ASIDPNE |
+		      ARM_SMMU_SCTLR_CFCFG;
 		if (IS_ENABLED(CONFIG_BIG_ENDIAN))
-			reg |= SCTLR_E;
+			reg |= ARM_SMMU_SCTLR_E;
 		iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);

drivers/iommu/virtio-iommu.c+3−11

 	iommu_dma_get_resv_regions(dev, head);
 }
-static void viommu_put_resv_regions(struct device *dev, struct list_head *head)
-{
-	struct iommu_resv_region *entry, *next;
-	list_for_each_entry_safe(entry, next, head, list)
-		kfree(entry);
-}
 static struct iommu_ops viommu_ops;
 static struct virtio_driver virtio_iommu_drv;
 err_unlink_dev:
 	iommu_device_unlink(&viommu->iommu, dev);
 err_free_dev:
-	viommu_put_resv_regions(dev, &vdev->resv_regions);
+	generic_iommu_put_resv_regions(dev, &vdev->resv_regions);
 	kfree(vdev);
 	return ret;
 	iommu_group_remove_device(dev);
 	iommu_device_unlink(&vdev->viommu->iommu, dev);
-	viommu_put_resv_regions(dev, &vdev->resv_regions);
+	generic_iommu_put_resv_regions(dev, &vdev->resv_regions);
 	kfree(vdev);
 }
 	.remove_device		= viommu_remove_device,
 	.device_group		= viommu_device_group,
 	.get_resv_regions	= viommu_get_resv_regions,
-	.put_resv_regions	= viommu_put_resv_regions,
+	.put_resv_regions	= generic_iommu_put_resv_regions,
 	.of_xlate		= viommu_of_xlate,
 };

drivers/pci/ats.c+2−0

 	dev->ats_enabled = 1;
 	return 0;
 }
+EXPORT_SYMBOL_GPL(pci_enable_ats);
 /**
  * pci_disable_ats - disable the ATS capability
 	dev->ats_enabled = 0;
 }
+EXPORT_SYMBOL_GPL(pci_disable_ats);
 void pci_restore_ats_state(struct pci_dev *dev)
 {

drivers/pci/pci.c+1−0

@@ -131,6 +131,7 @@ bool pci_ats_disabled(void) · 130 unmodified lines
131	{	131	{
132	return pcie_ats_disabled;	132	return pcie_ats_disabled;
133	}	133	}
		134	EXPORT_SYMBOL_GPL(pci_ats_disabled);
134		135
135	/* Disable bridge_d3 for all PCIe ports */	136	/* Disable bridge_d3 for all PCIe ports */
136	static bool pci_bridge_d3_disable;	137	static bool pci_bridge_d3_disable;

include/linux/intel-iommu.h+18−7

 #define VTD_STRIDE_SHIFT        (9)
 #define VTD_STRIDE_MASK         (((u64)-1) << VTD_STRIDE_SHIFT)
-#define DMA_PTE_READ (1)
+#define DMA_PTE_READ		BIT_ULL(0)
-#define DMA_PTE_WRITE (2)
+#define DMA_PTE_WRITE		BIT_ULL(1)
-#define DMA_PTE_LARGE_PAGE (1 << 7)
+#define DMA_PTE_LARGE_PAGE	BIT_ULL(7)
-#define DMA_PTE_SNP (1 << 11)
+#define DMA_PTE_SNP		BIT_ULL(11)
+#define DMA_FL_PTE_PRESENT	BIT_ULL(0)
+#define DMA_FL_PTE_XD		BIT_ULL(63)
 #define CONTEXT_TT_MULTI_LEVEL	0
 #define CONTEXT_TT_DEV_IOTLB	1
 #define VTD_FLAG_TRANS_PRE_ENABLED	(1 << 0)
 #define VTD_FLAG_IRQ_REMAP_PRE_ENABLED	(1 << 1)
+#define VTD_FLAG_SVM_CAPABLE		(1 << 2)
 extern int intel_iommu_sm;
+extern spinlock_t device_domain_lock;
 #define sm_supported(iommu)	(intel_iommu_sm && ecap_smts((iommu)->ecap))
 #define pasid_supported(iommu)	(sm_supported(iommu) &&			\
 static inline u64 dma_pte_addr(struct dma_pte *pte)
 {
 #ifdef CONFIG_64BIT
-	return pte->val & VTD_PAGE_MASK;
+	return pte->val & VTD_PAGE_MASK & (~DMA_FL_PTE_XD);
 #else
 	/* Must have a full atomic 64-bit read */
-	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
+	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) &
+			VTD_PAGE_MASK & (~DMA_FL_PTE_XD);
 #endif
 }
 			  unsigned int size_order, u64 type);
 extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
 			u16 qdep, u64 addr, unsigned mask);
+void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
+		     unsigned long npages, bool ih);
 extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu);
 extern int dmar_ir_support(void);
 				     void *data), void *data);
 void iommu_flush_write_buffer(struct intel_iommu *iommu);
 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev);
+struct dmar_domain *find_domain(struct device *dev);
 #ifdef CONFIG_INTEL_IOMMU_SVM
-int intel_svm_init(struct intel_iommu *iommu);
+extern void intel_svm_check(struct intel_iommu *iommu);
 extern int intel_svm_enable_prq(struct intel_iommu *iommu);
 extern int intel_svm_finish_prq(struct intel_iommu *iommu);
 };
 extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev);
+#else
+static inline void intel_svm_check(struct intel_iommu *iommu) {}
 #endif
 #ifdef CONFIG_INTEL_IOMMU_DEBUGFS

include/linux/io-pgtable.h+23−4

 	 * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
 	 *	on unmap, for DMA domains using the flush queue mechanism for
 	 *	delayed invalidation.
+	 *
+	 * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table
+	 *	for use in the upper half of a split address space.
 	 */
 	#define IO_PGTABLE_QUIRK_ARM_NS		BIT(0)
 	#define IO_PGTABLE_QUIRK_NO_PERMS	BIT(1)
 	#define IO_PGTABLE_QUIRK_TLBI_ON_MAP	BIT(2)
 	#define IO_PGTABLE_QUIRK_ARM_MTK_EXT	BIT(3)
 	#define IO_PGTABLE_QUIRK_NON_STRICT	BIT(4)
+	#define IO_PGTABLE_QUIRK_ARM_TTBR1	BIT(5)
 	unsigned long			quirks;
 	unsigned long			pgsize_bitmap;
 	unsigned int			ias;
 	/* Low-level data specific to the table format */
 	union {
 		struct {
-			u64	ttbr[2];
+			u64	ttbr;
-			u64	tcr;
+			struct {
+				u32	ips:3;
+				u32	tg:2;
+				u32	sh:2;
+				u32	orgn:2;
+				u32	irgn:2;
+				u32	tsz:6;
+			}	tcr;
 			u64	mair;
 		} arm_lpae_s1_cfg;
 		struct {
 			u64	vttbr;
-			u64	vtcr;
+			struct {
+				u32	ps:3;
+				u32	tg:2;
+				u32	sh:2;
+				u32	orgn:2;
+				u32	irgn:2;
+				u32	sl:2;
+				u32	tsz:6;
+			}	vtcr;
 		} arm_lpae_s2_cfg;
 		struct {
-			u32	ttbr[2];
+			u32	ttbr;
 			u32	tcr;
 			u32	nmrr;
 			u32	prrr;

include/linux/iommu.h+16−3

  * @sva_get_pasid: Get PASID associated to a SVA handle
  * @page_response: handle page request response
  * @cache_invalidate: invalidate translation caches
- * @pgsize_bitmap: bitmap of all possible supported page sizes
  * @sva_bind_gpasid: bind guest pasid and mm
  * @sva_unbind_gpasid: unbind guest pasid and mm
+ * @pgsize_bitmap: bitmap of all possible supported page sizes
+ * @owner: Driver module providing these ops
  */
 struct iommu_ops {
 	bool (*capable)(enum iommu_cap);
 	int (*sva_unbind_gpasid)(struct device *dev, int pasid);
 	unsigned long pgsize_bitmap;
+	struct module *owner;
 };
 /**
 int  iommu_device_link(struct iommu_device   *iommu, struct device *link);
 void iommu_device_unlink(struct iommu_device *iommu, struct device *link);
-static inline void iommu_device_set_ops(struct iommu_device *iommu,
+static inline void __iommu_device_set_ops(struct iommu_device *iommu,
-					const struct iommu_ops *ops)
+					  const struct iommu_ops *ops)
 {
 	iommu->ops = ops;
 }
+#define iommu_device_set_ops(iommu, ops)				\
+do {									\
+	struct iommu_ops *__ops = (struct iommu_ops *)(ops);		\
+	__ops->owner = THIS_MODULE;					\
+	__iommu_device_set_ops(iommu, __ops);				\
+} while (0)
 static inline void iommu_device_set_fwnode(struct iommu_device *iommu,
 					   struct fwnode_handle *fwnode)
 {
 extern void iommu_get_resv_regions(struct device *dev, struct list_head *list);
 extern void iommu_put_resv_regions(struct device *dev, struct list_head *list);
+extern void generic_iommu_put_resv_regions(struct device *dev,
+					   struct list_head *list);
 extern int iommu_request_dm_for_dev(struct device *dev);
 extern int iommu_request_dma_domain_for_dev(struct device *dev);
 extern void iommu_set_default_passthrough(bool cmd_line);
  * @ops: ops for this device's IOMMU
  * @iommu_fwnode: firmware handle for this device's IOMMU
  * @iommu_priv: IOMMU driver private data for this device
+ * @num_pasid_bits: number of PASID bits supported by this device
  * @num_ids: number of associated device IDs
  * @ids: IDs which this device may present to the IOMMU
  */
 	struct fwnode_handle	*iommu_fwnode;
 	void			*iommu_priv;
 	u32			flags;
+	u32			num_pasid_bits;
 	unsigned int		num_ids;
 	u32			ids[1];
 };

include/linux/pci-ats.h+3−0

 int pci_pasid_features(struct pci_dev *pdev);
 int pci_max_pasids(struct pci_dev *pdev);
 #else /* CONFIG_PCI_PASID */
+static inline int pci_enable_pasid(struct pci_dev *pdev, int features)
+{ return -EINVAL; }
+static inline void pci_disable_pasid(struct pci_dev *pdev) { }
 static inline int pci_pasid_features(struct pci_dev *pdev)
 { return -EINVAL; }
 static inline int pci_max_pasids(struct pci_dev *pdev)

include/trace/events/intel_iommu.h+42−6

@@ -49,12 +49,6 @@ DEFINE_EVENT(dma_map, map_single, · 48 unmodified lines
49	TP_ARGS(dev, dev_addr, phys_addr, size)	49	TP_ARGS(dev, dev_addr, phys_addr, size)
50	);	50	);
51		51
52	DEFINE_EVENT(dma_map, map_sg,
53	TP_PROTO(struct device *dev, dma_addr_t dev_addr, phys_addr_t phys_addr,
54	size_t size),
55	TP_ARGS(dev, dev_addr, phys_addr, size)
56	);
57
58	DEFINE_EVENT(dma_map, bounce_map_single,	52	DEFINE_EVENT(dma_map, bounce_map_single,
59	TP_PROTO(struct device *dev, dma_addr_t dev_addr, phys_addr_t phys_addr,	53	TP_PROTO(struct device *dev, dma_addr_t dev_addr, phys_addr_t phys_addr,
60	size_t size),	54	size_t size),
@@ -99,6 +93,48 @@ DEFINE_EVENT(dma_unmap, bounce_unmap_single, · 38 unmodified lines
99	TP_ARGS(dev, dev_addr, size)	93	TP_ARGS(dev, dev_addr, size)
100	);	94	);
101		95
		96	DECLARE_EVENT_CLASS(dma_map_sg,
		97	TP_PROTO(struct device *dev, int index, int total,
		98	struct scatterlist *sg),
		99
		100	TP_ARGS(dev, index, total, sg),
		101
		102	TP_STRUCT__entry(
		103	__string(dev_name, dev_name(dev))
		104	__field(dma_addr_t, dev_addr)
		105	__field(phys_addr_t, phys_addr)
		106	__field(size_t, size)
		107	__field(int, index)
		108	__field(int, total)
		109	),
		110
		111	TP_fast_assign(
		112	__assign_str(dev_name, dev_name(dev));
		113	__entry->dev_addr = sg->dma_address;
		114	__entry->phys_addr = sg_phys(sg);
		115	__entry->size = sg->dma_length;
		116	__entry->index = index;
		117	__entry->total = total;
		118	),
		119
		120	TP_printk("dev=%s [%d/%d] dev_addr=0x%llx phys_addr=0x%llx size=%zu",
		121	__get_str(dev_name), __entry->index, __entry->total,
		122	(unsigned long long)__entry->dev_addr,
		123	(unsigned long long)__entry->phys_addr,
		124	__entry->size)
		125	);
		126
		127	DEFINE_EVENT(dma_map_sg, map_sg,
		128	TP_PROTO(struct device *dev, int index, int total,
		129	struct scatterlist *sg),
		130	TP_ARGS(dev, index, total, sg)
		131	);
		132
		133	DEFINE_EVENT(dma_map_sg, bounce_map_sg,
		134	TP_PROTO(struct device *dev, int index, int total,
		135	struct scatterlist *sg),
		136	TP_ARGS(dev, index, total, sg)
		137	);
102	#endif /* _TRACE_INTEL_IOMMU_H */	138	#endif /* _TRACE_INTEL_IOMMU_H */
103		139
104	/* This part must be outside protection */	140	/* This part must be outside protection */

Merge tag 'iommu-updates-v5.6' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code