Merge tag 'iommu-updates-v5.6' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Pull iommu updates from Joerg Roedel: - Allow compiling the ARM-SMMU drivers as modules. - Fixes and cleanups for the ARM-SMMU drivers and io-pgtable code collected by Will Deacon. The merge-commit (6855d1ba7537) has all the details. - Cleanup of the iommu_put_resv_regions() call-backs in various drivers. - AMD IOMMU driver cleanups. - Update for the x2APIC support in the AMD IOMMU driver. - Preparation patches for Intel VT-d nested mode support. - RMRR and identity domain handling fixes for the Intel VT-d driver. - More small fixes and cleanups. * tag 'iommu-updates-v5.6' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (87 commits) iommu/amd: Remove the unnecessary assignment iommu/vt-d: Remove unnecessary WARN_ON_ONCE() iommu/vt-d: Unnecessary to handle default identity domain iommu/vt-d: Allow devices with RMRRs to use identity domain iommu/vt-d: Add RMRR base and end addresses sanity check iommu/vt-d: Mark firmware tainted if RMRR fails sanity check iommu/amd: Remove unused struct member iommu/amd: Replace two consecutive readl calls with one readq iommu/vt-d: Don't reject Host Bridge due to scope mismatch PCI/ATS: Add PASID stubs iommu/arm-smmu-v3: Return -EBUSY when trying to re-add a device iommu/arm-smmu-v3: Improve add_device() error handling iommu/arm-smmu-v3: Use WRITE_ONCE() when changing validity of an STE iommu/arm-smmu-v3: Add second level of context descriptor table iommu/arm-smmu-v3: Prepare for handling arm_smmu_write_ctx_desc() failure iommu/arm-smmu-v3: Propagate ssid_bits iommu/arm-smmu-v3: Add support for Substream IDs iommu/arm-smmu-v3: Add context descriptor tables allocators iommu/arm-smmu-v3: Prepare arm_smmu_s1_cfg for SSID support ACPI/IORT: Parse SSID property of named component node ...

Linus Torvalds committed 6 years ago

Commit 4fc2ea62 parents d271ab2 e3b5ee036 files · +1686 −822Browse files

Unified · Split

36 files

Documentation/devicetree/bindings/iommu/iommu.txt +6 −0
drivers/acpi/arm64/iort.c +20 −2
drivers/iommu/Kconfig +30 −5
drivers/iommu/Makefile +2 −1
drivers/iommu/amd_iommu.c +1 −11
drivers/iommu/amd_iommu_init.c +25 −30
drivers/iommu/amd_iommu_types.h +3 −4
drivers/iommu/arm-smmu-impl.c +1 −1
drivers/iommu/arm-smmu-v3.c +450 −150
drivers/iommu/arm-smmu.c +198 −136
drivers/iommu/arm-smmu.h +141 −87
drivers/iommu/dmar.c +42 −2
drivers/iommu/intel-iommu-debugfs.c +75 −0
drivers/iommu/intel-iommu.c +275 −91
drivers/iommu/intel-pasid.c +27 −70
drivers/iommu/intel-pasid.h +6 −0
drivers/iommu/intel-svm.c +99 −72
drivers/iommu/io-pgtable-arm-v7s.c +9 −13
drivers/iommu/io-pgtable-arm.c +77 −87
drivers/iommu/io-pgtable.c +1 −1
drivers/iommu/iommu-sysfs.c +5 −0
drivers/iommu/iommu.c +49 −2
drivers/iommu/iova.c +1 −1
drivers/iommu/ipmmu-vmsa.c +1 −1
drivers/iommu/msm_iommu.c +2 −2
drivers/iommu/mtk_iommu.c +2 −2
drivers/iommu/of_iommu.c +18 −7
drivers/iommu/qcom_iommu.c +12 −13
drivers/iommu/virtio-iommu.c +3 −11
drivers/pci/ats.c +2 −0
drivers/pci/pci.c +1 −0
include/linux/intel-iommu.h +18 −7
include/linux/io-pgtable.h +23 −4
include/linux/iommu.h +16 −3
include/linux/pci-ats.h +3 −0
include/trace/events/intel_iommu.h +42 −6

Documentation/devicetree/bindings/iommu/iommu.txt+6−0

@@ -86,6 +86,12 @@ have a means to turn off translation. But it is invalid in such cases to · 85 unmodified lines
86	86	disable the IOMMU's device tree node in the first place because it would
87	87	prevent any driver from properly setting up the translations.
88	88
	89	+Optional properties:
	90	+--------------------
	91	+- pasid-num-bits: Some masters support multiple address spaces for DMA, by
	92	+ tagging DMA transactions with an address space identifier. By default,
	93	+ this is 0, which means that the device only has one address space.
	94	+
89	95
90	96	Notes:
91	97	======

drivers/acpi/arm64/iort.c+20−2

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#define pr_fmt(fmt) "ACPI: IORT: " fmt
12	12
13	13	#include <linux/acpi_iort.h>
	14	+#include <linux/bitfield.h>
14	15	#include <linux/iommu.h>
15	16	#include <linux/kernel.h>
16	17	#include <linux/list.h>
@@ -902,9 +903,9 @@ static inline bool iort_iommu_driver_enabled(u8 type) · 885 unmodified lines
902	903	{
903	904	switch (type) {
904	905	case ACPI_IORT_NODE_SMMU_V3:
905		− return IS_BUILTIN(CONFIG_ARM_SMMU_V3);
	906	+ return IS_ENABLED(CONFIG_ARM_SMMU_V3);
906	907	case ACPI_IORT_NODE_SMMU:
907		− return IS_BUILTIN(CONFIG_ARM_SMMU);
	908	+ return IS_ENABLED(CONFIG_ARM_SMMU);
908	909	default:
909	910	pr_warn("IORT node type %u does not describe an SMMU\n", type);
910	911	return false;
@@ -976,6 +977,20 @@ static int iort_pci_iommu_init(struct pci_dev pdev, u16 alias, void data) · 65 unmodified lines
976	977	return iort_iommu_xlate(info->dev, parent, streamid);
977	978	}
978	979
	980	+static void iort_named_component_init(struct device *dev,
	981	+ struct acpi_iort_node *node)
	982	+{
	983	+ struct acpi_iort_named_component *nc;
	984	+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
	985	+
	986	+ if (!fwspec)
	987	+ return;
	988	+
	989	+ nc = (struct acpi_iort_named_component *)node->node_data;
	990	+ fwspec->num_pasid_bits = FIELD_GET(ACPI_IORT_NC_PASID_BITS,
	991	+ nc->node_flags);
	992	+}
	993	+
979	994	/**
980	995	* iort_iommu_configure - Set-up IOMMU configuration for a device.
981	996	*
@@ -1030,6 +1045,9 @@ const struct iommu_ops iort_iommu_configure(struct device dev) · 48 unmodified lines
1030	1045	if (parent)
1031	1046	err = iort_iommu_xlate(dev, parent, streamid);
1032	1047	} while (parent && !err);
	1048	+
	1049	+ if (!err)
	1050	+ iort_named_component_init(dev, node);
1033	1051	}
1034	1052
1035	1053	/*

drivers/iommu/Kconfig+30−5

@@ -82,7 +82,7 @@ config IOMMU_DEBUGFS · 81 unmodified lines
82	82	config IOMMU_DEFAULT_PASSTHROUGH
83	83	bool "IOMMU passthrough by default"
84	84	depends on IOMMU_API
85		− help
	85	+ help
86	86	Enable passthrough by default, removing the need to pass in
87	87	iommu.passthrough=on or iommu=pt through command line. If this
88	88	is enabled, you can still disable with iommu.passthrough=off
@@ -91,8 +91,8 @@ config IOMMU_DEFAULT_PASSTHROUGH · 2 unmodified lines
91	91	If unsure, say N here.
92	92
93	93	config OF_IOMMU
94		− def_bool y
95		− depends on OF && IOMMU_API
	94	+ def_bool y
	95	+ depends on OF && IOMMU_API
96	96
97	97	# IOMMU-agnostic DMA-mapping layer
98	98	config IOMMU_DMA
@@ -214,6 +214,7 @@ config INTEL_IOMMU_SVM · 115 unmodified lines
214	214	select PCI_PASID
215	215	select PCI_PRI
216	216	select MMU_NOTIFIER
	217	+ select IOASID
217	218	help
218	219	Shared Virtual Memory (SVM) provides a facility for devices
219	220	to access DMA resources through process address space by
@@ -248,6 +249,18 @@ config INTEL_IOMMU_FLOPPY_WA · 28 unmodified lines
248	249	workaround will setup a 1:1 mapping for the first
249	250	16MiB to make floppy (an ISA device) work.
250	251
	252	+config INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON
	253	+ bool "Enable Intel IOMMU scalable mode by default"
	254	+ depends on INTEL_IOMMU
	255	+ help
	256	+ Selecting this option will enable by default the scalable mode if
	257	+ hardware presents the capability. The scalable mode is defined in
	258	+ VT-d 3.0. The scalable mode capability could be checked by reading
	259	+ /sys/devices/virtual/iommu/dmar*/intel-iommu/ecap. If this option
	260	+ is not selected, scalable mode support could also be enabled by
	261	+ passing intel_iommu=sm_on to the kernel. If not sure, please use
	262	+ the default value.
	263	+
251	264	config IRQ_REMAP
252	265	bool "Support for Interrupt Remapping"
253	266	depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI
@@ -356,7 +369,7 @@ config SPAPR_TCE_IOMMU · 102 unmodified lines
356	369
357	370	# ARM IOMMU support
358	371	config ARM_SMMU
359		− bool "ARM Ltd. System MMU (SMMU) Support"
	372	+ tristate "ARM Ltd. System MMU (SMMU) Support"
360	373	depends on (ARM64 \|\| ARM) && MMU
361	374	select IOMMU_API
362	375	select IOMMU_IO_PGTABLE_LPAE
@@ -368,6 +381,18 @@ config ARM_SMMU · 5 unmodified lines
368	381	Say Y here if your SoC includes an IOMMU device implementing
369	382	the ARM SMMU architecture.
370	383
	384	+config ARM_SMMU_LEGACY_DT_BINDINGS
	385	+ bool "Support the legacy \"mmu-masters\" devicetree bindings"
	386	+ depends on ARM_SMMU=y && OF
	387	+ help
	388	+ Support for the badly designed and deprecated "mmu-masters"
	389	+ devicetree bindings. This allows some DMA masters to attach
	390	+ to the SMMU but does not provide any support via the DMA API.
	391	+ If you're lucky, you might be able to get VFIO up and running.
	392	+
	393	+ If you say Y here then you'll make me very sad. Instead, say N
	394	+ and move your firmware to the utopian future that was 2016.
	395	+
371	396	config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT
372	397	bool "Default to disabling bypass on ARM SMMU v1 and v2"
373	398	depends on ARM_SMMU
@@ -394,7 +419,7 @@ config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT · 20 unmodified lines
394	419	config.
395	420
396	421	config ARM_SMMU_V3
397		− bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
	422	+ tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
398	423	depends on ARM64
399	424	select IOMMU_API
400	425	select IOMMU_IO_PGTABLE_LPAE

drivers/iommu/Makefile+2−1

@@ -14,7 +14,8 @@ obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o · 13 unmodified lines
14	14	obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o amd_iommu_quirks.o
15	15	obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o
16	16	obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
17		−obj-$(CONFIG_ARM_SMMU) += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o
	17	+obj-$(CONFIG_ARM_SMMU) += arm-smmu-mod.o
	18	+arm-smmu-mod-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o
18	19	obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o
19	20	obj-$(CONFIG_DMAR_TABLE) += dmar.o
20	21	obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o

drivers/iommu/amd_iommu.c+1−11

@@ -2294,7 +2294,6 @@ int __init amd_iommu_init_api(void) · 2293 unmodified lines
2294	2294	int __init amd_iommu_init_dma_ops(void)
2295	2295	{
2296	2296	swiotlb = (iommu_default_passthrough() \|\| sme_me_mask) ? 1 : 0;
2297		− iommu_detected = 1;
2298	2297
2299	2298	if (amd_iommu_unmap_flush)
2300	2299	pr_info("IO/TLB flush on unmap enabled\n");
@@ -2638,15 +2637,6 @@ static void amd_iommu_get_resv_regions(struct device *dev, · 337 unmodified lines
2638	2637	list_add_tail(&region->list, head);
2639	2638	}
2640	2639
2641		−static void amd_iommu_put_resv_regions(struct device *dev,
2642		− struct list_head *head)
2643		−{
2644		− struct iommu_resv_region entry, next;
2645		−
2646		− list_for_each_entry_safe(entry, next, head, list)
2647		− kfree(entry);
2648		−}
2649		−
2650	2640	static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
2651	2641	struct device *dev)
2652	2642	{
@@ -2685,7 +2675,7 @@ const struct iommu_ops amd_iommu_ops = { · 32 unmodified lines
2685	2675	.device_group = amd_iommu_device_group,
2686	2676	.domain_get_attr = amd_iommu_domain_get_attr,
2687	2677	.get_resv_regions = amd_iommu_get_resv_regions,
2688		− .put_resv_regions = amd_iommu_put_resv_regions,
	2678	+ .put_resv_regions = generic_iommu_put_resv_regions,
2689	2679	.is_attach_deferred = amd_iommu_is_attach_deferred,
2690	2680	.pgsize_bitmap = AMD_IOMMU_PGSIZES,
2691	2681	.flush_iotlb_all = amd_iommu_flush_iotlb_all,

drivers/iommu/amd_iommu_init.c+25−30

@@ -71,6 +71,8 @@ · 70 unmodified lines
71	71	#define IVHD_FLAG_ISOC_EN_MASK 0x08
72	72
73	73	#define IVMD_FLAG_EXCL_RANGE 0x08
	74	+#define IVMD_FLAG_IW 0x04
	75	+#define IVMD_FLAG_IR 0x02
74	76	#define IVMD_FLAG_UNITY_MAP 0x01
75	77
76	78	#define ACPI_DEVFLAG_INITPASS 0x01
@@ -147,7 +149,7 @@ bool amd_iommu_dump; · 70 unmodified lines
147	149	bool amd_iommu_irq_remap __read_mostly;
148	150
149	151	int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
150		−static int amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
	152	+static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
151	153
152	154	static bool amd_iommu_detected;
153	155	static bool __initdata amd_iommu_disabled;
@@ -714,7 +716,7 @@ static void iommu_enable_ppr_log(struct amd_iommu *iommu) · 560 unmodified lines
714	716	writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
715	717	writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
716	718
717		− iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);
	719	+ iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);
718	720	iommu_feature_enable(iommu, CONTROL_PPR_EN);
719	721	}
720	722
@@ -1116,21 +1118,17 @@ static int __init add_early_maps(void) · 395 unmodified lines
1116	1118	*/
1117	1119	static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
1118	1120	{
1119		− struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
1120		−
1121	1121	if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
1122	1122	return;
1123	1123
1124		− if (iommu) {
1125		− /*
1126		− * We only can configure exclusion ranges per IOMMU, not
1127		− * per device. But we can enable the exclusion range per
1128		− * device. This is done here
1129		− */
1130		− set_dev_entry_bit(devid, DEV_ENTRY_EX);
1131		− iommu->exclusion_start = m->range_start;
1132		− iommu->exclusion_length = m->range_length;
1133		− }
	1124	+ /*
	1125	+ * Treat per-device exclusion ranges as r/w unity-mapped regions
	1126	+ * since some buggy BIOSes might lead to the overwritten exclusion
	1127	+ * range (exclusion_start and exclusion_length members). This
	1128	+ * happens when there are multiple exclusion ranges (IVMD entries)
	1129	+ * defined in ACPI table.
	1130	+ */
	1131	+ m->flags = (IVMD_FLAG_IW \| IVMD_FLAG_IR \| IVMD_FLAG_UNITY_MAP);
1134	1132	}
1135	1133
1136	1134	/*
@@ -1523,8 +1521,6 @@ static int __init init_iommu_one(struct amd_iommu iommu, struct ivhd_header h) · 386 unmodified lines
1523	1521	iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1524	1522	if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
1525	1523	amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1526		− if (((h->efr_attr & (0x1 << IOMMU_FEAT_XTSUP_SHIFT)) == 0))
1527		− amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
1528	1524	break;
1529	1525	case 0x11:
1530	1526	case 0x40:
@@ -1534,8 +1530,15 @@ static int __init init_iommu_one(struct amd_iommu iommu, struct ivhd_header h) · 3 unmodified lines
1534	1530	iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1535	1531	if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0))
1536	1532	amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1537		− if (((h->efr_reg & (0x1 << IOMMU_EFR_XTSUP_SHIFT)) == 0))
1538		− amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
	1533	+ /*
	1534	+ * Note: Since iommu_update_intcapxt() leverages
	1535	+ * the IOMMU MMIO access to MSI capability block registers
	1536	+ * for MSI address lo/hi/data, we need to check both
	1537	+ * EFR[XtSup] and EFR[MsiCapMmioSup] for x2APIC support.
	1538	+ */
	1539	+ if ((h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) &&
	1540	+ (h->efr_reg & BIT(IOMMU_EFR_MSICAPMMIOSUP_SHIFT)))
	1541	+ amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
1539	1542	break;
1540	1543	default:
1541	1544	return -EINVAL;
@@ -1727,7 +1730,6 @@ static const struct attribute_group *amd_iommu_groups[] = { · 185 unmodified lines
1727	1730	static int __init iommu_init_pci(struct amd_iommu *iommu)
1728	1731	{
1729	1732	int cap_ptr = iommu->cap_ptr;
1730		− u32 range, misc, low, high;
1731	1733	int ret;
1732	1734
1733	1735	iommu->dev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(iommu->devid),
@@ -1740,19 +1742,12 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) · 6 unmodified lines
1740	1742
1741	1743	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
1742	1744	&iommu->cap);
1743		− pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,
1744		− &range);
1745		− pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,
1746		− &misc);
1747	1745
1748	1746	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
1749	1747	amd_iommu_iotlb_sup = false;
1750	1748
1751	1749	/* read extended feature bits */
1752		− low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
1753		− high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
1754		−
1755		− iommu->features = ((u64)high << 32) \| low;
	1750	+ iommu->features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);
1756	1751
1757	1752	if (iommu_feature(iommu, FEATURE_GT)) {
1758	1753	int glxval;
@@ -1996,8 +1991,8 @@ static int iommu_init_intcapxt(struct amd_iommu *iommu) · 237 unmodified lines
1996	1991	struct irq_affinity_notify *notify = &iommu->intcapxt_notify;
1997	1992
1998	1993	/**
1999		− * IntCapXT requires XTSup=1, which can be inferred
2000		− * amd_iommu_xt_mode.
	1994	+ * IntCapXT requires XTSup=1 and MsiCapMmioSup=1,
	1995	+ * which can be inferred from amd_iommu_xt_mode.
2001	1996	*/
2002	1997	if (amd_iommu_xt_mode != IRQ_REMAP_X2APIC_MODE)
2003	1998	return 0;
@@ -2044,7 +2039,7 @@ enable_faults: · 40 unmodified lines
2044	2039	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
2045	2040
2046	2041	if (iommu->ppr_log != NULL)
2047		− iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
	2042	+ iommu_feature_enable(iommu, CONTROL_PPRINT_EN);
2048	2043
2049	2044	iommu_ga_log_enable(iommu);
2050	2045

drivers/iommu/amd_iommu_types.h+3−4

@@ -147,8 +147,8 @@ · 146 unmodified lines
147	147	#define CONTROL_COHERENT_EN 0x0aULL
148	148	#define CONTROL_ISOC_EN 0x0bULL
149	149	#define CONTROL_CMDBUF_EN 0x0cULL
150		−#define CONTROL_PPFLOG_EN 0x0dULL
151		−#define CONTROL_PPFINT_EN 0x0eULL
	150	+#define CONTROL_PPRLOG_EN 0x0dULL
	151	+#define CONTROL_PPRINT_EN 0x0eULL
152	152	#define CONTROL_PPR_EN 0x0fULL
153	153	#define CONTROL_GT_EN 0x10ULL
154	154	#define CONTROL_GA_EN 0x11ULL
@@ -377,12 +377,12 @@ · 222 unmodified lines
377	377	#define IOMMU_CAP_EFR 27
378	378
379	379	/* IOMMU Feature Reporting Field (for IVHD type 10h */
380		−#define IOMMU_FEAT_XTSUP_SHIFT 0
381	380	#define IOMMU_FEAT_GASUP_SHIFT 6
382	381
383	382	/* IOMMU Extended Feature Register (EFR) */
384	383	#define IOMMU_EFR_XTSUP_SHIFT 2
385	384	#define IOMMU_EFR_GASUP_SHIFT 7
	385	+#define IOMMU_EFR_MSICAPMMIOSUP_SHIFT 46
386	386
387	387	#define MAX_DOMAIN_ID 65536
388	388
@@ -463,7 +463,6 @@ struct amd_irte_ops; · 74 unmodified lines
463	463	* independent of their use.
464	464	*/
465	465	struct protection_domain {
466		− struct list_head list; /* for list of all protection domains */
467	466	struct list_head dev_list; /* List of all devices in this domain */
468	467	struct iommu_domain domain; /* generic domain handle used by
469	468	iommu core code */

drivers/iommu/arm-smmu-impl.c+1−1

@@ -119,7 +119,7 @@ int arm_mmu500_reset(struct arm_smmu_device *smmu) · 118 unmodified lines
119	119	* Secure has also cleared SACR.CACHE_LOCK for this to take effect...
120	120	*/
121	121	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
122		− major = FIELD_GET(ID7_MAJOR, reg);
	122	+ major = FIELD_GET(ARM_SMMU_ID7_MAJOR, reg);
123	123	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
124	124	if (major >= 2)
125	125	reg &= ~ARM_MMU500_ACR_CACHE_LOCK;

drivers/iommu/arm-smmu-v3.c+450−150

@@ -21,8 +21,7 @@ · 20 unmodified lines
21	21	#include <linux/io-pgtable.h>
22	22	#include <linux/iommu.h>
23	23	#include <linux/iopoll.h>
24		−#include <linux/init.h>
25		−#include <linux/moduleparam.h>
	24	+#include <linux/module.h>
26	25	#include <linux/msi.h>
27	26	#include <linux/of.h>
28	27	#include <linux/of_address.h>
@@ -224,9 +223,15 @@ · 195 unmodified lines
224	223
225	224	#define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
226	225	#define STRTAB_STE_0_S1FMT_LINEAR 0
	226	+#define STRTAB_STE_0_S1FMT_64K_L2 2
227	227	#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
228	228	#define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
229	229
	230	+#define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
	231	+#define STRTAB_STE_1_S1DSS_TERMINATE 0x0
	232	+#define STRTAB_STE_1_S1DSS_BYPASS 0x1
	233	+#define STRTAB_STE_1_S1DSS_SSID0 0x2
	234	+
230	235	#define STRTAB_STE_1_S1C_CACHE_NC 0UL
231	236	#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
232	237	#define STRTAB_STE_1_S1C_CACHE_WT 2UL
@@ -251,6 +256,13 @@ · 18 unmodified lines
251	256
252	257	#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
253	258	#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
	259	+#define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
	260	+#define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
	261	+#define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
	262	+#define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
	263	+#define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
	264	+#define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
	265	+#define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
254	266	#define STRTAB_STE_2_S2AA64 (1UL << 51)
255	267	#define STRTAB_STE_2_S2ENDI (1UL << 52)
256	268	#define STRTAB_STE_2_S2PTW (1UL << 54)
@@ -258,30 +270,34 @@ · 1 unmodified line
258	270
259	271	#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
260	272
261		−/* Context descriptor (stage-1 only) */
	273	+/*
	274	+ * Context descriptors.
	275	+ *
	276	+ * Linear: when less than 1024 SSIDs are supported
	277	+ * 2lvl: at most 1024 L1 entries,
	278	+ * 1024 lazy entries per table.
	279	+ */
	280	+#define CTXDESC_SPLIT 10
	281	+#define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
	282	+
	283	+#define CTXDESC_L1_DESC_DWORDS 1
	284	+#define CTXDESC_L1_DESC_V (1UL << 0)
	285	+#define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
	286	+
262	287	#define CTXDESC_CD_DWORDS 8
263	288	#define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
264		−#define ARM64_TCR_T0SZ GENMASK_ULL(5, 0)
265	289	#define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
266		−#define ARM64_TCR_TG0 GENMASK_ULL(15, 14)
267	290	#define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
268		−#define ARM64_TCR_IRGN0 GENMASK_ULL(9, 8)
269	291	#define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
270		−#define ARM64_TCR_ORGN0 GENMASK_ULL(11, 10)
271	292	#define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
272		−#define ARM64_TCR_SH0 GENMASK_ULL(13, 12)
273	293	#define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
274		−#define ARM64_TCR_EPD0 (1ULL << 7)
275	294	#define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
276		−#define ARM64_TCR_EPD1 (1ULL << 23)
277	295
278	296	#define CTXDESC_CD_0_ENDI (1UL << 15)
279	297	#define CTXDESC_CD_0_V (1UL << 31)
280	298
281	299	#define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
282		−#define ARM64_TCR_IPS GENMASK_ULL(34, 32)
283	300	#define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
284		−#define ARM64_TCR_TBI0 (1ULL << 37)
285	301
286	302	#define CTXDESC_CD_0_AA64 (1UL << 41)
287	303	#define CTXDESC_CD_0_S (1UL << 44)
@@ -292,9 +308,11 @@ · 4 unmodified lines
292	308
293	309	#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
294	310
295		−/* Convert between AArch64 (CPU) TCR format and SMMU CD format */
296		−#define ARM_SMMU_TCR2CD(tcr, fld) FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \
297		− FIELD_GET(ARM64_TCR_##fld, tcr))
	311	+/*
	312	+ * When the SMMU only supports linear context descriptor tables, pick a
	313	+ * reasonable size limit (64kB).
	314	+ */
	315	+#define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
298	316
299	317	/* Command queue */
300	318	#define CMDQ_ENT_SZ_SHIFT 4
@@ -323,6 +341,7 @@ · 22 unmodified lines
323	341	#define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
324	342	#define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
325	343
	344	+#define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
326	345	#define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
327	346	#define CMDQ_CFGI_1_LEAF (1UL << 0)
328	347	#define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
@@ -384,10 +403,6 @@ · 55 unmodified lines
384	403	#define MSI_IOVA_BASE 0x8000000
385	404	#define MSI_IOVA_LENGTH 0x100000
386	405
387		−/*
388		− * not really modular, but the easiest way to keep compat with existing
389		− * bootargs behaviour is to continue using module_param_named here.
390		− */
391	406	static bool disable_bypass = 1;
392	407	module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
393	408	MODULE_PARM_DESC(disable_bypass,
@@ -440,8 +455,11 @@ struct arm_smmu_cmdq_ent { · 46 unmodified lines
440	455
441	456	#define CMDQ_OP_CFGI_STE 0x3
442	457	#define CMDQ_OP_CFGI_ALL 0x4
	458	+ #define CMDQ_OP_CFGI_CD 0x5
	459	+ #define CMDQ_OP_CFGI_CD_ALL 0x6
443	460	struct {
444	461	u32 sid;
	462	+ u32 ssid;
445	463	union {
446	464	bool leaf;
447	465	u8 span;
@@ -547,16 +565,30 @@ struct arm_smmu_strtab_l1_desc { · 99 unmodified lines
547	565	dma_addr_t l2ptr_dma;
548	566	};
549	567
	568	+struct arm_smmu_ctx_desc {
	569	+ u16 asid;
	570	+ u64 ttbr;
	571	+ u64 tcr;
	572	+ u64 mair;
	573	+};
	574	+
	575	+struct arm_smmu_l1_ctx_desc {
	576	+ __le64 *l2ptr;
	577	+ dma_addr_t l2ptr_dma;
	578	+};
	579	+
	580	+struct arm_smmu_ctx_desc_cfg {
	581	+ __le64 *cdtab;
	582	+ dma_addr_t cdtab_dma;
	583	+ struct arm_smmu_l1_ctx_desc *l1_desc;
	584	+ unsigned int num_l1_ents;
	585	+};
	586	+
550	587	struct arm_smmu_s1_cfg {
551		− __le64 *cdptr;
552		− dma_addr_t cdptr_dma;
553		−
554		− struct arm_smmu_ctx_desc {
555		− u16 asid;
556		− u64 ttbr;
557		− u64 tcr;
558		− u64 mair;
559		− } cd;
	588	+ struct arm_smmu_ctx_desc_cfg cdcfg;
	589	+ struct arm_smmu_ctx_desc cd;
	590	+ u8 s1fmt;
	591	+ u8 s1cdmax;
560	592	};
561	593
562	594	struct arm_smmu_s2_cfg {
@@ -638,6 +670,7 @@ struct arm_smmu_master { · 75 unmodified lines
638	670	u32 *sids;
639	671	unsigned int num_sids;
640	672	bool ats_enabled;
	673	+ unsigned int ssid_bits;
641	674	};
642	675
643	676	/* SMMU private data for an IOMMU domain */
@@ -847,15 +880,22 @@ static int arm_smmu_cmdq_build_cmd(u64 cmd, struct arm_smmu_cmdq_ent ent) · 203 unmodified lines
847	880	cmd[1] \|= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
848	881	cmd[1] \|= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
849	882	break;
	883	+ case CMDQ_OP_CFGI_CD:
	884	+ cmd[0] \|= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
	885	+ /* Fallthrough */
850	886	case CMDQ_OP_CFGI_STE:
851	887	cmd[0] \|= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
852	888	cmd[1] \|= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
853	889	break;
	890	+ case CMDQ_OP_CFGI_CD_ALL:
	891	+ cmd[0] \|= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
	892	+ break;
854	893	case CMDQ_OP_CFGI_ALL:
855	894	/* Cover the entire SID range */
856	895	cmd[1] \|= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
857	896	break;
858	897	case CMDQ_OP_TLBI_NH_VA:
	898	+ cmd[0] \|= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
859	899	cmd[0] \|= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
860	900	cmd[1] \|= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
861	901	cmd[1] \|= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
@@ -1443,50 +1483,238 @@ static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu) · 581 unmodified lines
1443	1483	}
1444	1484
1445	1485	/* Context descriptor manipulation functions */
1446		−static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
	1486	+static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
	1487	+ int ssid, bool leaf)
1447	1488	{
1448		− u64 val = 0;
	1489	+ size_t i;
	1490	+ unsigned long flags;
	1491	+ struct arm_smmu_master *master;
	1492	+ struct arm_smmu_device *smmu = smmu_domain->smmu;
	1493	+ struct arm_smmu_cmdq_ent cmd = {
	1494	+ .opcode = CMDQ_OP_CFGI_CD,
	1495	+ .cfgi = {
	1496	+ .ssid = ssid,
	1497	+ .leaf = leaf,
	1498	+ },
	1499	+ };
	1500	+
	1501	+ spin_lock_irqsave(&smmu_domain->devices_lock, flags);
	1502	+ list_for_each_entry(master, &smmu_domain->devices, domain_head) {
	1503	+ for (i = 0; i < master->num_sids; i++) {
	1504	+ cmd.cfgi.sid = master->sids[i];
	1505	+ arm_smmu_cmdq_issue_cmd(smmu, &cmd);
	1506	+ }
	1507	+ }
	1508	+ spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
	1509	+
	1510	+ arm_smmu_cmdq_issue_sync(smmu);
	1511	+}
1449	1512
1450		− /* Repack the TCR. Just care about TTBR0 for now */
1451		− val \|= ARM_SMMU_TCR2CD(tcr, T0SZ);
1452		− val \|= ARM_SMMU_TCR2CD(tcr, TG0);
1453		− val \|= ARM_SMMU_TCR2CD(tcr, IRGN0);
1454		− val \|= ARM_SMMU_TCR2CD(tcr, ORGN0);
1455		− val \|= ARM_SMMU_TCR2CD(tcr, SH0);
1456		− val \|= ARM_SMMU_TCR2CD(tcr, EPD0);
1457		− val \|= ARM_SMMU_TCR2CD(tcr, EPD1);
1458		− val \|= ARM_SMMU_TCR2CD(tcr, IPS);
	1513	+static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
	1514	+ struct arm_smmu_l1_ctx_desc *l1_desc)
	1515	+{
	1516	+ size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1459	1517
1460		− return val;
	1518	+ l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
	1519	+ &l1_desc->l2ptr_dma, GFP_KERNEL);
	1520	+ if (!l1_desc->l2ptr) {
	1521	+ dev_warn(smmu->dev,
	1522	+ "failed to allocate context descriptor table\n");
	1523	+ return -ENOMEM;
	1524	+ }
	1525	+ return 0;
1461	1526	}
1462	1527
1463		−static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
1464		− struct arm_smmu_s1_cfg *cfg)
	1528	+static void arm_smmu_write_cd_l1_desc(__le64 *dst,
	1529	+ struct arm_smmu_l1_ctx_desc *l1_desc)
1465	1530	{
1466		− u64 val;
	1531	+ u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) \|
	1532	+ CTXDESC_L1_DESC_V;
	1533	+
	1534	+ WRITE_ONCE(*dst, cpu_to_le64(val));
	1535	+}
	1536	+
	1537	+static __le64 arm_smmu_get_cd_ptr(struct arm_smmu_domain smmu_domain,
	1538	+ u32 ssid)
	1539	+{
	1540	+ __le64 *l1ptr;
	1541	+ unsigned int idx;
	1542	+ struct arm_smmu_l1_ctx_desc *l1_desc;
	1543	+ struct arm_smmu_device *smmu = smmu_domain->smmu;
	1544	+ struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1467	1545
	1546	+ if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
	1547	+ return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
	1548	+
	1549	+ idx = ssid >> CTXDESC_SPLIT;
	1550	+ l1_desc = &cdcfg->l1_desc[idx];
	1551	+ if (!l1_desc->l2ptr) {
	1552	+ if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
	1553	+ return NULL;
	1554	+
	1555	+ l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
	1556	+ arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
	1557	+ /* An invalid L1CD can be cached */
	1558	+ arm_smmu_sync_cd(smmu_domain, ssid, false);
	1559	+ }
	1560	+ idx = ssid & (CTXDESC_L2_ENTRIES - 1);
	1561	+ return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
	1562	+}
	1563	+
	1564	+static int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain,
	1565	+ int ssid, struct arm_smmu_ctx_desc *cd)
	1566	+{
1468	1567	/*
1469		− * We don't need to issue any invalidation here, as we'll invalidate
1470		− * the STE when installing the new entry anyway.
	1568	+ * This function handles the following cases:
	1569	+ *
	1570	+ * (1) Install primary CD, for normal DMA traffic (SSID = 0).
	1571	+ * (2) Install a secondary CD, for SID+SSID traffic.
	1572	+ * (3) Update ASID of a CD. Atomically write the first 64 bits of the
	1573	+ * CD, then invalidate the old entry and mappings.
	1574	+ * (4) Remove a secondary CD.
1471	1575	*/
1472		− val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) \|
	1576	+ u64 val;
	1577	+ bool cd_live;
	1578	+ __le64 *cdptr;
	1579	+ struct arm_smmu_device *smmu = smmu_domain->smmu;
	1580	+
	1581	+ if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
	1582	+ return -E2BIG;
	1583	+
	1584	+ cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
	1585	+ if (!cdptr)
	1586	+ return -ENOMEM;
	1587	+
	1588	+ val = le64_to_cpu(cdptr[0]);
	1589	+ cd_live = !!(val & CTXDESC_CD_0_V);
	1590	+
	1591	+ if (!cd) { /* (4) */
	1592	+ val = 0;
	1593	+ } else if (cd_live) { /* (3) */
	1594	+ val &= ~CTXDESC_CD_0_ASID;
	1595	+ val \|= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
	1596	+ /*
	1597	+ * Until CD+TLB invalidation, both ASIDs may be used for tagging
	1598	+ * this substream's traffic
	1599	+ */
	1600	+ } else { /* (1) and (2) */
	1601	+ cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
	1602	+ cdptr[2] = 0;
	1603	+ cdptr[3] = cpu_to_le64(cd->mair);
	1604	+
	1605	+ /*
	1606	+ * STE is live, and the SMMU might read dwords of this CD in any
	1607	+ * order. Ensure that it observes valid values before reading
	1608	+ * V=1.
	1609	+ */
	1610	+ arm_smmu_sync_cd(smmu_domain, ssid, true);
	1611	+
	1612	+ val = cd->tcr \|
1473	1613	#ifdef __BIG_ENDIAN
1474		− CTXDESC_CD_0_ENDI \|
	1614	+ CTXDESC_CD_0_ENDI \|
1475	1615	#endif
1476		− CTXDESC_CD_0_R \| CTXDESC_CD_0_A \| CTXDESC_CD_0_ASET \|
1477		− CTXDESC_CD_0_AA64 \| FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) \|
1478		− CTXDESC_CD_0_V;
	1616	+ CTXDESC_CD_0_R \| CTXDESC_CD_0_A \| CTXDESC_CD_0_ASET \|
	1617	+ CTXDESC_CD_0_AA64 \|
	1618	+ FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) \|
	1619	+ CTXDESC_CD_0_V;
1479	1620
1480		− /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1481		− if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1482		− val \|= CTXDESC_CD_0_S;
	1621	+ /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
	1622	+ if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
	1623	+ val \|= CTXDESC_CD_0_S;
	1624	+ }
1483	1625
1484		− cfg->cdptr[0] = cpu_to_le64(val);
	1626	+ /*
	1627	+ * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
	1628	+ * "Configuration structures and configuration invalidation completion"
	1629	+ *
	1630	+ * The size of single-copy atomic reads made by the SMMU is
	1631	+ * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single
	1632	+ * field within an aligned 64-bit span of a structure can be altered
	1633	+ * without first making the structure invalid.
	1634	+ */
	1635	+ WRITE_ONCE(cdptr[0], cpu_to_le64(val));
	1636	+ arm_smmu_sync_cd(smmu_domain, ssid, true);
	1637	+ return 0;
	1638	+}
	1639	+
	1640	+static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
	1641	+{
	1642	+ int ret;
	1643	+ size_t l1size;
	1644	+ size_t max_contexts;
	1645	+ struct arm_smmu_device *smmu = smmu_domain->smmu;
	1646	+ struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
	1647	+ struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
	1648	+
	1649	+ max_contexts = 1 << cfg->s1cdmax;
	1650	+
	1651	+ if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) \|\|
	1652	+ max_contexts <= CTXDESC_L2_ENTRIES) {
	1653	+ cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
	1654	+ cdcfg->num_l1_ents = max_contexts;
1485	1655
1486		− val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;
1487		− cfg->cdptr[1] = cpu_to_le64(val);
	1656	+ l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
	1657	+ } else {
	1658	+ cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
	1659	+ cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
	1660	+ CTXDESC_L2_ENTRIES);
	1661	+
	1662	+ cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
	1663	+ sizeof(*cdcfg->l1_desc),
	1664	+ GFP_KERNEL);
	1665	+ if (!cdcfg->l1_desc)
	1666	+ return -ENOMEM;
	1667	+
	1668	+ l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
	1669	+ }
	1670	+
	1671	+ cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
	1672	+ GFP_KERNEL);
	1673	+ if (!cdcfg->cdtab) {
	1674	+ dev_warn(smmu->dev, "failed to allocate context descriptor\n");
	1675	+ ret = -ENOMEM;
	1676	+ goto err_free_l1;
	1677	+ }
	1678	+
	1679	+ return 0;
1488	1680
1489		− cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);
	1681	+err_free_l1:
	1682	+ if (cdcfg->l1_desc) {
	1683	+ devm_kfree(smmu->dev, cdcfg->l1_desc);
	1684	+ cdcfg->l1_desc = NULL;
	1685	+ }
	1686	+ return ret;
	1687	+}
	1688	+
	1689	+static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
	1690	+{
	1691	+ int i;
	1692	+ size_t size, l1size;
	1693	+ struct arm_smmu_device *smmu = smmu_domain->smmu;
	1694	+ struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
	1695	+
	1696	+ if (cdcfg->l1_desc) {
	1697	+ size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
	1698	+
	1699	+ for (i = 0; i < cdcfg->num_l1_ents; i++) {
	1700	+ if (!cdcfg->l1_desc[i].l2ptr)
	1701	+ continue;
	1702	+
	1703	+ dmam_free_coherent(smmu->dev, size,
	1704	+ cdcfg->l1_desc[i].l2ptr,
	1705	+ cdcfg->l1_desc[i].l2ptr_dma);
	1706	+ }
	1707	+ devm_kfree(smmu->dev, cdcfg->l1_desc);
	1708	+ cdcfg->l1_desc = NULL;
	1709	+
	1710	+ l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
	1711	+ } else {
	1712	+ l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
	1713	+ }
	1714	+
	1715	+ dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
	1716	+ cdcfg->cdtab_dma = 0;
	1717	+ cdcfg->cdtab = NULL;
1490	1718	}
1491	1719
1492	1720	/* Stream table manipulation functions */
@@ -1608,6 +1836,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, · 115 unmodified lines
1608	1836	if (s1_cfg) {
1609	1837	BUG_ON(ste_live);
1610	1838	dst[1] = cpu_to_le64(
	1839	+ FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) \|
1611	1840	FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) \|
1612	1841	FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) \|
1613	1842	FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) \|
@@ -1617,8 +1846,10 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, · 3 unmodified lines
1617	1846	!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1618	1847	dst[1] \|= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1619	1848
1620		− val \|= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) \|
1621		− FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS);
	1849	+ val \|= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) \|
	1850	+ FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) \|
	1851	+ FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) \|
	1852	+ FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
1622	1853	}
1623	1854
1624	1855	if (s2_cfg) {
@@ -1642,7 +1873,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, · 17 unmodified lines
1642	1873	STRTAB_STE_1_EATS_TRANS));
1643	1874
1644	1875	arm_smmu_sync_ste_for_sid(smmu, sid);
1645		− dst[0] = cpu_to_le64(val);
	1876	+ /* See comment in arm_smmu_write_ctx_desc() */
	1877	+ WRITE_ONCE(dst[0], cpu_to_le64(val));
1646	1878	arm_smmu_sync_ste_for_sid(smmu, sid);
1647	1879
1648	1880	/* It's likely that we'll want to use the new STE soon */
@@ -1675,7 +1907,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) · 26 unmodified lines
1675	1907
1676	1908	desc->span = STRTAB_SPLIT + 1;
1677	1909	desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1678		− GFP_KERNEL \| __GFP_ZERO);
	1910	+ GFP_KERNEL);
1679	1911	if (!desc->l2ptr) {
1680	1912	dev_err(smmu->dev,
1681	1913	"failed to allocate l2 stream table for SID %u\n",
@@ -2131,12 +2363,8 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) · 449 unmodified lines
2131	2363	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2132	2364	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2133	2365
2134		− if (cfg->cdptr) {
2135		− dmam_free_coherent(smmu_domain->smmu->dev,
2136		− CTXDESC_CD_DWORDS << 3,
2137		− cfg->cdptr,
2138		− cfg->cdptr_dma);
2139		−
	2366	+ if (cfg->cdcfg.cdtab) {
	2367	+ arm_smmu_free_cd_tables(smmu_domain);
2140	2368	arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
2141	2369	}
2142	2370	} else {
@@ -2149,55 +2377,82 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) · 6 unmodified lines
2149	2377	}
2150	2378
2151	2379	static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
	2380	+ struct arm_smmu_master *master,
2152	2381	struct io_pgtable_cfg *pgtbl_cfg)
2153	2382	{
2154	2383	int ret;
2155	2384	int asid;
2156	2385	struct arm_smmu_device *smmu = smmu_domain->smmu;
2157	2386	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
	2387	+ typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2158	2388
2159	2389	asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
2160	2390	if (asid < 0)
2161	2391	return asid;
2162	2392
2163		− cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
2164		− &cfg->cdptr_dma,
2165		− GFP_KERNEL \| __GFP_ZERO);
2166		− if (!cfg->cdptr) {
2167		− dev_warn(smmu->dev, "failed to allocate context descriptor\n");
2168		− ret = -ENOMEM;
	2393	+ cfg->s1cdmax = master->ssid_bits;
	2394	+
	2395	+ ret = arm_smmu_alloc_cd_tables(smmu_domain);
	2396	+ if (ret)
2169	2397	goto out_free_asid;
2170		− }
2171	2398
2172	2399	cfg->cd.asid = (u16)asid;
2173		− cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
2174		− cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
	2400	+ cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
	2401	+ cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) \|
	2402	+ FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) \|
	2403	+ FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) \|
	2404	+ FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) \|
	2405	+ FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) \|
	2406	+ FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) \|
	2407	+ CTXDESC_CD_0_TCR_EPD1 \| CTXDESC_CD_0_AA64;
2175	2408	cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;
	2409	+
	2410	+ /*
	2411	+ * Note that this will end up calling arm_smmu_sync_cd() before
	2412	+ * the master has been added to the devices list for this domain.
	2413	+ * This isn't an issue because the STE hasn't been installed yet.
	2414	+ */
	2415	+ ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd);
	2416	+ if (ret)
	2417	+ goto out_free_cd_tables;
	2418	+
2176	2419	return 0;
2177	2420
	2421	+out_free_cd_tables:
	2422	+ arm_smmu_free_cd_tables(smmu_domain);
2178	2423	out_free_asid:
2179	2424	arm_smmu_bitmap_free(smmu->asid_map, asid);
2180	2425	return ret;
2181	2426	}
2182	2427
2183	2428	static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
	2429	+ struct arm_smmu_master *master,
2184	2430	struct io_pgtable_cfg *pgtbl_cfg)
2185	2431	{
2186	2432	int vmid;
2187	2433	struct arm_smmu_device *smmu = smmu_domain->smmu;
2188	2434	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
	2435	+ typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;
2189	2436
2190	2437	vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
2191	2438	if (vmid < 0)
2192	2439	return vmid;
2193	2440
	2441	+ vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2194	2442	cfg->vmid = (u16)vmid;
2195	2443	cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2196		− cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
	2444	+ cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) \|
	2445	+ FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) \|
	2446	+ FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) \|
	2447	+ FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) \|
	2448	+ FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) \|
	2449	+ FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) \|
	2450	+ FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
2197	2451	return 0;
2198	2452	}
2199	2453
2200		−static int arm_smmu_domain_finalise(struct iommu_domain *domain)
	2454	+static int arm_smmu_domain_finalise(struct iommu_domain *domain,
	2455	+ struct arm_smmu_master *master)
2201	2456	{
2202	2457	int ret;
2203	2458	unsigned long ias, oas;
@@ -2205,6 +2460,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) · 1 unmodified line
2205	2460	struct io_pgtable_cfg pgtbl_cfg;
2206	2461	struct io_pgtable_ops *pgtbl_ops;
2207	2462	int (finalise_stage_fn)(struct arm_smmu_domain ,
	2463	+ struct arm_smmu_master *,
2208	2464	struct io_pgtable_cfg *);
2209	2465	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2210	2466	struct arm_smmu_device *smmu = smmu_domain->smmu;
@@ -2259,7 +2515,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) · 48 unmodified lines
2259	2515	domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2260	2516	domain->geometry.force_aperture = true;
2261	2517
2262		− ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
	2518	+ ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);
2263	2519	if (ret < 0) {
2264	2520	free_io_pgtable_ops(pgtbl_ops);
2265	2521	return ret;
@@ -2412,7 +2668,7 @@ static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev) · 146 unmodified lines
2412	2668
2413	2669	if (!smmu_domain->smmu) {
2414	2670	smmu_domain->smmu = smmu;
2415		− ret = arm_smmu_domain_finalise(domain);
	2671	+ ret = arm_smmu_domain_finalise(domain, master);
2416	2672	if (ret) {
2417	2673	smmu_domain->smmu = NULL;
2418	2674	goto out_unlock;
@@ -2424,6 +2680,13 @@ static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev) · 5 unmodified lines
2424	2680	dev_name(smmu->dev));
2425	2681	ret = -ENXIO;
2426	2682	goto out_unlock;
	2683	+ } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
	2684	+ master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
	2685	+ dev_err(dev,
	2686	+ "cannot attach to incompatible domain (%u SSID bits != %u)\n",
	2687	+ smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);
	2688	+ ret = -EINVAL;
	2689	+ goto out_unlock;
2427	2690	}
2428	2691
2429	2692	master->domain = smmu_domain;
@@ -2431,9 +2694,6 @@ static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev) · 1 unmodified line
2431	2694	if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2432	2695	master->ats_enabled = arm_smmu_ats_supported(master);
2433	2696
2434		− if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2435		− arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg);
2436		−
2437	2697	arm_smmu_install_ste_for_dev(master);
2438	2698
2439	2699	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
@@ -2534,51 +2794,66 @@ static int arm_smmu_add_device(struct device *dev) · 94 unmodified lines
2534	2794
2535	2795	if (!fwspec \|\| fwspec->ops != &arm_smmu_ops)
2536	2796	return -ENODEV;
2537		− /*
2538		− * We _can_ actually withstand dodgy bus code re-calling add_device()
2539		− * without an intervening remove_device()/of_xlate() sequence, but
2540		− * we're not going to do so quietly...
2541		− */
2542		− if (WARN_ON_ONCE(fwspec->iommu_priv)) {
2543		− master = fwspec->iommu_priv;
2544		− smmu = master->smmu;
2545		− } else {
2546		− smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2547		− if (!smmu)
2548		− return -ENODEV;
2549		− master = kzalloc(sizeof(*master), GFP_KERNEL);
2550		− if (!master)
2551		− return -ENOMEM;
2552	2797
2553		− master->dev = dev;
2554		− master->smmu = smmu;
2555		− master->sids = fwspec->ids;
2556		− master->num_sids = fwspec->num_ids;
2557		− fwspec->iommu_priv = master;
2558		− }
	2798	+ if (WARN_ON_ONCE(fwspec->iommu_priv))
	2799	+ return -EBUSY;
	2800	+
	2801	+ smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
	2802	+ if (!smmu)
	2803	+ return -ENODEV;
	2804	+
	2805	+ master = kzalloc(sizeof(*master), GFP_KERNEL);
	2806	+ if (!master)
	2807	+ return -ENOMEM;
	2808	+
	2809	+ master->dev = dev;
	2810	+ master->smmu = smmu;
	2811	+ master->sids = fwspec->ids;
	2812	+ master->num_sids = fwspec->num_ids;
	2813	+ fwspec->iommu_priv = master;
2559	2814
2560	2815	/* Check the SIDs are in range of the SMMU and our stream table */
2561	2816	for (i = 0; i < master->num_sids; i++) {
2562	2817	u32 sid = master->sids[i];
2563	2818
2564		− if (!arm_smmu_sid_in_range(smmu, sid))
2565		− return -ERANGE;
	2819	+ if (!arm_smmu_sid_in_range(smmu, sid)) {
	2820	+ ret = -ERANGE;
	2821	+ goto err_free_master;
	2822	+ }
2566	2823
2567	2824	/* Ensure l2 strtab is initialised */
2568	2825	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2569	2826	ret = arm_smmu_init_l2_strtab(smmu, sid);
2570	2827	if (ret)
2571		− return ret;
	2828	+ goto err_free_master;
2572	2829	}
2573	2830	}
2574	2831
	2832	+ master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits);
	2833	+
	2834	+ if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
	2835	+ master->ssid_bits = min_t(u8, master->ssid_bits,
	2836	+ CTXDESC_LINEAR_CDMAX);
	2837	+
	2838	+ ret = iommu_device_link(&smmu->iommu, dev);
	2839	+ if (ret)
	2840	+ goto err_free_master;
	2841	+
2575	2842	group = iommu_group_get_for_dev(dev);
2576		− if (!IS_ERR(group)) {
2577		− iommu_group_put(group);
2578		− iommu_device_link(&smmu->iommu, dev);
	2843	+ if (IS_ERR(group)) {
	2844	+ ret = PTR_ERR(group);
	2845	+ goto err_unlink;
2579	2846	}
2580	2847
2581		− return PTR_ERR_OR_ZERO(group);
	2848	+ iommu_group_put(group);
	2849	+ return 0;
	2850	+
	2851	+err_unlink:
	2852	+ iommu_device_unlink(&smmu->iommu, dev);
	2853	+err_free_master:
	2854	+ kfree(master);
	2855	+ fwspec->iommu_priv = NULL;
	2856	+ return ret;
2582	2857	}
2583	2858
2584	2859	static void arm_smmu_remove_device(struct device *dev)
@@ -2710,15 +2985,6 @@ static void arm_smmu_get_resv_regions(struct device *dev, · 125 unmodified lines
2710	2985	iommu_dma_get_resv_regions(dev, head);
2711	2986	}
2712	2987
2713		−static void arm_smmu_put_resv_regions(struct device *dev,
2714		− struct list_head *head)
2715		−{
2716		− struct iommu_resv_region entry, next;
2717		−
2718		− list_for_each_entry_safe(entry, next, head, list)
2719		− kfree(entry);
2720		−}
2721		−
2722	2988	static struct iommu_ops arm_smmu_ops = {
2723	2989	.capable = arm_smmu_capable,
2724	2990	.domain_alloc = arm_smmu_domain_alloc,
@@ -2736,7 +3002,7 @@ static struct iommu_ops arm_smmu_ops = { · 11 unmodified lines
2736	3002	.domain_set_attr = arm_smmu_domain_set_attr,
2737	3003	.of_xlate = arm_smmu_of_xlate,
2738	3004	.get_resv_regions = arm_smmu_get_resv_regions,
2739		− .put_resv_regions = arm_smmu_put_resv_regions,
	3005	+ .put_resv_regions = generic_iommu_put_resv_regions,
2740	3006	.pgsize_bitmap = -1UL, /* Restricted during device attach */
2741	3007	};
2742	3008
@@ -2883,7 +3149,7 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) · 140 unmodified lines
2883	3149
2884	3150	l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2885	3151	strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2886		− GFP_KERNEL \| __GFP_ZERO);
	3152	+ GFP_KERNEL);
2887	3153	if (!strtab) {
2888	3154	dev_err(smmu->dev,
2889	3155	"failed to allocate l1 stream table (%u bytes)\n",
@@ -2910,7 +3176,7 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) · 20 unmodified lines
2910	3176
2911	3177	size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2912	3178	strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2913		− GFP_KERNEL \| __GFP_ZERO);
	3179	+ GFP_KERNEL);
2914	3180	if (!strtab) {
2915	3181	dev_err(smmu->dev,
2916	3182	"failed to allocate linear stream table (%u bytes)\n",
@@ -3570,6 +3836,43 @@ static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) · 653 unmodified lines
3570	3836	return SZ_128K;
3571	3837	}
3572	3838
	3839	+static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
	3840	+{
	3841	+ int err;
	3842	+
	3843	+#ifdef CONFIG_PCI
	3844	+ if (pci_bus_type.iommu_ops != ops) {
	3845	+ err = bus_set_iommu(&pci_bus_type, ops);
	3846	+ if (err)
	3847	+ return err;
	3848	+ }
	3849	+#endif
	3850	+#ifdef CONFIG_ARM_AMBA
	3851	+ if (amba_bustype.iommu_ops != ops) {
	3852	+ err = bus_set_iommu(&amba_bustype, ops);
	3853	+ if (err)
	3854	+ goto err_reset_pci_ops;
	3855	+ }
	3856	+#endif
	3857	+ if (platform_bus_type.iommu_ops != ops) {
	3858	+ err = bus_set_iommu(&platform_bus_type, ops);
	3859	+ if (err)
	3860	+ goto err_reset_amba_ops;
	3861	+ }
	3862	+
	3863	+ return 0;
	3864	+
	3865	+err_reset_amba_ops:
	3866	+#ifdef CONFIG_ARM_AMBA
	3867	+ bus_set_iommu(&amba_bustype, NULL);
	3868	+#endif
	3869	+err_reset_pci_ops: __maybe_unused;
	3870	+#ifdef CONFIG_PCI
	3871	+ bus_set_iommu(&pci_bus_type, NULL);
	3872	+#endif
	3873	+ return err;
	3874	+}
	3875	+
3573	3876	static int arm_smmu_device_probe(struct platform_device *pdev)
3574	3877	{
3575	3878	int irq, ret;
@@ -3599,7 +3902,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 23 unmodified lines
3599	3902
3600	3903	/* Base address */
3601	3904	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3602		− if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) {
	3905	+ if (resource_size(res) < arm_smmu_resource_size(smmu)) {
3603	3906	dev_err(dev, "MMIO region too small (%pr)\n", res);
3604	3907	return -EINVAL;
3605	3908	}
@@ -3660,48 +3963,45 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 54 unmodified lines
3660	3963	return ret;
3661	3964	}
3662	3965
3663		−#ifdef CONFIG_PCI
3664		− if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
3665		− pci_request_acs();
3666		− ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
3667		− if (ret)
3668		− return ret;
3669		− }
3670		−#endif
3671		−#ifdef CONFIG_ARM_AMBA
3672		− if (amba_bustype.iommu_ops != &arm_smmu_ops) {
3673		− ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
3674		− if (ret)
3675		− return ret;
3676		− }
3677		−#endif
3678		− if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
3679		− ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
3680		− if (ret)
3681		− return ret;
3682		− }
3683		− return 0;
	3966	+ return arm_smmu_set_bus_ops(&arm_smmu_ops);
3684	3967	}
3685	3968
3686		−static void arm_smmu_device_shutdown(struct platform_device *pdev)
	3969	+static int arm_smmu_device_remove(struct platform_device *pdev)
3687	3970	{
3688	3971	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3689	3972
	3973	+ arm_smmu_set_bus_ops(NULL);
	3974	+ iommu_device_unregister(&smmu->iommu);
	3975	+ iommu_device_sysfs_remove(&smmu->iommu);
3690	3976	arm_smmu_device_disable(smmu);
	3977	+
	3978	+ return 0;
	3979	+}
	3980	+
	3981	+static void arm_smmu_device_shutdown(struct platform_device *pdev)
	3982	+{
	3983	+ arm_smmu_device_remove(pdev);
3691	3984	}
3692	3985
3693	3986	static const struct of_device_id arm_smmu_of_match[] = {
3694	3987	{ .compatible = "arm,smmu-v3", },
3695	3988	{ },
3696	3989	};
	3990	+MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
3697	3991
3698	3992	static struct platform_driver arm_smmu_driver = {
3699	3993	.driver = {
3700		− .name = "arm-smmu-v3",
3701		− .of_match_table = of_match_ptr(arm_smmu_of_match),
3702		− .suppress_bind_attrs = true,
	3994	+ .name = "arm-smmu-v3",
	3995	+ .of_match_table = arm_smmu_of_match,
	3996	+ .suppress_bind_attrs = true,
3703	3997	},
3704	3998	.probe = arm_smmu_device_probe,
	3999	+ .remove = arm_smmu_device_remove,
3705	4000	.shutdown = arm_smmu_device_shutdown,
3706	4001	};
3707		−builtin_platform_driver(arm_smmu_driver);
	4002	+module_platform_driver(arm_smmu_driver);
	4003	+
	4004	+MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
	4005	+MODULE_AUTHOR("Will Deacon <will@kernel.org>");
	4006	+MODULE_ALIAS("platform:arm-smmu-v3");
	4007	+MODULE_LICENSE("GPL v2");

drivers/iommu/arm-smmu.c+198−136

@@ -27,8 +27,7 @@ · 26 unmodified lines
27	27	#include <linux/interrupt.h>
28	28	#include <linux/io.h>
29	29	#include <linux/iopoll.h>
30		−#include <linux/init.h>
31		−#include <linux/moduleparam.h>
	30	+#include <linux/module.h>
32	31	#include <linux/of.h>
33	32	#include <linux/of_address.h>
34	33	#include <linux/of_device.h>
@@ -60,10 +59,6 @@ · 25 unmodified lines
60	59	#define MSI_IOVA_LENGTH 0x100000
61	60
62	61	static int force_stage;
63		−/*
64		− * not really modular, but the easiest way to keep compat with existing
65		− * bootargs behaviour is to continue using module_param() here.
66		− */
67	62	module_param(force_stage, int, S_IRUGO);
68	63	MODULE_PARM_DESC(force_stage,
69	64	"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
@@ -131,6 +126,12 @@ static struct arm_smmu_domain to_smmu_domain(struct iommu_domain dom) · 61 unmodified lines
131	126	return container_of(dom, struct arm_smmu_domain, domain);
132	127	}
133	128
	129	+static struct platform_driver arm_smmu_driver;
	130	+static struct iommu_ops arm_smmu_ops;
	131	+
	132	+#ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
	133	+static int arm_smmu_bus_init(struct iommu_ops *ops);
	134	+
134	135	static struct device_node dev_get_dev_node(struct device dev)
135	136	{
136	137	if (dev_is_pci(dev)) {
@@ -166,9 +167,6 @@ static int __find_legacy_master_phandle(struct device dev, void data) · 29 unmodified lines
166	167	return err == -ENOENT ? 0 : err;
167	168	}
168	169
169		−static struct platform_driver arm_smmu_driver;
170		−static struct iommu_ops arm_smmu_ops;
171		−
172	170	static int arm_smmu_register_legacy_master(struct device *dev,
173	171	struct arm_smmu_device **smmu)
174	172	{
@@ -220,6 +218,27 @@ static int arm_smmu_register_legacy_master(struct device *dev, · 45 unmodified lines
220	218	return err;
221	219	}
222	220
	221	+/*
	222	+ * With the legacy DT binding in play, we have no guarantees about
	223	+ * probe order, but then we're also not doing default domains, so we can
	224	+ * delay setting bus ops until we're sure every possible SMMU is ready,
	225	+ * and that way ensure that no add_device() calls get missed.
	226	+ */
	227	+static int arm_smmu_legacy_bus_init(void)
	228	+{
	229	+ if (using_legacy_binding)
	230	+ return arm_smmu_bus_init(&arm_smmu_ops);
	231	+ return 0;
	232	+}
	233	+device_initcall_sync(arm_smmu_legacy_bus_init);
	234	+#else
	235	+static int arm_smmu_register_legacy_master(struct device *dev,
	236	+ struct arm_smmu_device **smmu)
	237	+{
	238	+ return -ENODEV;
	239	+}
	240	+#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
	241	+
223	242	static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
224	243	{
225	244	int idx;
@@ -252,7 +271,7 @@ static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, · 26 unmodified lines
252	271	for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
253	272	for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
254	273	reg = arm_smmu_readl(smmu, page, status);
255		− if (!(reg & sTLBGSTATUS_GSACTIVE))
	274	+ if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
256	275	return;
257	276	cpu_relax();
258	277	}
@@ -459,7 +478,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) · 200 unmodified lines
459	478	int idx = smmu_domain->cfg.cbndx;
460	479
461	480	fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
462		− if (!(fsr & FSR_FAULT))
	481	+ if (!(fsr & ARM_SMMU_FSR_FAULT))
463	482	return IRQ_NONE;
464	483
465	484	fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
@@ -491,7 +510,7 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) · 25 unmodified lines
491	510
492	511	if (__ratelimit(&rs)) {
493	512	if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
494		− (gfsr & sGFSR_USF))
	513	+ (gfsr & ARM_SMMU_sGFSR_USF))
495	514	dev_err(smmu->dev,
496	515	"Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
497	516	(u16)gfsynr1);
@@ -521,26 +540,28 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, · 23 unmodified lines
521	540	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
522	541	cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
523	542	} else {
524		− cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
525		− cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
526		− cb->tcr[1] \|= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
	543	+ cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
	544	+ cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
527	545	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
528		− cb->tcr[1] \|= TCR2_AS;
	546	+ cb->tcr[1] \|= ARM_SMMU_TCR2_AS;
	547	+ else
	548	+ cb->tcr[0] \|= ARM_SMMU_TCR_EAE;
529	549	}
530	550	} else {
531		− cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
	551	+ cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
532	552	}
533	553
534	554	/* TTBRs */
535	555	if (stage1) {
536	556	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
537		− cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
538		− cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
	557	+ cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
	558	+ cb->ttbr[1] = 0;
539	559	} else {
540		− cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
541		− cb->ttbr[0] \|= FIELD_PREP(TTBRn_ASID, cfg->asid);
542		− cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
543		− cb->ttbr[1] \|= FIELD_PREP(TTBRn_ASID, cfg->asid);
	560	+ cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
	561	+ cb->ttbr[0] \|= FIELD_PREP(ARM_SMMU_TTBRn_ASID,
	562	+ cfg->asid);
	563	+ cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
	564	+ cfg->asid);
544	565	}
545	566	} else {
546	567	cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
@@ -576,31 +597,33 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) · 29 unmodified lines
576	597	/* CBA2R */
577	598	if (smmu->version > ARM_SMMU_V1) {
578	599	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
579		− reg = CBA2R_VA64;
	600	+ reg = ARM_SMMU_CBA2R_VA64;
580	601	else
581	602	reg = 0;
582	603	/* 16-bit VMIDs live in CBA2R */
583	604	if (smmu->features & ARM_SMMU_FEAT_VMID16)
584		− reg \|= FIELD_PREP(CBA2R_VMID16, cfg->vmid);
	605	+ reg \|= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
585	606
586	607	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
587	608	}
588	609
589	610	/* CBAR */
590		− reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);
	611	+ reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
591	612	if (smmu->version < ARM_SMMU_V2)
592		− reg \|= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);
	613	+ reg \|= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
593	614
594	615	/*
595	616	* Use the weakest shareability/memory types, so they are
596	617	* overridden by the ttbcr/pte.
597	618	*/
598	619	if (stage1) {
599		− reg \|= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) \|
600		− FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);
	620	+ reg \|= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
	621	+ ARM_SMMU_CBAR_S1_BPSHCFG_NSH) \|
	622	+ FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
	623	+ ARM_SMMU_CBAR_S1_MEMATTR_WB);
601	624	} else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
602	625	/* 8-bit VMIDs live in CBAR */
603		− reg \|= FIELD_PREP(CBAR_VMID, cfg->vmid);
	626	+ reg \|= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
604	627	}
605	628	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
606	629
@@ -632,11 +655,12 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) · 25 unmodified lines
632	655	}
633	656
634	657	/* SCTLR */
635		− reg = SCTLR_CFIE \| SCTLR_CFRE \| SCTLR_AFE \| SCTLR_TRE \| SCTLR_M;
	658	+ reg = ARM_SMMU_SCTLR_CFIE \| ARM_SMMU_SCTLR_CFRE \| ARM_SMMU_SCTLR_AFE \|
	659	+ ARM_SMMU_SCTLR_TRE \| ARM_SMMU_SCTLR_M;
636	660	if (stage1)
637		− reg \|= SCTLR_S1_ASIDPNE;
	661	+ reg \|= ARM_SMMU_SCTLR_S1_ASIDPNE;
638	662	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
639		− reg \|= SCTLR_E;
	663	+ reg \|= ARM_SMMU_SCTLR_E;
640	664
641	665	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
642	666	}
@@ -818,7 +842,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, · 175 unmodified lines
818	842	if (ret < 0) {
819	843	dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
820	844	cfg->irptndx, irq);
821		− cfg->irptndx = INVALID_IRPTNDX;
	845	+ cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
822	846	}
823	847
824	848	mutex_unlock(&smmu_domain->init_mutex);
@@ -856,7 +880,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) · 31 unmodified lines
856	880	smmu->cbs[cfg->cbndx].cfg = NULL;
857	881	arm_smmu_write_context_bank(smmu, cfg->cbndx);
858	882
859		− if (cfg->irptndx != INVALID_IRPTNDX) {
	883	+ if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
860	884	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
861	885	devm_free_irq(smmu->dev, irq, domain);
862	886	}
@@ -912,23 +936,24 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) · 49 unmodified lines
912	936	static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
913	937	{
914	938	struct arm_smmu_smr *smr = smmu->smrs + idx;
915		− u32 reg = FIELD_PREP(SMR_ID, smr->id) \| FIELD_PREP(SMR_MASK, smr->mask);
	939	+ u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) \|
	940	+ FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
916	941
917	942	if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
918		− reg \|= SMR_VALID;
	943	+ reg \|= ARM_SMMU_SMR_VALID;
919	944	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
920	945	}
921	946
922	947	static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
923	948	{
924	949	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
925		− u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) \|
926		− FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) \|
927		− FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);
	950	+ u32 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) \|
	951	+ FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) \|
	952	+ FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
928	953
929	954	if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
930	955	smmu->smrs[idx].valid)
931		− reg \|= S2CR_EXIDVALID;
	956	+ reg \|= ARM_SMMU_S2CR_EXIDVALID;
932	957	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
933	958	}
934	959
@@ -946,24 +971,37 @@ static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) · 11 unmodified lines
946	971	static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
947	972	{
948	973	u32 smr;
	974	+ int i;
949	975
950	976	if (!smmu->smrs)
951	977	return;
952		−
	978	+ /*
	979	+ * If we've had to accommodate firmware memory regions, we may
	980	+ * have live SMRs by now; tread carefully...
	981	+ *
	982	+ * Somewhat perversely, not having a free SMR for this test implies we
	983	+ * can get away without it anyway, as we'll only be able to 'allocate'
	984	+ * these SMRs for the ID/mask values we're already trusting to be OK.
	985	+ */
	986	+ for (i = 0; i < smmu->num_mapping_groups; i++)
	987	+ if (!smmu->smrs[i].valid)
	988	+ goto smr_ok;
	989	+ return;
	990	+smr_ok:
953	991	/*
954	992	* SMR.ID bits may not be preserved if the corresponding MASK
955	993	* bits are set, so check each one separately. We can reject
956	994	* masters later if they try to claim IDs outside these masks.
957	995	*/
958		− smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
959		− arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
960		− smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
961		− smmu->streamid_mask = FIELD_GET(SMR_ID, smr);
	996	+ smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
	997	+ arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
	998	+ smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
	999	+ smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
962	1000
963		− smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
964		− arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
965		− smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
966		− smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
	1001	+ smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
	1002	+ arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
	1003	+ smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
	1004	+ smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
967	1005	}
968	1006
969	1007	static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
@@ -1032,8 +1070,8 @@ static int arm_smmu_master_alloc_smes(struct device *dev) · 62 unmodified lines
1032	1070	mutex_lock(&smmu->stream_map_mutex);
1033	1071	/* Figure out a viable stream map entry allocation */
1034	1072	for_each_cfg_sme(fwspec, i, idx) {
1035		− u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1036		− u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
	1073	+ u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
	1074	+ u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1037	1075
1038	1076	if (idx != INVALID_SMENDX) {
1039	1077	ret = -EEXIST;
@@ -1277,7 +1315,8 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, · 237 unmodified lines
1277	1315	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1278	1316
1279	1317	reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1280		− if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) {
	1318	+ if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
	1319	+ 5, 50)) {
1281	1320	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1282	1321	dev_err(dev,
1283	1322	"iova to phys timed out on %pad. Falling back to software table walk.\n",
@@ -1287,7 +1326,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, · 3 unmodified lines
1287	1326
1288	1327	phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1289	1328	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1290		− if (phys & CB_PAR_F) {
	1329	+ if (phys & ARM_SMMU_CB_PAR_F) {
1291	1330	dev_err(dev, "translation fault!\n");
1292	1331	dev_err(dev, "PAR = 0x%llx\n", phys);
1293	1332	return 0;
@@ -1368,8 +1407,8 @@ static int arm_smmu_add_device(struct device *dev) · 74 unmodified lines
1368	1407
1369	1408	ret = -EINVAL;
1370	1409	for (i = 0; i < fwspec->num_ids; i++) {
1371		− u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1372		− u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
	1410	+ u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
	1411	+ u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1373	1412
1374	1413	if (sid & ~smmu->streamid_mask) {
1375	1414	dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
@@ -1550,12 +1589,12 @@ static int arm_smmu_of_xlate(struct device dev, struct of_phandle_args args) · 174 unmodified lines
1550	1589	u32 mask, fwid = 0;
1551	1590
1552	1591	if (args->args_count > 0)
1553		− fwid \|= FIELD_PREP(SMR_ID, args->args[0]);
	1592	+ fwid \|= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1554	1593
1555	1594	if (args->args_count > 1)
1556		− fwid \|= FIELD_PREP(SMR_MASK, args->args[1]);
	1595	+ fwid \|= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1557	1596	else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1558		− fwid \|= FIELD_PREP(SMR_MASK, mask);
	1597	+ fwid \|= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1559	1598
1560	1599	return iommu_fwspec_add_ids(dev, &fwid, 1);
1561	1600	}
@@ -1576,15 +1615,6 @@ static void arm_smmu_get_resv_regions(struct device *dev, · 14 unmodified lines
1576	1615	iommu_dma_get_resv_regions(dev, head);
1577	1616	}
1578	1617
1579		−static void arm_smmu_put_resv_regions(struct device *dev,
1580		− struct list_head *head)
1581		−{
1582		− struct iommu_resv_region entry, next;
1583		−
1584		− list_for_each_entry_safe(entry, next, head, list)
1585		− kfree(entry);
1586		−}
1587		−
1588	1618	static struct iommu_ops arm_smmu_ops = {
1589	1619	.capable = arm_smmu_capable,
1590	1620	.domain_alloc = arm_smmu_domain_alloc,
@@ -1602,7 +1632,7 @@ static struct iommu_ops arm_smmu_ops = { · 11 unmodified lines
1602	1632	.domain_set_attr = arm_smmu_domain_set_attr,
1603	1633	.of_xlate = arm_smmu_of_xlate,
1604	1634	.get_resv_regions = arm_smmu_get_resv_regions,
1605		− .put_resv_regions = arm_smmu_put_resv_regions,
	1635	+ .put_resv_regions = generic_iommu_put_resv_regions,
1606	1636	.pgsize_bitmap = -1UL, /* Restricted during device attach */
1607	1637	};
1608	1638
@@ -1625,7 +1655,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) · 16 unmodified lines
1625	1655	/* Make sure all context banks are disabled and clear CB_FSR */
1626	1656	for (i = 0; i < smmu->num_context_banks; ++i) {
1627	1657	arm_smmu_write_context_bank(smmu, i);
1628		− arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT);
	1658	+ arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1629	1659	}
1630	1660
1631	1661	/* Invalidate the TLB, just in case */
@@ -1635,29 +1665,30 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) · 3 unmodified lines
1635	1665	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1636	1666
1637	1667	/* Enable fault reporting */
1638		− reg \|= (sCR0_GFRE \| sCR0_GFIE \| sCR0_GCFGFRE \| sCR0_GCFGFIE);
	1668	+ reg \|= (ARM_SMMU_sCR0_GFRE \| ARM_SMMU_sCR0_GFIE \|
	1669	+ ARM_SMMU_sCR0_GCFGFRE \| ARM_SMMU_sCR0_GCFGFIE);
1639	1670
1640	1671	/* Disable TLB broadcasting. */
1641		− reg \|= (sCR0_VMIDPNE \| sCR0_PTM);
	1672	+ reg \|= (ARM_SMMU_sCR0_VMIDPNE \| ARM_SMMU_sCR0_PTM);
1642	1673
1643	1674	/* Enable client access, handling unmatched streams as appropriate */
1644		− reg &= ~sCR0_CLIENTPD;
	1675	+ reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1645	1676	if (disable_bypass)
1646		− reg \|= sCR0_USFCFG;
	1677	+ reg \|= ARM_SMMU_sCR0_USFCFG;
1647	1678	else
1648		− reg &= ~sCR0_USFCFG;
	1679	+ reg &= ~ARM_SMMU_sCR0_USFCFG;
1649	1680
1650	1681	/* Disable forced broadcasting */
1651		− reg &= ~sCR0_FB;
	1682	+ reg &= ~ARM_SMMU_sCR0_FB;
1652	1683
1653	1684	/* Don't upgrade barriers */
1654		− reg &= ~(sCR0_BSU);
	1685	+ reg &= ~(ARM_SMMU_sCR0_BSU);
1655	1686
1656	1687	if (smmu->features & ARM_SMMU_FEAT_VMID16)
1657		− reg \|= sCR0_VMID16EN;
	1688	+ reg \|= ARM_SMMU_sCR0_VMID16EN;
1658	1689
1659	1690	if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1660		− reg \|= sCR0_EXIDENABLE;
	1691	+ reg \|= ARM_SMMU_sCR0_EXIDENABLE;
1661	1692
1662	1693	if (smmu->impl && smmu->impl->reset)
1663	1694	smmu->impl->reset(smmu);
@@ -1702,21 +1733,21 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 38 unmodified lines
1702	1733
1703	1734	/* Restrict available stages based on module parameter */
1704	1735	if (force_stage == 1)
1705		− id &= ~(ID0_S2TS \| ID0_NTS);
	1736	+ id &= ~(ARM_SMMU_ID0_S2TS \| ARM_SMMU_ID0_NTS);
1706	1737	else if (force_stage == 2)
1707		− id &= ~(ID0_S1TS \| ID0_NTS);
	1738	+ id &= ~(ARM_SMMU_ID0_S1TS \| ARM_SMMU_ID0_NTS);
1708	1739
1709		− if (id & ID0_S1TS) {
	1740	+ if (id & ARM_SMMU_ID0_S1TS) {
1710	1741	smmu->features \|= ARM_SMMU_FEAT_TRANS_S1;
1711	1742	dev_notice(smmu->dev, "\tstage 1 translation\n");
1712	1743	}
1713	1744
1714		− if (id & ID0_S2TS) {
	1745	+ if (id & ARM_SMMU_ID0_S2TS) {
1715	1746	smmu->features \|= ARM_SMMU_FEAT_TRANS_S2;
1716	1747	dev_notice(smmu->dev, "\tstage 2 translation\n");
1717	1748	}
1718	1749
1719		− if (id & ID0_NTS) {
	1750	+ if (id & ARM_SMMU_ID0_NTS) {
1720	1751	smmu->features \|= ARM_SMMU_FEAT_TRANS_NESTED;
1721	1752	dev_notice(smmu->dev, "\tnested translation\n");
1722	1753	}
@@ -1727,8 +1758,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 4 unmodified lines
1727	1758	return -ENODEV;
1728	1759	}
1729	1760
1730		− if ((id & ID0_S1TS) &&
1731		− ((smmu->version < ARM_SMMU_V2) \|\| !(id & ID0_ATOSNS))) {
	1761	+ if ((id & ARM_SMMU_ID0_S1TS) &&
	1762	+ ((smmu->version < ARM_SMMU_V2) \|\| !(id & ARM_SMMU_ID0_ATOSNS))) {
1732	1763	smmu->features \|= ARM_SMMU_FEAT_TRANS_OPS;
1733	1764	dev_notice(smmu->dev, "\taddress translation ops\n");
1734	1765	}
@@ -1739,7 +1770,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 4 unmodified lines
1739	1770	* Fortunately, this also opens up a workaround for systems where the
1740	1771	* ID register value has ended up configured incorrectly.
1741	1772	*/
1742		− cttw_reg = !!(id & ID0_CTTW);
	1773	+ cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1743	1774	if (cttw_fw \|\| cttw_reg)
1744	1775	dev_notice(smmu->dev, "\t%scoherent table walk\n",
1745	1776	cttw_fw ? "" : "non-");
@@ -1748,16 +1779,16 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 2 unmodified lines
1748	1779	"\t(IDR0.CTTW overridden by FW configuration)\n");
1749	1780
1750	1781	/* Max. number of entries we have for stream matching/indexing */
1751		− if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
	1782	+ if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1752	1783	smmu->features \|= ARM_SMMU_FEAT_EXIDS;
1753	1784	size = 1 << 16;
1754	1785	} else {
1755		− size = 1 << FIELD_GET(ID0_NUMSIDB, id);
	1786	+ size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1756	1787	}
1757	1788	smmu->streamid_mask = size - 1;
1758		− if (id & ID0_SMS) {
	1789	+ if (id & ARM_SMMU_ID0_SMS) {
1759	1790	smmu->features \|= ARM_SMMU_FEAT_STREAM_MATCH;
1760		− size = FIELD_GET(ID0_NUMSMRG, id);
	1791	+ size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1761	1792	if (size == 0) {
1762	1793	dev_err(smmu->dev,
1763	1794	"stream-matching supported, but no SMRs present!\n");
@@ -1785,18 +1816,19 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 21 unmodified lines
1785	1816	mutex_init(&smmu->stream_map_mutex);
1786	1817	spin_lock_init(&smmu->global_sync_lock);
1787	1818
1788		− if (smmu->version < ARM_SMMU_V2 \|\| !(id & ID0_PTFS_NO_AARCH32)) {
	1819	+ if (smmu->version < ARM_SMMU_V2 \|\|
	1820	+ !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1789	1821	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH32_L;
1790		− if (!(id & ID0_PTFS_NO_AARCH32S))
	1822	+ if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1791	1823	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH32_S;
1792	1824	}
1793	1825
1794	1826	/* ID1 */
1795	1827	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1796		− smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
	1828	+ smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1797	1829
1798	1830	/* Check for size mismatch of SMMU address space from mapped region */
1799		− size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);
	1831	+ size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1800	1832	if (smmu->numpage != 2 * size << smmu->pgshift)
1801	1833	dev_warn(smmu->dev,
1802	1834	"SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
@@ -1804,8 +1836,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 1 unmodified line
1804	1836	/* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1805	1837	smmu->numpage = size;
1806	1838
1807		− smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);
1808		− smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);
	1839	+ smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
	1840	+ smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1809	1841	if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1810	1842	dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1811	1843	return -ENODEV;
@@ -1819,14 +1851,14 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 7 unmodified lines
1819	1851
1820	1852	/* ID2 */
1821	1853	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1822		− size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
	1854	+ size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1823	1855	smmu->ipa_size = size;
1824	1856
1825	1857	/* The output mask is also applied for bypass */
1826		− size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));
	1858	+ size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1827	1859	smmu->pa_size = size;
1828	1860
1829		− if (id & ID2_VMID16)
	1861	+ if (id & ARM_SMMU_ID2_VMID16)
1830	1862	smmu->features \|= ARM_SMMU_FEAT_VMID16;
1831	1863
1832	1864	/*
@@ -1843,13 +1875,13 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) · 10 unmodified lines
1843	1875	if (smmu->version == ARM_SMMU_V1_64K)
1844	1876	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1845	1877	} else {
1846		− size = FIELD_GET(ID2_UBS, id);
	1878	+ size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1847	1879	smmu->va_size = arm_smmu_id_size_to_bits(size);
1848		− if (id & ID2_PTFS_4K)
	1880	+ if (id & ARM_SMMU_ID2_PTFS_4K)
1849	1881	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1850		− if (id & ID2_PTFS_16K)
	1882	+ if (id & ARM_SMMU_ID2_PTFS_16K)
1851	1883	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1852		− if (id & ID2_PTFS_64K)
	1884	+ if (id & ARM_SMMU_ID2_PTFS_64K)
1853	1885	smmu->features \|= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1854	1886	}
1855	1887
@@ -1911,6 +1943,7 @@ static const struct of_device_id arm_smmu_of_match[] = { · 55 unmodified lines
1911	1943	{ .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1912	1944	{ },
1913	1945	};
	1946	+MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1914	1947
1915	1948	#ifdef CONFIG_ACPI
1916	1949	static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
@@ -1997,8 +2030,10 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev, · 80 unmodified lines
1997	2030
1998	2031	legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1999	2032	if (legacy_binding && !using_generic_binding) {
2000		− if (!using_legacy_binding)
2001		− pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
	2033	+ if (!using_legacy_binding) {
	2034	+ pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
	2035	+ IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
	2036	+ }
2002	2037	using_legacy_binding = true;
2003	2038	} else if (!legacy_binding && !using_legacy_binding) {
2004	2039	using_generic_binding = true;
@@ -2013,25 +2048,50 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev, · 8 unmodified lines
2013	2048	return 0;
2014	2049	}
2015	2050
2016		−static void arm_smmu_bus_init(void)
	2051	+static int arm_smmu_bus_init(struct iommu_ops *ops)
2017	2052	{
	2053	+ int err;
	2054	+
2018	2055	/* Oh, for a proper bus abstraction */
2019		− if (!iommu_present(&platform_bus_type))
2020		− bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
	2056	+ if (!iommu_present(&platform_bus_type)) {
	2057	+ err = bus_set_iommu(&platform_bus_type, ops);
	2058	+ if (err)
	2059	+ return err;
	2060	+ }
2021	2061	#ifdef CONFIG_ARM_AMBA
2022		− if (!iommu_present(&amba_bustype))
2023		− bus_set_iommu(&amba_bustype, &arm_smmu_ops);
	2062	+ if (!iommu_present(&amba_bustype)) {
	2063	+ err = bus_set_iommu(&amba_bustype, ops);
	2064	+ if (err)
	2065	+ goto err_reset_platform_ops;
	2066	+ }
2024	2067	#endif
2025	2068	#ifdef CONFIG_PCI
2026	2069	if (!iommu_present(&pci_bus_type)) {
2027		− pci_request_acs();
2028		− bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
	2070	+ err = bus_set_iommu(&pci_bus_type, ops);
	2071	+ if (err)
	2072	+ goto err_reset_amba_ops;
2029	2073	}
2030	2074	#endif
2031	2075	#ifdef CONFIG_FSL_MC_BUS
2032		− if (!iommu_present(&fsl_mc_bus_type))
2033		− bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
	2076	+ if (!iommu_present(&fsl_mc_bus_type)) {
	2077	+ err = bus_set_iommu(&fsl_mc_bus_type, ops);
	2078	+ if (err)
	2079	+ goto err_reset_pci_ops;
	2080	+ }
2034	2081	#endif
	2082	+ return 0;
	2083	+
	2084	+err_reset_pci_ops: __maybe_unused;
	2085	+#ifdef CONFIG_PCI
	2086	+ bus_set_iommu(&pci_bus_type, NULL);
	2087	+#endif
	2088	+err_reset_amba_ops: __maybe_unused;
	2089	+#ifdef CONFIG_ARM_AMBA
	2090	+ bus_set_iommu(&amba_bustype, NULL);
	2091	+#endif
	2092	+err_reset_platform_ops: __maybe_unused;
	2093	+ bus_set_iommu(&platform_bus_type, NULL);
	2094	+ return err;
2035	2095	}
2036	2096
2037	2097	static int arm_smmu_device_probe(struct platform_device *pdev)
@@ -2177,38 +2237,28 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 139 unmodified lines
2177	2237	* ready to handle default domain setup as soon as any SMMU exists.
2178	2238	*/
2179	2239	if (!using_legacy_binding)
2180		− arm_smmu_bus_init();
	2240	+ return arm_smmu_bus_init(&arm_smmu_ops);
2181	2241
2182	2242	return 0;
2183	2243	}
2184	2244
2185		−/*
2186		− * With the legacy DT binding in play, though, we have no guarantees about
2187		− * probe order, but then we're also not doing default domains, so we can
2188		− * delay setting bus ops until we're sure every possible SMMU is ready,
2189		− * and that way ensure that no add_device() calls get missed.
2190		− */
2191		−static int arm_smmu_legacy_bus_init(void)
2192		−{
2193		− if (using_legacy_binding)
2194		− arm_smmu_bus_init();
2195		− return 0;
2196		−}
2197		−device_initcall_sync(arm_smmu_legacy_bus_init);
2198		−
2199		−static void arm_smmu_device_shutdown(struct platform_device *pdev)
	2245	+static int arm_smmu_device_remove(struct platform_device *pdev)
2200	2246	{
2201	2247	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2202	2248
2203	2249	if (!smmu)
2204		− return;
	2250	+ return -ENODEV;
2205	2251
2206	2252	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2207	2253	dev_err(&pdev->dev, "removing device with active domains!\n");
2208	2254
	2255	+ arm_smmu_bus_init(NULL);
	2256	+ iommu_device_unregister(&smmu->iommu);
	2257	+ iommu_device_sysfs_remove(&smmu->iommu);
	2258	+
2209	2259	arm_smmu_rpm_get(smmu);
2210	2260	/* Turn the thing off */
2211		− arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD);
	2261	+ arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2212	2262	arm_smmu_rpm_put(smmu);
2213	2263
2214	2264	if (pm_runtime_enabled(smmu->dev))
@@ -2217,6 +2267,12 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev) · 2 unmodified lines
2217	2267	clk_bulk_disable(smmu->num_clks, smmu->clks);
2218	2268
2219	2269	clk_bulk_unprepare(smmu->num_clks, smmu->clks);
	2270	+ return 0;
	2271	+}
	2272	+
	2273	+static void arm_smmu_device_shutdown(struct platform_device *pdev)
	2274	+{
	2275	+ arm_smmu_device_remove(pdev);
2220	2276	}
2221	2277
2222	2278	static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
@@ -2267,11 +2323,17 @@ static const struct dev_pm_ops arm_smmu_pm_ops = { · 44 unmodified lines
2267	2323	static struct platform_driver arm_smmu_driver = {
2268	2324	.driver = {
2269	2325	.name = "arm-smmu",
2270		− .of_match_table = of_match_ptr(arm_smmu_of_match),
	2326	+ .of_match_table = arm_smmu_of_match,
2271	2327	.pm = &arm_smmu_pm_ops,
2272		− .suppress_bind_attrs = true,
	2328	+ .suppress_bind_attrs = true,
2273	2329	},
2274	2330	.probe = arm_smmu_device_probe,
	2331	+ .remove = arm_smmu_device_remove,
2275	2332	.shutdown = arm_smmu_device_shutdown,
2276	2333	};
2277		−builtin_platform_driver(arm_smmu_driver);
	2334	+module_platform_driver(arm_smmu_driver);
	2335	+
	2336	+MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
	2337	+MODULE_AUTHOR("Will Deacon <will@kernel.org>");
	2338	+MODULE_ALIAS("platform:arm-smmu");
	2339	+MODULE_LICENSE("GPL v2");

drivers/iommu/arm-smmu.h+141−87

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#define _ARM_SMMU_H
12	12
13	13	#include <linux/atomic.h>
	14	+#include <linux/bitfield.h>
14	15	#include <linux/bits.h>
15	16	#include <linux/clk.h>
16	17	#include <linux/device.h>
@@ -23,51 +24,51 @@ · 6 unmodified lines
23	24
24	25	/* Configuration registers */
25	26	#define ARM_SMMU_GR0_sCR0 0x0
26		−#define sCR0_VMID16EN BIT(31)
27		−#define sCR0_BSU GENMASK(15, 14)
28		−#define sCR0_FB BIT(13)
29		−#define sCR0_PTM BIT(12)
30		−#define sCR0_VMIDPNE BIT(11)
31		−#define sCR0_USFCFG BIT(10)
32		−#define sCR0_GCFGFIE BIT(5)
33		−#define sCR0_GCFGFRE BIT(4)
34		−#define sCR0_EXIDENABLE BIT(3)
35		−#define sCR0_GFIE BIT(2)
36		−#define sCR0_GFRE BIT(1)
37		−#define sCR0_CLIENTPD BIT(0)
	27	+#define ARM_SMMU_sCR0_VMID16EN BIT(31)
	28	+#define ARM_SMMU_sCR0_BSU GENMASK(15, 14)
	29	+#define ARM_SMMU_sCR0_FB BIT(13)
	30	+#define ARM_SMMU_sCR0_PTM BIT(12)
	31	+#define ARM_SMMU_sCR0_VMIDPNE BIT(11)
	32	+#define ARM_SMMU_sCR0_USFCFG BIT(10)
	33	+#define ARM_SMMU_sCR0_GCFGFIE BIT(5)
	34	+#define ARM_SMMU_sCR0_GCFGFRE BIT(4)
	35	+#define ARM_SMMU_sCR0_EXIDENABLE BIT(3)
	36	+#define ARM_SMMU_sCR0_GFIE BIT(2)
	37	+#define ARM_SMMU_sCR0_GFRE BIT(1)
	38	+#define ARM_SMMU_sCR0_CLIENTPD BIT(0)
38	39
39	40	/* Auxiliary Configuration register */
40	41	#define ARM_SMMU_GR0_sACR 0x10
41	42
42	43	/* Identification registers */
43	44	#define ARM_SMMU_GR0_ID0 0x20
44		−#define ID0_S1TS BIT(30)
45		−#define ID0_S2TS BIT(29)
46		−#define ID0_NTS BIT(28)
47		−#define ID0_SMS BIT(27)
48		−#define ID0_ATOSNS BIT(26)
49		−#define ID0_PTFS_NO_AARCH32 BIT(25)
50		−#define ID0_PTFS_NO_AARCH32S BIT(24)
51		−#define ID0_NUMIRPT GENMASK(23, 16)
52		−#define ID0_CTTW BIT(14)
53		−#define ID0_NUMSIDB GENMASK(12, 9)
54		−#define ID0_EXIDS BIT(8)
55		−#define ID0_NUMSMRG GENMASK(7, 0)
	45	+#define ARM_SMMU_ID0_S1TS BIT(30)
	46	+#define ARM_SMMU_ID0_S2TS BIT(29)
	47	+#define ARM_SMMU_ID0_NTS BIT(28)
	48	+#define ARM_SMMU_ID0_SMS BIT(27)
	49	+#define ARM_SMMU_ID0_ATOSNS BIT(26)
	50	+#define ARM_SMMU_ID0_PTFS_NO_AARCH32 BIT(25)
	51	+#define ARM_SMMU_ID0_PTFS_NO_AARCH32S BIT(24)
	52	+#define ARM_SMMU_ID0_NUMIRPT GENMASK(23, 16)
	53	+#define ARM_SMMU_ID0_CTTW BIT(14)
	54	+#define ARM_SMMU_ID0_NUMSIDB GENMASK(12, 9)
	55	+#define ARM_SMMU_ID0_EXIDS BIT(8)
	56	+#define ARM_SMMU_ID0_NUMSMRG GENMASK(7, 0)
56	57
57	58	#define ARM_SMMU_GR0_ID1 0x24
58		−#define ID1_PAGESIZE BIT(31)
59		−#define ID1_NUMPAGENDXB GENMASK(30, 28)
60		−#define ID1_NUMS2CB GENMASK(23, 16)
61		−#define ID1_NUMCB GENMASK(7, 0)
	59	+#define ARM_SMMU_ID1_PAGESIZE BIT(31)
	60	+#define ARM_SMMU_ID1_NUMPAGENDXB GENMASK(30, 28)
	61	+#define ARM_SMMU_ID1_NUMS2CB GENMASK(23, 16)
	62	+#define ARM_SMMU_ID1_NUMCB GENMASK(7, 0)
62	63
63	64	#define ARM_SMMU_GR0_ID2 0x28
64		−#define ID2_VMID16 BIT(15)
65		−#define ID2_PTFS_64K BIT(14)
66		−#define ID2_PTFS_16K BIT(13)
67		−#define ID2_PTFS_4K BIT(12)
68		−#define ID2_UBS GENMASK(11, 8)
69		−#define ID2_OAS GENMASK(7, 4)
70		−#define ID2_IAS GENMASK(3, 0)
	65	+#define ARM_SMMU_ID2_VMID16 BIT(15)
	66	+#define ARM_SMMU_ID2_PTFS_64K BIT(14)
	67	+#define ARM_SMMU_ID2_PTFS_16K BIT(13)
	68	+#define ARM_SMMU_ID2_PTFS_4K BIT(12)
	69	+#define ARM_SMMU_ID2_UBS GENMASK(11, 8)
	70	+#define ARM_SMMU_ID2_OAS GENMASK(7, 4)
	71	+#define ARM_SMMU_ID2_IAS GENMASK(3, 0)
71	72
72	73	#define ARM_SMMU_GR0_ID3 0x2c
73	74	#define ARM_SMMU_GR0_ID4 0x30
@@ -75,11 +76,11 @@ · 1 unmodified line
75	76	#define ARM_SMMU_GR0_ID6 0x38
76	77
77	78	#define ARM_SMMU_GR0_ID7 0x3c
78		−#define ID7_MAJOR GENMASK(7, 4)
79		−#define ID7_MINOR GENMASK(3, 0)
	79	+#define ARM_SMMU_ID7_MAJOR GENMASK(7, 4)
	80	+#define ARM_SMMU_ID7_MINOR GENMASK(3, 0)
80	81
81	82	#define ARM_SMMU_GR0_sGFSR 0x48
82		−#define sGFSR_USF BIT(1)
	83	+#define ARM_SMMU_sGFSR_USF BIT(1)
83	84
84	85	#define ARM_SMMU_GR0_sGFSYNR0 0x50
85	86	#define ARM_SMMU_GR0_sGFSYNR1 0x54
@@ -92,106 +93,132 @@ · 6 unmodified lines
92	93	#define ARM_SMMU_GR0_sTLBGSYNC 0x70
93	94
94	95	#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
95		−#define sTLBGSTATUS_GSACTIVE BIT(0)
	96	+#define ARM_SMMU_sTLBGSTATUS_GSACTIVE BIT(0)
96	97
97	98	/* Stream mapping registers */
98	99	#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
99		−#define SMR_VALID BIT(31)
100		−#define SMR_MASK GENMASK(31, 16)
101		−#define SMR_ID GENMASK(15, 0)
	100	+#define ARM_SMMU_SMR_VALID BIT(31)
	101	+#define ARM_SMMU_SMR_MASK GENMASK(31, 16)
	102	+#define ARM_SMMU_SMR_ID GENMASK(15, 0)
102	103
103	104	#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
104		−#define S2CR_PRIVCFG GENMASK(25, 24)
	105	+#define ARM_SMMU_S2CR_PRIVCFG GENMASK(25, 24)
105	106	enum arm_smmu_s2cr_privcfg {
106	107	S2CR_PRIVCFG_DEFAULT,
107	108	S2CR_PRIVCFG_DIPAN,
108	109	S2CR_PRIVCFG_UNPRIV,
109	110	S2CR_PRIVCFG_PRIV,
110	111	};
111		−#define S2CR_TYPE GENMASK(17, 16)
	112	+#define ARM_SMMU_S2CR_TYPE GENMASK(17, 16)
112	113	enum arm_smmu_s2cr_type {
113	114	S2CR_TYPE_TRANS,
114	115	S2CR_TYPE_BYPASS,
115	116	S2CR_TYPE_FAULT,
116	117	};
117		−#define S2CR_EXIDVALID BIT(10)
118		−#define S2CR_CBNDX GENMASK(7, 0)
	118	+#define ARM_SMMU_S2CR_EXIDVALID BIT(10)
	119	+#define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0)
119	120
120	121	/* Context bank attribute registers */
121	122	#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
122		−#define CBAR_IRPTNDX GENMASK(31, 24)
123		−#define CBAR_TYPE GENMASK(17, 16)
	123	+#define ARM_SMMU_CBAR_IRPTNDX GENMASK(31, 24)
	124	+#define ARM_SMMU_CBAR_TYPE GENMASK(17, 16)
124	125	enum arm_smmu_cbar_type {
125	126	CBAR_TYPE_S2_TRANS,
126	127	CBAR_TYPE_S1_TRANS_S2_BYPASS,
127	128	CBAR_TYPE_S1_TRANS_S2_FAULT,
128	129	CBAR_TYPE_S1_TRANS_S2_TRANS,
129	130	};
130		−#define CBAR_S1_MEMATTR GENMASK(15, 12)
131		−#define CBAR_S1_MEMATTR_WB 0xf
132		−#define CBAR_S1_BPSHCFG GENMASK(9, 8)
133		−#define CBAR_S1_BPSHCFG_NSH 3
134		−#define CBAR_VMID GENMASK(7, 0)
	131	+#define ARM_SMMU_CBAR_S1_MEMATTR GENMASK(15, 12)
	132	+#define ARM_SMMU_CBAR_S1_MEMATTR_WB 0xf
	133	+#define ARM_SMMU_CBAR_S1_BPSHCFG GENMASK(9, 8)
	134	+#define ARM_SMMU_CBAR_S1_BPSHCFG_NSH 3
	135	+#define ARM_SMMU_CBAR_VMID GENMASK(7, 0)
135	136
136	137	#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
137	138
138	139	#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
139		−#define CBA2R_VMID16 GENMASK(31, 16)
140		−#define CBA2R_VA64 BIT(0)
	140	+#define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16)
	141	+#define ARM_SMMU_CBA2R_VA64 BIT(0)
141	142
142	143	#define ARM_SMMU_CB_SCTLR 0x0
143		−#define SCTLR_S1_ASIDPNE BIT(12)
144		−#define SCTLR_CFCFG BIT(7)
145		−#define SCTLR_CFIE BIT(6)
146		−#define SCTLR_CFRE BIT(5)
147		−#define SCTLR_E BIT(4)
148		−#define SCTLR_AFE BIT(2)
149		−#define SCTLR_TRE BIT(1)
150		−#define SCTLR_M BIT(0)
	144	+#define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12)
	145	+#define ARM_SMMU_SCTLR_CFCFG BIT(7)
	146	+#define ARM_SMMU_SCTLR_CFIE BIT(6)
	147	+#define ARM_SMMU_SCTLR_CFRE BIT(5)
	148	+#define ARM_SMMU_SCTLR_E BIT(4)
	149	+#define ARM_SMMU_SCTLR_AFE BIT(2)
	150	+#define ARM_SMMU_SCTLR_TRE BIT(1)
	151	+#define ARM_SMMU_SCTLR_M BIT(0)
151	152
152	153	#define ARM_SMMU_CB_ACTLR 0x4
153	154
154	155	#define ARM_SMMU_CB_RESUME 0x8
155		−#define RESUME_TERMINATE BIT(0)
	156	+#define ARM_SMMU_RESUME_TERMINATE BIT(0)
156	157
157	158	#define ARM_SMMU_CB_TCR2 0x10
158		−#define TCR2_SEP GENMASK(17, 15)
159		−#define TCR2_SEP_UPSTREAM 0x7
160		−#define TCR2_AS BIT(4)
	159	+#define ARM_SMMU_TCR2_SEP GENMASK(17, 15)
	160	+#define ARM_SMMU_TCR2_SEP_UPSTREAM 0x7
	161	+#define ARM_SMMU_TCR2_AS BIT(4)
	162	+#define ARM_SMMU_TCR2_PASIZE GENMASK(3, 0)
161	163
162	164	#define ARM_SMMU_CB_TTBR0 0x20
163	165	#define ARM_SMMU_CB_TTBR1 0x28
164		−#define TTBRn_ASID GENMASK_ULL(63, 48)
	166	+#define ARM_SMMU_TTBRn_ASID GENMASK_ULL(63, 48)
165	167
166	168	#define ARM_SMMU_CB_TCR 0x30
	169	+#define ARM_SMMU_TCR_EAE BIT(31)
	170	+#define ARM_SMMU_TCR_EPD1 BIT(23)
	171	+#define ARM_SMMU_TCR_TG0 GENMASK(15, 14)
	172	+#define ARM_SMMU_TCR_SH0 GENMASK(13, 12)
	173	+#define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10)
	174	+#define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8)
	175	+#define ARM_SMMU_TCR_T0SZ GENMASK(5, 0)
	176	+
	177	+#define ARM_SMMU_VTCR_RES1 BIT(31)
	178	+#define ARM_SMMU_VTCR_PS GENMASK(18, 16)
	179	+#define ARM_SMMU_VTCR_TG0 ARM_SMMU_TCR_TG0
	180	+#define ARM_SMMU_VTCR_SH0 ARM_SMMU_TCR_SH0
	181	+#define ARM_SMMU_VTCR_ORGN0 ARM_SMMU_TCR_ORGN0
	182	+#define ARM_SMMU_VTCR_IRGN0 ARM_SMMU_TCR_IRGN0
	183	+#define ARM_SMMU_VTCR_SL0 GENMASK(7, 6)
	184	+#define ARM_SMMU_VTCR_T0SZ ARM_SMMU_TCR_T0SZ
	185	+
167	186	#define ARM_SMMU_CB_CONTEXTIDR 0x34
168	187	#define ARM_SMMU_CB_S1_MAIR0 0x38
169	188	#define ARM_SMMU_CB_S1_MAIR1 0x3c
170	189
171	190	#define ARM_SMMU_CB_PAR 0x50
172		−#define CB_PAR_F BIT(0)
	191	+#define ARM_SMMU_CB_PAR_F BIT(0)
173	192
174	193	#define ARM_SMMU_CB_FSR 0x58
175		−#define FSR_MULTI BIT(31)
176		−#define FSR_SS BIT(30)
177		−#define FSR_UUT BIT(8)
178		−#define FSR_ASF BIT(7)
179		−#define FSR_TLBLKF BIT(6)
180		−#define FSR_TLBMCF BIT(5)
181		−#define FSR_EF BIT(4)
182		−#define FSR_PF BIT(3)
183		−#define FSR_AFF BIT(2)
184		−#define FSR_TF BIT(1)
185		−
186		−#define FSR_IGN (FSR_AFF \| FSR_ASF \| \
187		− FSR_TLBMCF \| FSR_TLBLKF)
188		−#define FSR_FAULT (FSR_MULTI \| FSR_SS \| FSR_UUT \| \
189		− FSR_EF \| FSR_PF \| FSR_TF \| FSR_IGN)
	194	+#define ARM_SMMU_FSR_MULTI BIT(31)
	195	+#define ARM_SMMU_FSR_SS BIT(30)
	196	+#define ARM_SMMU_FSR_UUT BIT(8)
	197	+#define ARM_SMMU_FSR_ASF BIT(7)
	198	+#define ARM_SMMU_FSR_TLBLKF BIT(6)
	199	+#define ARM_SMMU_FSR_TLBMCF BIT(5)
	200	+#define ARM_SMMU_FSR_EF BIT(4)
	201	+#define ARM_SMMU_FSR_PF BIT(3)
	202	+#define ARM_SMMU_FSR_AFF BIT(2)
	203	+#define ARM_SMMU_FSR_TF BIT(1)
	204	+
	205	+#define ARM_SMMU_FSR_IGN (ARM_SMMU_FSR_AFF \| \
	206	+ ARM_SMMU_FSR_ASF \| \
	207	+ ARM_SMMU_FSR_TLBMCF \| \
	208	+ ARM_SMMU_FSR_TLBLKF)
	209	+
	210	+#define ARM_SMMU_FSR_FAULT (ARM_SMMU_FSR_MULTI \| \
	211	+ ARM_SMMU_FSR_SS \| \
	212	+ ARM_SMMU_FSR_UUT \| \
	213	+ ARM_SMMU_FSR_EF \| \
	214	+ ARM_SMMU_FSR_PF \| \
	215	+ ARM_SMMU_FSR_TF \| \
	216	+ ARM_SMMU_FSR_IGN)
190	217
191	218	#define ARM_SMMU_CB_FAR 0x60
192	219
193	220	#define ARM_SMMU_CB_FSYNR0 0x68
194		−#define FSYNR0_WNR BIT(4)
	221	+#define ARM_SMMU_FSYNR0_WNR BIT(4)
195	222
196	223	#define ARM_SMMU_CB_S1_TLBIVA 0x600
197	224	#define ARM_SMMU_CB_S1_TLBIASID 0x610
@@ -203,7 +230,7 @@ enum arm_smmu_cbar_type { · 5 unmodified lines
203	230	#define ARM_SMMU_CB_ATS1PR 0x800
204	231
205	232	#define ARM_SMMU_CB_ATSR 0x8f0
206		−#define ATSR_ACTIVE BIT(0)
	233	+#define ARM_SMMU_ATSR_ACTIVE BIT(0)
207	234
208	235
209	236	/* Maximum number of context banks per SMMU */
@@ -297,7 +324,7 @@ struct arm_smmu_cfg { · 87 unmodified lines
297	324	enum arm_smmu_cbar_type cbar;
298	325	enum arm_smmu_context_fmt fmt;
299	326	};
300		−#define INVALID_IRPTNDX 0xff
	327	+#define ARM_SMMU_INVALID_IRPTNDX 0xff
301	328
302	329	enum arm_smmu_domain_stage {
303	330	ARM_SMMU_DOMAIN_S1 = 0,
@@ -318,6 +345,33 @@ struct arm_smmu_domain { · 14 unmodified lines
318	345	struct iommu_domain domain;
319	346	};
320	347
	348	+static inline u32 arm_smmu_lpae_tcr(struct io_pgtable_cfg *cfg)
	349	+{
	350	+ return ARM_SMMU_TCR_EPD1 \|
	351	+ FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) \|
	352	+ FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) \|
	353	+ FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) \|
	354	+ FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) \|
	355	+ FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
	356	+}
	357	+
	358	+static inline u32 arm_smmu_lpae_tcr2(struct io_pgtable_cfg *cfg)
	359	+{
	360	+ return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) \|
	361	+ FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
	362	+}
	363	+
	364	+static inline u32 arm_smmu_lpae_vtcr(struct io_pgtable_cfg *cfg)
	365	+{
	366	+ return ARM_SMMU_VTCR_RES1 \|
	367	+ FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) \|
	368	+ FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) \|
	369	+ FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) \|
	370	+ FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) \|
	371	+ FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) \|
	372	+ FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) \|
	373	+ FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
	374	+}
321	375
322	376	/* Implementation details, yay! */
323	377	struct arm_smmu_impl {

drivers/iommu/dmar.c+42−2

@@ -244,7 +244,7 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, · 243 unmodified lines
244	244	info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) \|\|
245	245	(scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
246	246	(info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
247		− info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {
	247	+ info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) {
248	248	pr_warn("Device scope type does not match for %s\n",
249	249	pci_name(info->dev));
250	250	return -EINVAL;
@@ -1354,7 +1354,6 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, · 1103 unmodified lines
1354	1354	struct qi_desc desc;
1355	1355
1356	1356	if (mask) {
1357		− WARN_ON_ONCE(addr & ((1ULL << (VTD_PAGE_SHIFT + mask)) - 1));
1358	1357	addr \|= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1359	1358	desc.qw1 = QI_DEV_IOTLB_ADDR(addr) \| QI_DEV_IOTLB_SIZE;
1360	1359	} else
@@ -1371,6 +1370,47 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, · 10 unmodified lines
1371	1370	qi_submit_sync(&desc, iommu);
1372	1371	}
1373	1372
	1373	+/* PASID-based IOTLB invalidation */
	1374	+void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
	1375	+ unsigned long npages, bool ih)
	1376	+{
	1377	+ struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
	1378	+
	1379	+ /*
	1380	+ * npages == -1 means a PASID-selective invalidation, otherwise,
	1381	+ * a positive value for Page-selective-within-PASID invalidation.
	1382	+ * 0 is not a valid input.
	1383	+ */
	1384	+ if (WARN_ON(!npages)) {
	1385	+ pr_err("Invalid input npages = %ld\n", npages);
	1386	+ return;
	1387	+ }
	1388	+
	1389	+ if (npages == -1) {
	1390	+ desc.qw0 = QI_EIOTLB_PASID(pasid) \|
	1391	+ QI_EIOTLB_DID(did) \|
	1392	+ QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) \|
	1393	+ QI_EIOTLB_TYPE;
	1394	+ desc.qw1 = 0;
	1395	+ } else {
	1396	+ int mask = ilog2(__roundup_pow_of_two(npages));
	1397	+ unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
	1398	+
	1399	+ if (WARN_ON_ONCE(!ALIGN(addr, align)))
	1400	+ addr &= ~(align - 1);
	1401	+
	1402	+ desc.qw0 = QI_EIOTLB_PASID(pasid) \|
	1403	+ QI_EIOTLB_DID(did) \|
	1404	+ QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) \|
	1405	+ QI_EIOTLB_TYPE;
	1406	+ desc.qw1 = QI_EIOTLB_ADDR(addr) \|
	1407	+ QI_EIOTLB_IH(ih) \|
	1408	+ QI_EIOTLB_AM(mask);
	1409	+ }
	1410	+
	1411	+ qi_submit_sync(&desc, iommu);
	1412	+}
	1413	+
1374	1414	/*
1375	1415	* Disable Queued Invalidation interface.
1376	1416	*/

drivers/iommu/intel-iommu-debugfs.c+75−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	* Authors: Gayatri Kammela <gayatri.kammela@intel.com>
6	6	* Sohil Mehta <sohil.mehta@intel.com>
7	7	* Jacob Pan <jacob.jun.pan@linux.intel.com>
	8	+ * Lu Baolu <baolu.lu@linux.intel.com>
8	9	*/
9	10
10	11	#include <linux/debugfs.h>
@@ -283,6 +284,77 @@ static int dmar_translation_struct_show(struct seq_file m, void unused) · 272 unmodified lines
283	284	}
284	285	DEFINE_SHOW_ATTRIBUTE(dmar_translation_struct);
285	286
	287	+static inline unsigned long level_to_directory_size(int level)
	288	+{
	289	+ return BIT_ULL(VTD_PAGE_SHIFT + VTD_STRIDE_SHIFT * (level - 1));
	290	+}
	291	+
	292	+static inline void
	293	+dump_page_info(struct seq_file m, unsigned long iova, u64 path)
	294	+{
	295	+ seq_printf(m, "0x%013lx \|\t0x%016llx\t0x%016llx\t0x%016llx\t0x%016llx\t0x%016llx\n",
	296	+ iova >> VTD_PAGE_SHIFT, path[5], path[4],
	297	+ path[3], path[2], path[1]);
	298	+}
	299	+
	300	+static void pgtable_walk_level(struct seq_file m, struct dma_pte pde,
	301	+ int level, unsigned long start,
	302	+ u64 *path)
	303	+{
	304	+ int i;
	305	+
	306	+ if (level > 5 \|\| level < 1)
	307	+ return;
	308	+
	309	+ for (i = 0; i < BIT_ULL(VTD_STRIDE_SHIFT);
	310	+ i++, pde++, start += level_to_directory_size(level)) {
	311	+ if (!dma_pte_present(pde))
	312	+ continue;
	313	+
	314	+ path[level] = pde->val;
	315	+ if (dma_pte_superpage(pde) \|\| level == 1)
	316	+ dump_page_info(m, start, path);
	317	+ else
	318	+ pgtable_walk_level(m, phys_to_virt(dma_pte_addr(pde)),
	319	+ level - 1, start, path);
	320	+ path[level] = 0;
	321	+ }
	322	+}
	323	+
	324	+static int show_device_domain_translation(struct device dev, void data)
	325	+{
	326	+ struct dmar_domain *domain = find_domain(dev);
	327	+ struct seq_file *m = data;
	328	+ u64 path[6] = { 0 };
	329	+
	330	+ if (!domain)
	331	+ return 0;
	332	+
	333	+ seq_printf(m, "Device %s with pasid %d @0x%llx\n",
	334	+ dev_name(dev), domain->default_pasid,
	335	+ (u64)virt_to_phys(domain->pgd));
	336	+ seq_puts(m, "IOVA_PFN\t\tPML5E\t\t\tPML4E\t\t\tPDPE\t\t\tPDE\t\t\tPTE\n");
	337	+
	338	+ pgtable_walk_level(m, domain->pgd, domain->agaw + 2, 0, path);
	339	+ seq_putc(m, '\n');
	340	+
	341	+ return 0;
	342	+}
	343	+
	344	+static int domain_translation_struct_show(struct seq_file m, void unused)
	345	+{
	346	+ unsigned long flags;
	347	+ int ret;
	348	+
	349	+ spin_lock_irqsave(&device_domain_lock, flags);
	350	+ ret = bus_for_each_dev(&pci_bus_type, NULL, m,
	351	+ show_device_domain_translation);
	352	+ spin_unlock_irqrestore(&device_domain_lock, flags);
	353	+
	354	+ return ret;
	355	+}
	356	+DEFINE_SHOW_ATTRIBUTE(domain_translation_struct);
	357	+
286	358	#ifdef CONFIG_IRQ_REMAP
287	359	static void ir_tbl_remap_entry_show(struct seq_file *m,
288	360	struct intel_iommu *iommu)
@@ -396,6 +468,9 @@ void __init intel_iommu_debugfs_init(void) · 107 unmodified lines
396	468	&iommu_regset_fops);
397	469	debugfs_create_file("dmar_translation_struct", 0444, intel_iommu_debug,
398	470	NULL, &dmar_translation_struct_fops);
	471	+ debugfs_create_file("domain_translation_struct", 0444,
	472	+ intel_iommu_debug, NULL,
	473	+ &domain_translation_struct_fops);
399	474	#ifdef CONFIG_IRQ_REMAP
400	475	debugfs_create_file("ir_translation_struct", 0444, intel_iommu_debug,
401	476	NULL, &ir_translation_struct_fops);

drivers/iommu/intel-iommu.c+275−91

@@ -307,6 +307,20 @@ static int hw_pass_through = 1; · 306 unmodified lines
307	307	*/
308	308	#define DOMAIN_FLAG_LOSE_CHILDREN BIT(1)
309	309
	310	+/*
	311	+ * When VT-d works in the scalable mode, it allows DMA translation to
	312	+ * happen through either first level or second level page table. This
	313	+ * bit marks that the DMA translation for the domain goes through the
	314	+ * first level page table, otherwise, it goes through the second level.
	315	+ */
	316	+#define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(2)
	317	+
	318	+/*
	319	+ * Domain represents a virtual machine which demands iommu nested
	320	+ * translation mode support.
	321	+ */
	322	+#define DOMAIN_FLAG_NESTING_MODE BIT(3)
	323	+
310	324	#define for_each_domain_iommu(idx, domain) \
311	325	for (idx = 0; idx < g_num_of_iommus; idx++) \
312	326	if (domain->iommu_refcnt[idx])
@@ -355,9 +369,14 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, · 42 unmodified lines
355	369	int dmar_disabled = 0;
356	370	#else
357	371	int dmar_disabled = 1;
358		−#endif /CONFIG_INTEL_IOMMU_DEFAULT_ON/
	372	+#endif /* CONFIG_INTEL_IOMMU_DEFAULT_ON */
359	373
	374	+#ifdef INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON
	375	+int intel_iommu_sm = 1;
	376	+#else
360	377	int intel_iommu_sm;
	378	+#endif /* INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON */
	379	+
361	380	int intel_iommu_enabled = 0;
362	381	EXPORT_SYMBOL_GPL(intel_iommu_enabled);
363	382
@@ -368,7 +387,6 @@ static int intel_iommu_superpage = 1; · 4 unmodified lines
368	387	static int iommu_identity_mapping;
369	388	static int intel_no_bounce;
370	389
371		−#define IDENTMAP_ALL 1
372	390	#define IDENTMAP_GFX 2
373	391	#define IDENTMAP_AZALIA 4
374	392
@@ -377,7 +395,7 @@ EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); · 2 unmodified lines
377	395
378	396	#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
379	397	#define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
380		−static DEFINE_SPINLOCK(device_domain_lock);
	398	+DEFINE_SPINLOCK(device_domain_lock);
381	399	static LIST_HEAD(device_domain_list);
382	400
383	401	#define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \
@@ -552,6 +570,11 @@ static inline int domain_type_is_si(struct dmar_domain *domain) · 168 unmodified lines
552	570	return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
553	571	}
554	572
	573	+static inline bool domain_use_first_level(struct dmar_domain *domain)
	574	+{
	575	+ return domain->flags & DOMAIN_FLAG_USE_FIRST_LEVEL;
	576	+}
	577	+
555	578	static inline int domain_pfn_supported(struct dmar_domain *domain,
556	579	unsigned long pfn)
557	580	{
@@ -661,11 +684,12 @@ static int domain_update_iommu_snooping(struct intel_iommu *skip) · 103 unmodified lines
661	684	return ret;
662	685	}
663	686
664		−static int domain_update_iommu_superpage(struct intel_iommu *skip)
	687	+static int domain_update_iommu_superpage(struct dmar_domain *domain,
	688	+ struct intel_iommu *skip)
665	689	{
666	690	struct dmar_drhd_unit *drhd;
667	691	struct intel_iommu *iommu;
668		− int mask = 0xf;
	692	+ int mask = 0x3;
669	693
670	694	if (!intel_iommu_superpage) {
671	695	return 0;
@@ -675,7 +699,13 @@ static int domain_update_iommu_superpage(struct intel_iommu *skip) · 3 unmodified lines
675	699	rcu_read_lock();
676	700	for_each_active_iommu(iommu, drhd) {
677	701	if (iommu != skip) {
678		− mask &= cap_super_page_val(iommu->cap);
	702	+ if (domain && domain_use_first_level(domain)) {
	703	+ if (!cap_fl1gp_support(iommu->cap))
	704	+ mask = 0x1;
	705	+ } else {
	706	+ mask &= cap_super_page_val(iommu->cap);
	707	+ }
	708	+
679	709	if (!mask)
680	710	break;
681	711	}
@@ -690,7 +720,7 @@ static void domain_update_iommu_cap(struct dmar_domain *domain) · 8 unmodified lines
690	720	{
691	721	domain_update_iommu_coherency(domain);
692	722	domain->iommu_snooping = domain_update_iommu_snooping(NULL);
693		− domain->iommu_superpage = domain_update_iommu_superpage(NULL);
	723	+ domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL);
694	724	}
695	725
696	726	struct context_entry iommu_context_addr(struct intel_iommu iommu, u8 bus,
@@ -907,6 +937,8 @@ static struct dma_pte pfn_to_dma_pte(struct dmar_domain domain, · 210 unmodified lines
907	937
908	938	domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
909	939	pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) \| DMA_PTE_READ \| DMA_PTE_WRITE;
	940	+ if (domain_use_first_level(domain))
	941	+ pteval \|= DMA_FL_PTE_XD;
910	942	if (cmpxchg64(&pte->val, 0ULL, pteval))
911	943	/* Someone else set it while we were thinking; use theirs. */
912	944	free_pgtable_page(tmp_page);
@@ -1477,6 +1509,20 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, · 564 unmodified lines
1477	1509	spin_unlock_irqrestore(&device_domain_lock, flags);
1478	1510	}
1479	1511
	1512	+static void domain_flush_piotlb(struct intel_iommu *iommu,
	1513	+ struct dmar_domain *domain,
	1514	+ u64 addr, unsigned long npages, bool ih)
	1515	+{
	1516	+ u16 did = domain->iommu_did[iommu->seq_id];
	1517	+
	1518	+ if (domain->default_pasid)
	1519	+ qi_flush_piotlb(iommu, did, domain->default_pasid,
	1520	+ addr, npages, ih);
	1521	+
	1522	+ if (!list_empty(&domain->devices))
	1523	+ qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, npages, ih);
	1524	+}
	1525	+
1480	1526	static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1481	1527	struct dmar_domain *domain,
1482	1528	unsigned long pfn, unsigned int pages,
@@ -1490,18 +1536,23 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, · 7 unmodified lines
1490	1536
1491	1537	if (ih)
1492	1538	ih = 1 << 6;
1493		− /*
1494		− * Fallback to domain selective flush if no PSI support or the size is
1495		− * too big.
1496		− * PSI requires page size to be 2 ^ x, and the base address is naturally
1497		− * aligned to the size
1498		− */
1499		− if (!cap_pgsel_inv(iommu->cap) \|\| mask > cap_max_amask_val(iommu->cap))
1500		− iommu->flush.flush_iotlb(iommu, did, 0, 0,
1501		− DMA_TLB_DSI_FLUSH);
1502		− else
1503		− iommu->flush.flush_iotlb(iommu, did, addr \| ih, mask,
1504		− DMA_TLB_PSI_FLUSH);
	1539	+
	1540	+ if (domain_use_first_level(domain)) {
	1541	+ domain_flush_piotlb(iommu, domain, addr, pages, ih);
	1542	+ } else {
	1543	+ /*
	1544	+ * Fallback to domain selective flush if no PSI support or
	1545	+ * the size is too big. PSI requires page size to be 2 ^ x,
	1546	+ * and the base address is naturally aligned to the size.
	1547	+ */
	1548	+ if (!cap_pgsel_inv(iommu->cap) \|\|
	1549	+ mask > cap_max_amask_val(iommu->cap))
	1550	+ iommu->flush.flush_iotlb(iommu, did, 0, 0,
	1551	+ DMA_TLB_DSI_FLUSH);
	1552	+ else
	1553	+ iommu->flush.flush_iotlb(iommu, did, addr \| ih, mask,
	1554	+ DMA_TLB_PSI_FLUSH);
	1555	+ }
1505	1556
1506	1557	/*
1507	1558	* In caching mode, changes of pages from non-present to present require
@@ -1516,8 +1567,11 @@ static inline void __mapping_notify_one(struct intel_iommu *iommu, · 8 unmodified lines
1516	1567	struct dmar_domain *domain,
1517	1568	unsigned long pfn, unsigned int pages)
1518	1569	{
1519		− /* It's a non-present to present mapping. Only flush if caching mode */
1520		− if (cap_caching_mode(iommu->cap))
	1570	+ /*
	1571	+ * It's a non-present to present mapping. Only flush if caching mode
	1572	+ * and second level.
	1573	+ */
	1574	+ if (cap_caching_mode(iommu->cap) && !domain_use_first_level(domain))
1521	1575	iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1522	1576	else
1523	1577	iommu_flush_write_buffer(iommu);
@@ -1534,7 +1588,11 @@ static void iommu_flush_iova(struct iova_domain *iovad) · 10 unmodified lines
1534	1588	struct intel_iommu *iommu = g_iommus[idx];
1535	1589	u16 did = domain->iommu_did[iommu->seq_id];
1536	1590
1537		− iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
	1591	+ if (domain_use_first_level(domain))
	1592	+ domain_flush_piotlb(iommu, domain, 0, -1, 0);
	1593	+ else
	1594	+ iommu->flush.flush_iotlb(iommu, did, 0, 0,
	1595	+ DMA_TLB_DSI_FLUSH);
1538	1596
1539	1597	if (!cap_caching_mode(iommu->cap))
1540	1598	iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
@@ -1703,6 +1761,33 @@ static void free_dmar_iommu(struct intel_iommu *iommu) · 162 unmodified lines
1703	1761	#endif
1704	1762	}
1705	1763
	1764	+/*
	1765	+ * Check and return whether first level is used by default for
	1766	+ * DMA translation.
	1767	+ */
	1768	+static bool first_level_by_default(void)
	1769	+{
	1770	+ struct dmar_drhd_unit *drhd;
	1771	+ struct intel_iommu *iommu;
	1772	+ static int first_level_support = -1;
	1773	+
	1774	+ if (likely(first_level_support != -1))
	1775	+ return first_level_support;
	1776	+
	1777	+ first_level_support = 1;
	1778	+
	1779	+ rcu_read_lock();
	1780	+ for_each_active_iommu(iommu, drhd) {
	1781	+ if (!sm_supported(iommu) \|\| !ecap_flts(iommu->ecap)) {
	1782	+ first_level_support = 0;
	1783	+ break;
	1784	+ }
	1785	+ }
	1786	+ rcu_read_unlock();
	1787	+
	1788	+ return first_level_support;
	1789	+}
	1790	+
1706	1791	static struct dmar_domain *alloc_domain(int flags)
1707	1792	{
1708	1793	struct dmar_domain *domain;
@@ -1714,6 +1799,8 @@ static struct dmar_domain *alloc_domain(int flags) · 5 unmodified lines
1714	1799	memset(domain, 0, sizeof(*domain));
1715	1800	domain->nid = NUMA_NO_NODE;
1716	1801	domain->flags = flags;
	1802	+ if (first_level_by_default())
	1803	+ domain->flags \|= DOMAIN_FLAG_USE_FIRST_LEVEL;
1717	1804	domain->has_iotlb_device = false;
1718	1805	INIT_LIST_HEAD(&domain->devices);
1719	1806
@@ -1843,14 +1930,16 @@ static int domain_init(struct dmar_domain domain, struct intel_iommu iommu, · 123 unmodified lines
1843	1930	{
1844	1931	int adjust_width, agaw;
1845	1932	unsigned long sagaw;
1846		− int err;
	1933	+ int ret;
1847	1934
1848	1935	init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1849	1936
1850		− err = init_iova_flush_queue(&domain->iovad,
1851		− iommu_flush_iova, iova_entry_free);
1852		− if (err)
1853		− return err;
	1937	+ if (!intel_iommu_strict) {
	1938	+ ret = init_iova_flush_queue(&domain->iovad,
	1939	+ iommu_flush_iova, iova_entry_free);
	1940	+ if (ret)
	1941	+ pr_info("iova flush queue initialization failed\n");
	1942	+ }
1854	1943
1855	1944	domain_reserve_special_ranges(domain);
1856	1945
@@ -2223,17 +2312,20 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, · 366 unmodified lines
2223	2312	unsigned long sg_res = 0;
2224	2313	unsigned int largepage_lvl = 0;
2225	2314	unsigned long lvl_pages = 0;
	2315	+ u64 attr;
2226	2316
2227	2317	BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2228	2318
2229	2319	if ((prot & (DMA_PTE_READ\|DMA_PTE_WRITE)) == 0)
2230	2320	return -EINVAL;
2231	2321
2232		− prot &= DMA_PTE_READ \| DMA_PTE_WRITE \| DMA_PTE_SNP;
	2322	+ attr = prot & (DMA_PTE_READ \| DMA_PTE_WRITE \| DMA_PTE_SNP);
	2323	+ if (domain_use_first_level(domain))
	2324	+ attr \|= DMA_FL_PTE_PRESENT \| DMA_FL_PTE_XD;
2233	2325
2234	2326	if (!sg) {
2235	2327	sg_res = nr_pages;
2236		− pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) \| prot;
	2328	+ pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) \| attr;
2237	2329	}
2238	2330
2239	2331	while (nr_pages > 0) {
@@ -2245,7 +2337,7 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, · 5 unmodified lines
2245	2337	sg_res = aligned_nrpages(sg->offset, sg->length);
2246	2338	sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2247	2339	sg->dma_length = sg->length;
2248		− pteval = (sg_phys(sg) - pgoff) \| prot;
	2340	+ pteval = (sg_phys(sg) - pgoff) \| attr;
2249	2341	phys_pfn = pteval >> VTD_PAGE_SHIFT;
2250	2342	}
2251	2343
@@ -2414,7 +2506,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain) · 162 unmodified lines
2414	2506	spin_unlock_irqrestore(&device_domain_lock, flags);
2415	2507	}
2416	2508
2417		−static struct dmar_domain find_domain(struct device dev)
	2509	+struct dmar_domain find_domain(struct device dev)
2418	2510	{
2419	2511	struct device_domain_info *info;
2420	2512
@@ -2460,6 +2552,36 @@ dmar_search_domain_by_dev_info(int segment, int bus, int devfn) · 39 unmodified lines
2460	2552	return NULL;
2461	2553	}
2462	2554
	2555	+static int domain_setup_first_level(struct intel_iommu *iommu,
	2556	+ struct dmar_domain *domain,
	2557	+ struct device *dev,
	2558	+ int pasid)
	2559	+{
	2560	+ int flags = PASID_FLAG_SUPERVISOR_MODE;
	2561	+ struct dma_pte *pgd = domain->pgd;
	2562	+ int agaw, level;
	2563	+
	2564	+ /*
	2565	+ * Skip top levels of page tables for iommu which has
	2566	+ * less agaw than default. Unnecessary for PT mode.
	2567	+ */
	2568	+ for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
	2569	+ pgd = phys_to_virt(dma_pte_addr(pgd));
	2570	+ if (!dma_pte_present(pgd))
	2571	+ return -ENOMEM;
	2572	+ }
	2573	+
	2574	+ level = agaw_to_level(agaw);
	2575	+ if (level != 4 && level != 5)
	2576	+ return -EINVAL;
	2577	+
	2578	+ flags \|= (level == 5) ? PASID_FLAG_FL5LP : 0;
	2579	+
	2580	+ return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid,
	2581	+ domain->iommu_did[iommu->seq_id],
	2582	+ flags);
	2583	+}
	2584	+
2463	2585	static struct dmar_domain dmar_insert_one_dev_info(struct intel_iommu iommu,
2464	2586	int bus, int devfn,
2465	2587	struct device *dev,
@@ -2559,6 +2681,9 @@ static struct dmar_domain dmar_insert_one_dev_info(struct intel_iommu iommu, · 93 unmodified lines
2559	2681	if (hw_pass_through && domain_type_is_si(domain))
2560	2682	ret = intel_pasid_setup_pass_through(iommu, domain,
2561	2683	dev, PASID_RID2PASID);
	2684	+ else if (domain_use_first_level(domain))
	2685	+ ret = domain_setup_first_level(iommu, domain, dev,
	2686	+ PASID_RID2PASID);
2562	2687	else
2563	2688	ret = intel_pasid_setup_second_level(iommu, domain,
2564	2689	dev, PASID_RID2PASID);
@@ -2764,10 +2889,8 @@ static int __init si_domain_init(int hw) · 199 unmodified lines
2764	2889	}
2765	2890
2766	2891	/*
2767		− * Normally we use DMA domains for devices which have RMRRs. But we
2768		− * loose this requirement for graphic and usb devices. Identity map
2769		− * the RMRRs for graphic and USB devices so that they could use the
2770		− * si_domain.
	2892	+ * Identity map the RMRRs so that devices with RMRRs could also use
	2893	+ * the si_domain.
2771	2894	*/
2772	2895	for_each_rmrr_units(rmrr) {
2773	2896	for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
@@ -2775,9 +2898,6 @@ static int __init si_domain_init(int hw) · 1 unmodified line
2775	2898	unsigned long long start = rmrr->base_address;
2776	2899	unsigned long long end = rmrr->end_address;
2777	2900
2778		− if (device_is_rmrr_locked(dev))
2779		− continue;
2780		−
2781	2901	if (WARN_ON(end < start \|\|
2782	2902	end >> agaw_to_width(si_domain->agaw)))
2783	2903	continue;
@@ -2916,9 +3036,6 @@ static int device_def_domain_type(struct device *dev) · 132 unmodified lines
2916	3036	if (dev_is_pci(dev)) {
2917	3037	struct pci_dev *pdev = to_pci_dev(dev);
2918	3038
2919		− if (device_is_rmrr_locked(dev))
2920		− return IOMMU_DOMAIN_DMA;
2921		−
2922	3039	/*
2923	3040	* Prevent any device marked as untrusted from getting
2924	3041	* placed into the statically identity mapping domain.
@@ -2956,13 +3073,9 @@ static int device_def_domain_type(struct device *dev) · 31 unmodified lines
2956	3073	return IOMMU_DOMAIN_DMA;
2957	3074	} else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2958	3075	return IOMMU_DOMAIN_DMA;
2959		− } else {
2960		− if (device_has_rmrr(dev))
2961		− return IOMMU_DOMAIN_DMA;
2962	3076	}
2963	3077
2964		− return (iommu_identity_mapping & IDENTMAP_ALL) ?
2965		− IOMMU_DOMAIN_IDENTITY : 0;
	3078	+ return 0;
2966	3079	}
2967	3080
2968	3081	static void intel_iommu_init_qi(struct intel_iommu *iommu)
@@ -3291,10 +3404,7 @@ static int __init init_dmars(void) · 322 unmodified lines
3291	3404
3292	3405	if (!ecap_pass_through(iommu->ecap))
3293	3406	hw_pass_through = 0;
3294		−#ifdef CONFIG_INTEL_IOMMU_SVM
3295		− if (pasid_supported(iommu))
3296		− intel_svm_init(iommu);
3297		−#endif
	3407	+ intel_svm_check(iommu);
3298	3408	}
3299	3409
3300	3410	/*
@@ -3309,9 +3419,6 @@ static int __init init_dmars(void) · 8 unmodified lines
3309	3419	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3310	3420	}
3311	3421
3312		− if (iommu_default_passthrough())
3313		− iommu_identity_mapping \|= IDENTMAP_ALL;
3314		−
3315	3422	#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3316	3423	dmar_map_gfx = 0;
3317	3424	#endif
@@ -3384,8 +3491,21 @@ static unsigned long intel_alloc_iova(struct device *dev, · 66 unmodified lines
3384	3491	{
3385	3492	unsigned long iova_pfn;
3386	3493
3387		− /* Restrict dma_mask to the width that the iommu can handle */
3388		− dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
	3494	+ /*
	3495	+ * Restrict dma_mask to the width that the iommu can handle.
	3496	+ * First-level translation restricts the input-address to a
	3497	+ * canonical address (i.e., address bits 63:N have the same
	3498	+ * value as address bit [N-1], where N is 48-bits with 4-level
	3499	+ * paging and 57-bits with 5-level paging). Hence, skip bit
	3500	+ * [N-1].
	3501	+ */
	3502	+ if (domain_use_first_level(domain))
	3503	+ dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw - 1),
	3504	+ dma_mask);
	3505	+ else
	3506	+ dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw),
	3507	+ dma_mask);
	3508	+
3389	3509	/* Ensure we reserve the whole size-aligned region */
3390	3510	nrpages = __roundup_pow_of_two(nrpages);
3391	3511
@@ -3403,7 +3523,8 @@ static unsigned long intel_alloc_iova(struct device *dev, · 11 unmodified lines
3403	3523	iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3404	3524	IOVA_PFN(dma_mask), true);
3405	3525	if (unlikely(!iova_pfn)) {
3406		− dev_err(dev, "Allocating %ld-page iova failed", nrpages);
	3526	+ dev_err_once(dev, "Allocating %ld-page iova failed\n",
	3527	+ nrpages);
3407	3528	return 0;
3408	3529	}
3409	3530
@@ -3771,8 +3892,8 @@ static int intel_map_sg(struct device dev, struct scatterlist sglist, int nele · 361 unmodified lines
3771	3892	return 0;
3772	3893	}
3773	3894
3774		− trace_map_sg(dev, iova_pfn << PAGE_SHIFT,
3775		− sg_phys(sglist), size << VTD_PAGE_SHIFT);
	3895	+ for_each_sg(sglist, sg, nelems, i)
	3896	+ trace_map_sg(dev, i + 1, nelems, sg);
3776	3897
3777	3898	return nelems;
3778	3899	}
@@ -3984,6 +4105,9 @@ bounce_map_sg(struct device dev, struct scatterlist sglist, int nelems, · 205 unmodified lines
3984	4105	sg_dma_len(sg) = sg->length;
3985	4106	}
3986	4107
	4108	+ for_each_sg(sglist, sg, nelems, i)
	4109	+ trace_bounce_map_sg(dev, i + 1, nelems, sg);
	4110	+
3987	4111	return nelems;
3988	4112
3989	4113	out_unmap:
@@ -4312,16 +4436,31 @@ static void __init init_iommu_pm_ops(void) · 322 unmodified lines
4312	4436	static inline void init_iommu_pm_ops(void) {}
4313	4437	#endif /* CONFIG_PM */
4314	4438
	4439	+static int rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
	4440	+{
	4441	+ if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) \|\|
	4442	+ !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) \|\|
	4443	+ rmrr->end_address <= rmrr->base_address \|\|
	4444	+ arch_rmrr_sanity_check(rmrr))
	4445	+ return -EINVAL;
	4446	+
	4447	+ return 0;
	4448	+}
	4449	+
4315	4450	int __init dmar_parse_one_rmrr(struct acpi_dmar_header header, void arg)
4316	4451	{
4317	4452	struct acpi_dmar_reserved_memory *rmrr;
4318	4453	struct dmar_rmrr_unit *rmrru;
4319		− int ret;
4320	4454
4321	4455	rmrr = (struct acpi_dmar_reserved_memory *)header;
4322		− ret = arch_rmrr_sanity_check(rmrr);
4323		− if (ret)
4324		− return ret;
	4456	+ if (rmrr_sanity_check(rmrr))
	4457	+ WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
	4458	+ "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
	4459	+ "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
	4460	+ rmrr->base_address, rmrr->end_address,
	4461	+ dmi_get_system_info(DMI_BIOS_VENDOR),
	4462	+ dmi_get_system_info(DMI_BIOS_VERSION),
	4463	+ dmi_get_system_info(DMI_PRODUCT_VERSION));
4325	4464
4326	4465	rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4327	4466	if (!rmrru)
@@ -4467,7 +4606,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) · 139 unmodified lines
4467	4606	iommu->name);
4468	4607	return -ENXIO;
4469	4608	}
4470		− sp = domain_update_iommu_superpage(iommu) - 1;
	4609	+ sp = domain_update_iommu_superpage(NULL, iommu) - 1;
4471	4610	if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4472	4611	pr_warn("%s: Doesn't support large page.\n",
4473	4612	iommu->name);
@@ -4487,10 +4626,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) · 13 unmodified lines
4487	4626	if (ret)
4488	4627	goto out;
4489	4628
4490		−#ifdef CONFIG_INTEL_IOMMU_SVM
4491		− if (pasid_supported(iommu))
4492		− intel_svm_init(iommu);
4493		−#endif
	4629	+ intel_svm_check(iommu);
4494	4630
4495	4631	if (dmaru->ignored) {
4496	4632	/*
@@ -4895,7 +5031,7 @@ static int __init platform_optin_force_iommu(void) · 398 unmodified lines
4895	5031	* map for all devices except those marked as being untrusted.
4896	5032	*/
4897	5033	if (dmar_disabled)
4898		− iommu_identity_mapping \|= IDENTMAP_ALL;
	5034	+ iommu_set_default_passthrough(false);
4899	5035
4900	5036	dmar_disabled = 0;
4901	5037	no_iommu = 0;
@@ -5195,6 +5331,7 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) · 293 unmodified lines
5195	5331	{
5196	5332	struct dmar_domain *dmar_domain;
5197	5333	struct iommu_domain *domain;
	5334	+ int ret;
5198	5335
5199	5336	switch (type) {
5200	5337	case IOMMU_DOMAIN_DMA:
@@ -5211,11 +5348,12 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) · 10 unmodified lines
5211	5348	return NULL;
5212	5349	}
5213	5350
5214		− if (type == IOMMU_DOMAIN_DMA &&
5215		− init_iova_flush_queue(&dmar_domain->iovad,
5216		− iommu_flush_iova, iova_entry_free)) {
5217		− pr_warn("iova flush queue initialization failed\n");
5218		− intel_iommu_strict = 1;
	5351	+ if (!intel_iommu_strict && type == IOMMU_DOMAIN_DMA) {
	5352	+ ret = init_iova_flush_queue(&dmar_domain->iovad,
	5353	+ iommu_flush_iova,
	5354	+ iova_entry_free);
	5355	+ if (ret)
	5356	+ pr_info("iova flush queue initialization failed\n");
5219	5357	}
5220	5358
5221	5359	domain_update_iommu_cap(dmar_domain);
@@ -5281,7 +5419,7 @@ static void auxiliary_unlink_device(struct dmar_domain *domain, · 59 unmodified lines
5281	5419	domain->auxd_refcnt--;
5282	5420
5283	5421	if (!domain->auxd_refcnt && domain->default_pasid > 0)
5284		− intel_pasid_free_id(domain->default_pasid);
	5422	+ ioasid_free(domain->default_pasid);
5285	5423	}
5286	5424
5287	5425	static int aux_domain_add_dev(struct dmar_domain *domain,
@@ -5299,10 +5437,11 @@ static int aux_domain_add_dev(struct dmar_domain *domain, · 11 unmodified lines
5299	5437	if (domain->default_pasid <= 0) {
5300	5438	int pasid;
5301	5439
5302		− pasid = intel_pasid_alloc_id(domain, PASID_MIN,
5303		− pci_max_pasids(to_pci_dev(dev)),
5304		− GFP_KERNEL);
5305		− if (pasid <= 0) {
	5440	+ /* No private data needed for the default pasid */
	5441	+ pasid = ioasid_alloc(NULL, PASID_MIN,
	5442	+ pci_max_pasids(to_pci_dev(dev)) - 1,
	5443	+ NULL);
	5444	+ if (pasid == INVALID_IOASID) {
5306	5445	pr_err("Can't allocate default pasid\n");
5307	5446	return -ENODEV;
5308	5447	}
@@ -5320,8 +5459,12 @@ static int aux_domain_add_dev(struct dmar_domain *domain, · 11 unmodified lines
5320	5459	goto attach_failed;
5321	5460
5322	5461	/* Setup the PASID entry for mediated devices: */
5323		− ret = intel_pasid_setup_second_level(iommu, domain, dev,
5324		− domain->default_pasid);
	5462	+ if (domain_use_first_level(domain))
	5463	+ ret = domain_setup_first_level(iommu, domain, dev,
	5464	+ domain->default_pasid);
	5465	+ else
	5466	+ ret = intel_pasid_setup_second_level(iommu, domain, dev,
	5467	+ domain->default_pasid);
5325	5468	if (ret)
5326	5469	goto table_failed;
5327	5470	spin_unlock(&iommu->lock);
@@ -5338,7 +5481,7 @@ attach_failed: · 10 unmodified lines
5338	5481	spin_unlock(&iommu->lock);
5339	5482	spin_unlock_irqrestore(&device_domain_lock, flags);
5340	5483	if (!domain->auxd_refcnt && domain->default_pasid > 0)
5341		− intel_pasid_free_id(domain->default_pasid);
	5484	+ ioasid_free(domain->default_pasid);
5342	5485
5343	5486	return ret;
5344	5487	}
@@ -5592,6 +5735,24 @@ static inline bool iommu_pasid_support(void) · 247 unmodified lines
5592	5735	return ret;
5593	5736	}
5594	5737
	5738	+static inline bool nested_mode_support(void)
	5739	+{
	5740	+ struct dmar_drhd_unit *drhd;
	5741	+ struct intel_iommu *iommu;
	5742	+ bool ret = true;
	5743	+
	5744	+ rcu_read_lock();
	5745	+ for_each_active_iommu(iommu, drhd) {
	5746	+ if (!sm_supported(iommu) \|\| !ecap_nest(iommu->ecap)) {
	5747	+ ret = false;
	5748	+ break;
	5749	+ }
	5750	+ }
	5751	+ rcu_read_unlock();
	5752	+
	5753	+ return ret;
	5754	+}
	5755	+
5595	5756	static bool intel_iommu_capable(enum iommu_cap cap)
5596	5757	{
5597	5758	if (cap == IOMMU_CAP_CACHE_COHERENCY)
@@ -5749,15 +5910,6 @@ static void intel_iommu_get_resv_regions(struct device *device, · 151 unmodified lines
5749	5910	list_add_tail(&reg->list, head);
5750	5911	}
5751	5912
5752		−static void intel_iommu_put_resv_regions(struct device *dev,
5753		− struct list_head *head)
5754		−{
5755		− struct iommu_resv_region entry, next;
5756		−
5757		− list_for_each_entry_safe(entry, next, head, list)
5758		− kfree(entry);
5759		−}
5760		−
5761	5913	int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev)
5762	5914	{
5763	5915	struct device_domain_info *info;
@@ -5984,10 +6136,42 @@ static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain, · 220 unmodified lines
5984	6136	return dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO;
5985	6137	}
5986	6138
	6139	+static int
	6140	+intel_iommu_domain_set_attr(struct iommu_domain *domain,
	6141	+ enum iommu_attr attr, void *data)
	6142	+{
	6143	+ struct dmar_domain *dmar_domain = to_dmar_domain(domain);
	6144	+ unsigned long flags;
	6145	+ int ret = 0;
	6146	+
	6147	+ if (domain->type != IOMMU_DOMAIN_UNMANAGED)
	6148	+ return -EINVAL;
	6149	+
	6150	+ switch (attr) {
	6151	+ case DOMAIN_ATTR_NESTING:
	6152	+ spin_lock_irqsave(&device_domain_lock, flags);
	6153	+ if (nested_mode_support() &&
	6154	+ list_empty(&dmar_domain->devices)) {
	6155	+ dmar_domain->flags \|= DOMAIN_FLAG_NESTING_MODE;
	6156	+ dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL;
	6157	+ } else {
	6158	+ ret = -ENODEV;
	6159	+ }
	6160	+ spin_unlock_irqrestore(&device_domain_lock, flags);
	6161	+ break;
	6162	+ default:
	6163	+ ret = -EINVAL;
	6164	+ break;
	6165	+ }
	6166	+
	6167	+ return ret;
	6168	+}
	6169	+
5987	6170	const struct iommu_ops intel_iommu_ops = {
5988	6171	.capable = intel_iommu_capable,
5989	6172	.domain_alloc = intel_iommu_domain_alloc,
5990	6173	.domain_free = intel_iommu_domain_free,
	6174	+ .domain_set_attr = intel_iommu_domain_set_attr,
5991	6175	.attach_dev = intel_iommu_attach_device,
5992	6176	.detach_dev = intel_iommu_detach_device,
5993	6177	.aux_attach_dev = intel_iommu_aux_attach_device,
@@ -5999,7 +6183,7 @@ const struct iommu_ops intel_iommu_ops = { · 5 unmodified lines
5999	6183	.add_device = intel_iommu_add_device,
6000	6184	.remove_device = intel_iommu_remove_device,
6001	6185	.get_resv_regions = intel_iommu_get_resv_regions,
6002		− .put_resv_regions = intel_iommu_put_resv_regions,
	6186	+ .put_resv_regions = generic_iommu_put_resv_regions,
6003	6187	.apply_resv_region = intel_iommu_apply_resv_region,
6004	6188	.device_group = intel_iommu_device_group,
6005	6189	.dev_has_feat = intel_iommu_dev_has_feat,

drivers/iommu/intel-pasid.c+27−70

@@ -26,42 +26,6 @@ · 25 unmodified lines
26	26	*/
27	27	static DEFINE_SPINLOCK(pasid_lock);
28	28	u32 intel_pasid_max_id = PASID_MAX;
29		−static DEFINE_IDR(pasid_idr);
30		−
31		−int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp)
32		−{
33		− int ret, min, max;
34		−
35		− min = max_t(int, start, PASID_MIN);
36		− max = min_t(int, end, intel_pasid_max_id);
37		−
38		− WARN_ON(in_interrupt());
39		− idr_preload(gfp);
40		− spin_lock(&pasid_lock);
41		− ret = idr_alloc(&pasid_idr, ptr, min, max, GFP_ATOMIC);
42		− spin_unlock(&pasid_lock);
43		− idr_preload_end();
44		−
45		− return ret;
46		−}
47		−
48		−void intel_pasid_free_id(int pasid)
49		−{
50		− spin_lock(&pasid_lock);
51		− idr_remove(&pasid_idr, pasid);
52		− spin_unlock(&pasid_lock);
53		−}
54		−
55		−void *intel_pasid_lookup_id(int pasid)
56		−{
57		− void *p;
58		−
59		− spin_lock(&pasid_lock);
60		− p = idr_find(&pasid_idr, pasid);
61		− spin_unlock(&pasid_lock);
62		−
63		− return p;
64		−}
65	29
66	30	/*
67	31	* Per device pasid table management:
@@ -465,6 +429,21 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, · 397 unmodified lines
465	429	devtlb_invalidation_with_pasid(iommu, dev, pasid);
466	430	}
467	431
	432	+static void pasid_flush_caches(struct intel_iommu *iommu,
	433	+ struct pasid_entry *pte,
	434	+ int pasid, u16 did)
	435	+{
	436	+ if (!ecap_coherent(iommu->ecap))
	437	+ clflush_cache_range(pte, sizeof(*pte));
	438	+
	439	+ if (cap_caching_mode(iommu->cap)) {
	440	+ pasid_cache_invalidation_with_pasid(iommu, did, pasid);
	441	+ iotlb_invalidation_with_pasid(iommu, did, pasid);
	442	+ } else {
	443	+ iommu_flush_write_buffer(iommu);
	444	+ }
	445	+}
	446	+
468	447	/*
469	448	* Set up the scalable mode pasid table entry for first only
470	449	* translation type.
@@ -498,10 +477,15 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, · 27 unmodified lines
498	477	pasid_set_sre(pte);
499	478	}
500	479
501		−#ifdef CONFIG_X86
502		− if (cpu_feature_enabled(X86_FEATURE_LA57))
503		− pasid_set_flpm(pte, 1);
504		−#endif /* CONFIG_X86 */
	480	+ if (flags & PASID_FLAG_FL5LP) {
	481	+ if (cap_5lp_support(iommu->cap)) {
	482	+ pasid_set_flpm(pte, 1);
	483	+ } else {
	484	+ pr_err("No 5-level paging support for first-level\n");
	485	+ pasid_clear_entry(pte);
	486	+ return -EINVAL;
	487	+ }
	488	+ }
505	489
506	490	pasid_set_domain_id(pte, did);
507	491	pasid_set_address_width(pte, iommu->agaw);
@@ -510,16 +494,7 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, · 2 unmodified lines
510	494	/* Setup Present and PASID Granular Transfer Type: */
511	495	pasid_set_translation_type(pte, 1);
512	496	pasid_set_present(pte);
513		−
514		− if (!ecap_coherent(iommu->ecap))
515		− clflush_cache_range(pte, sizeof(*pte));
516		−
517		− if (cap_caching_mode(iommu->cap)) {
518		− pasid_cache_invalidation_with_pasid(iommu, did, pasid);
519		− iotlb_invalidation_with_pasid(iommu, did, pasid);
520		− } else {
521		− iommu_flush_write_buffer(iommu);
522		− }
	497	+ pasid_flush_caches(iommu, pte, pasid, did);
523	498
524	499	return 0;
525	500	}
@@ -583,16 +558,7 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, · 57 unmodified lines
583	558	*/
584	559	pasid_set_sre(pte);
585	560	pasid_set_present(pte);
586		−
587		− if (!ecap_coherent(iommu->ecap))
588		− clflush_cache_range(pte, sizeof(*pte));
589		−
590		− if (cap_caching_mode(iommu->cap)) {
591		− pasid_cache_invalidation_with_pasid(iommu, did, pasid);
592		− iotlb_invalidation_with_pasid(iommu, did, pasid);
593		− } else {
594		− iommu_flush_write_buffer(iommu);
595		− }
	561	+ pasid_flush_caches(iommu, pte, pasid, did);
596	562
597	563	return 0;
598	564	}
@@ -626,16 +592,7 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, · 27 unmodified lines
626	592	*/
627	593	pasid_set_sre(pte);
628	594	pasid_set_present(pte);
629		−
630		− if (!ecap_coherent(iommu->ecap))
631		− clflush_cache_range(pte, sizeof(*pte));
632		−
633		− if (cap_caching_mode(iommu->cap)) {
634		− pasid_cache_invalidation_with_pasid(iommu, did, pasid);
635		− iotlb_invalidation_with_pasid(iommu, did, pasid);
636		− } else {
637		− iommu_flush_write_buffer(iommu);
638		− }
	595	+ pasid_flush_caches(iommu, pte, pasid, did);
639	596
640	597	return 0;
641	598	}

drivers/iommu/intel-pasid.h+6−0

@@ -37,6 +37,12 @@ · 36 unmodified lines
37	37	*/
38	38	#define PASID_FLAG_SUPERVISOR_MODE BIT(0)
39	39
	40	+/*
	41	+ * The PASID_FLAG_FL5LP flag Indicates using 5-level paging for first-
	42	+ * level translation, otherwise, 4-level paging will be used.
	43	+ */
	44	+#define PASID_FLAG_FL5LP BIT(1)
	45	+
40	46	struct pasid_dir_entry {
41	47	u64 val;
42	48	};

drivers/iommu/intel-svm.c+99−72

@@ -17,25 +17,13 @@ · 16 unmodified lines
17	17	#include <linux/dmar.h>
18	18	#include <linux/interrupt.h>
19	19	#include <linux/mm_types.h>
	20	+#include <linux/ioasid.h>
20	21	#include <asm/page.h>
21	22
22	23	#include "intel-pasid.h"
23	24
24	25	static irqreturn_t prq_event_thread(int irq, void *d);
25	26
26		−int intel_svm_init(struct intel_iommu *iommu)
27		−{
28		− if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
29		− !cap_fl1gp_support(iommu->cap))
30		− return -EINVAL;
31		−
32		− if (cpu_feature_enabled(X86_FEATURE_LA57) &&
33		− !cap_5lp_support(iommu->cap))
34		− return -EINVAL;
35		−
36		− return 0;
37		−}
38		−
39	27	#define PRQ_ORDER 0
40	28
41	29	int intel_svm_enable_prq(struct intel_iommu *iommu)
@@ -99,6 +87,33 @@ int intel_svm_finish_prq(struct intel_iommu *iommu) · 57 unmodified lines
99	87	return 0;
100	88	}
101	89
	90	+static inline bool intel_svm_capable(struct intel_iommu *iommu)
	91	+{
	92	+ return iommu->flags & VTD_FLAG_SVM_CAPABLE;
	93	+}
	94	+
	95	+void intel_svm_check(struct intel_iommu *iommu)
	96	+{
	97	+ if (!pasid_supported(iommu))
	98	+ return;
	99	+
	100	+ if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
	101	+ !cap_fl1gp_support(iommu->cap)) {
	102	+ pr_err("%s SVM disabled, incompatible 1GB page capability\n",
	103	+ iommu->name);
	104	+ return;
	105	+ }
	106	+
	107	+ if (cpu_feature_enabled(X86_FEATURE_LA57) &&
	108	+ !cap_5lp_support(iommu->cap)) {
	109	+ pr_err("%s SVM disabled, incompatible paging mode\n",
	110	+ iommu->name);
	111	+ return;
	112	+ }
	113	+
	114	+ iommu->flags \|= VTD_FLAG_SVM_CAPABLE;
	115	+}
	116	+
102	117	static void intel_flush_svm_range_dev (struct intel_svm svm, struct intel_svm_dev sdev,
103	118	unsigned long address, unsigned long pages, int ih)
104	119	{
@@ -207,6 +222,10 @@ static const struct mmu_notifier_ops intel_mmuops = { · 102 unmodified lines
207	222	static DEFINE_MUTEX(pasid_mutex);
208	223	static LIST_HEAD(global_svm_list);
209	224
	225	+#define for_each_svm_dev(sdev, svm, d) \
	226	+ list_for_each_entry((sdev), &(svm)->devs, list) \
	227	+ if ((d) != (sdev)->dev) {} else
	228	+
210	229	int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ops *ops)
211	230	{
212	231	struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
@@ -220,6 +239,9 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 7 unmodified lines
220	239	if (!iommu \|\| dmar_disabled)
221	240	return -EINVAL;
222	241
	242	+ if (!intel_svm_capable(iommu))
	243	+ return -ENOTSUPP;
	244	+
223	245	if (dev_is_pci(dev)) {
224	246	pasid_max = pci_max_pasids(to_pci_dev(dev));
225	247	if (pasid_max < 0)
@@ -252,15 +274,14 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 26 unmodified lines
252	274	goto out;
253	275	}
254	276
255		− list_for_each_entry(sdev, &svm->devs, list) {
256		− if (dev == sdev->dev) {
257		− if (sdev->ops != ops) {
258		− ret = -EBUSY;
259		− goto out;
260		− }
261		− sdev->users++;
262		− goto success;
	277	+ /* Find the matching device in svm list */
	278	+ for_each_svm_dev(sdev, svm, dev) {
	279	+ if (sdev->ops != ops) {
	280	+ ret = -EBUSY;
	281	+ goto out;
263	282	}
	283	+ sdev->users++;
	284	+ goto success;
264	285	}
265	286
266	287	break;
@@ -314,16 +335,15 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 47 unmodified lines
314	335	if (pasid_max > intel_pasid_max_id)
315	336	pasid_max = intel_pasid_max_id;
316	337
317		− /* Do not use PASID 0 in caching mode (virtualised IOMMU) */
318		− ret = intel_pasid_alloc_id(svm,
319		− !!cap_caching_mode(iommu->cap),
320		− pasid_max - 1, GFP_KERNEL);
321		− if (ret < 0) {
	338	+ /* Do not use PASID 0, reserved for RID to PASID */
	339	+ svm->pasid = ioasid_alloc(NULL, PASID_MIN,
	340	+ pasid_max - 1, svm);
	341	+ if (svm->pasid == INVALID_IOASID) {
322	342	kfree(svm);
323	343	kfree(sdev);
	344	+ ret = -ENOSPC;
324	345	goto out;
325	346	}
326		− svm->pasid = ret;
327	347	svm->notifier.ops = &intel_mmuops;
328	348	svm->mm = mm;
329	349	svm->flags = flags;
@@ -333,7 +353,7 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 3 unmodified lines
333	353	if (mm) {
334	354	ret = mmu_notifier_register(&svm->notifier, mm);
335	355	if (ret) {
336		− intel_pasid_free_id(svm->pasid);
	356	+ ioasid_free(svm->pasid);
337	357	kfree(svm);
338	358	kfree(sdev);
339	359	goto out;
@@ -344,12 +364,14 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 4 unmodified lines
344	364	ret = intel_pasid_setup_first_level(iommu, dev,
345	365	mm ? mm->pgd : init_mm.pgd,
346	366	svm->pasid, FLPT_DEFAULT_DID,
347		− mm ? 0 : PASID_FLAG_SUPERVISOR_MODE);
	367	+ (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) \|
	368	+ (cpu_feature_enabled(X86_FEATURE_LA57) ?
	369	+ PASID_FLAG_FL5LP : 0));
348	370	spin_unlock(&iommu->lock);
349	371	if (ret) {
350	372	if (mm)
351	373	mmu_notifier_unregister(&svm->notifier, mm);
352		− intel_pasid_free_id(svm->pasid);
	374	+ ioasid_free(svm->pasid);
353	375	kfree(svm);
354	376	kfree(sdev);
355	377	goto out;
@@ -365,7 +387,9 @@ int intel_svm_bind_mm(struct device dev, int pasid, int flags, struct svm_dev_ · 9 unmodified lines
365	387	ret = intel_pasid_setup_first_level(iommu, dev,
366	388	mm ? mm->pgd : init_mm.pgd,
367	389	svm->pasid, FLPT_DEFAULT_DID,
368		− mm ? 0 : PASID_FLAG_SUPERVISOR_MODE);
	390	+ (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) \|
	391	+ (cpu_feature_enabled(X86_FEATURE_LA57) ?
	392	+ PASID_FLAG_FL5LP : 0));
369	393	spin_unlock(&iommu->lock);
370	394	if (ret) {
371	395	kfree(sdev);
@@ -397,44 +421,45 @@ int intel_svm_unbind_mm(struct device *dev, int pasid) · 25 unmodified lines
397	421	if (!iommu)
398	422	goto out;
399	423
400		− svm = intel_pasid_lookup_id(pasid);
	424	+ svm = ioasid_find(NULL, pasid, NULL);
401	425	if (!svm)
402	426	goto out;
403	427
404		− list_for_each_entry(sdev, &svm->devs, list) {
405		− if (dev == sdev->dev) {
406		− ret = 0;
407		− sdev->users--;
408		− if (!sdev->users) {
409		− list_del_rcu(&sdev->list);
410		− /* Flush the PASID cache and IOTLB for this device.
411		− * Note that we do depend on the hardware not using
412		− * the PASID any more. Just as we depend on other
413		− * devices never using PASIDs that they have no right
414		− * to use. We have a shared PASID table, because it's
415		− * large and has to be physically contiguous. So it's
416		− * hard to be as defensive as we might like. */
417		− intel_pasid_tear_down_entry(iommu, dev, svm->pasid);
418		− intel_flush_svm_range_dev(svm, sdev, 0, -1, 0);
419		− kfree_rcu(sdev, rcu);
420		−
421		− if (list_empty(&svm->devs)) {
422		− intel_pasid_free_id(svm->pasid);
423		− if (svm->mm)
424		− mmu_notifier_unregister(&svm->notifier, svm->mm);
425		−
426		− list_del(&svm->list);
427		−
428		− /* We mandate that no page faults may be outstanding
429		− * for the PASID when intel_svm_unbind_mm() is called.
430		− * If that is not obeyed, subtle errors will happen.
431		− * Let's make them less subtle... */
432		− memset(svm, 0x6b, sizeof(*svm));
433		− kfree(svm);
434		− }
	428	+ if (IS_ERR(svm)) {
	429	+ ret = PTR_ERR(svm);
	430	+ goto out;
	431	+ }
	432	+
	433	+ for_each_svm_dev(sdev, svm, dev) {
	434	+ ret = 0;
	435	+ sdev->users--;
	436	+ if (!sdev->users) {
	437	+ list_del_rcu(&sdev->list);
	438	+ /* Flush the PASID cache and IOTLB for this device.
	439	+ * Note that we do depend on the hardware not using
	440	+ * the PASID any more. Just as we depend on other
	441	+ * devices never using PASIDs that they have no right
	442	+ * to use. We have a shared PASID table, because it's
	443	+ * large and has to be physically contiguous. So it's
	444	+ * hard to be as defensive as we might like. */
	445	+ intel_pasid_tear_down_entry(iommu, dev, svm->pasid);
	446	+ intel_flush_svm_range_dev(svm, sdev, 0, -1, 0);
	447	+ kfree_rcu(sdev, rcu);
	448	+
	449	+ if (list_empty(&svm->devs)) {
	450	+ ioasid_free(svm->pasid);
	451	+ if (svm->mm)
	452	+ mmu_notifier_unregister(&svm->notifier, svm->mm);
	453	+ list_del(&svm->list);
	454	+ /* We mandate that no page faults may be outstanding
	455	+ * for the PASID when intel_svm_unbind_mm() is called.
	456	+ * If that is not obeyed, subtle errors will happen.
	457	+ * Let's make them less subtle... */
	458	+ memset(svm, 0x6b, sizeof(*svm));
	459	+ kfree(svm);
435	460	}
436		− break;
437	461	}
	462	+ break;
438	463	}
439	464	out:
440	465	mutex_unlock(&pasid_mutex);
@@ -454,10 +479,14 @@ int intel_svm_is_pasid_valid(struct device *dev, int pasid) · 13 unmodified lines
454	479	if (!iommu)
455	480	goto out;
456	481
457		− svm = intel_pasid_lookup_id(pasid);
	482	+ svm = ioasid_find(NULL, pasid, NULL);
458	483	if (!svm)
459	484	goto out;
460	485
	486	+ if (IS_ERR(svm)) {
	487	+ ret = PTR_ERR(svm);
	488	+ goto out;
	489	+ }
461	490	/* init_mm is used in this case */
462	491	if (!svm->mm)
463	492	ret = 1;
@@ -564,13 +593,12 @@ static irqreturn_t prq_event_thread(int irq, void *d) · 100 unmodified lines
564	593
565	594	if (!svm \|\| svm->pasid != req->pasid) {
566	595	rcu_read_lock();
567		− svm = intel_pasid_lookup_id(req->pasid);
	596	+ svm = ioasid_find(NULL, req->pasid, NULL);
568	597	/* It can't go away, because the driver is not permitted
569	598	* to unbind the mm while any page faults are outstanding.
570	599	* So we only need RCU to protect the internal idr code. */
571	600	rcu_read_unlock();
572		−
573		− if (!svm) {
	601	+ if (IS_ERR_OR_NULL(svm)) {
574	602	pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n",
575	603	iommu->name, req->pasid, ((unsigned long long *)req)[0],
576	604	((unsigned long long *)req)[1]);
@@ -654,11 +682,10 @@ static irqreturn_t prq_event_thread(int irq, void *d) · 77 unmodified lines
654	682	if (req->priv_data_present)
655	683	memcpy(&resp.qw2, req->priv_data,
656	684	sizeof(req->priv_data));
	685	+ resp.qw2 = 0;
	686	+ resp.qw3 = 0;
	687	+ qi_submit_sync(&resp, iommu);
657	688	}
658		− resp.qw2 = 0;
659		− resp.qw3 = 0;
660		− qi_submit_sync(&resp, iommu);
661		−
662	689	head = (head + sizeof(*req)) & PRQ_RING_MASK;
663	690	}
664	691

drivers/iommu/io-pgtable-arm-v7s.c+9−13

@@ -149,8 +149,6 @@ · 148 unmodified lines
149	149	#define ARM_V7S_TTBR_IRGN_ATTR(attr) \
150	150	((((attr) & 0x1) << 6) \| (((attr) & 0x2) >> 1))
151	151
152		−#define ARM_V7S_TCR_PD1 BIT(5)
153		−
154	152	#ifdef CONFIG_ZONE_DMA32
155	153	#define ARM_V7S_TABLE_GFP_DMA GFP_DMA32
156	154	#define ARM_V7S_TABLE_SLAB_FLAGS SLAB_CACHE_DMA32
@@ -798,8 +796,8 @@ static struct io_pgtable arm_v7s_alloc_pgtable(struct io_pgtable_cfg cfg, · 641 unmodified lines
798	796	*/
799	797	cfg->pgsize_bitmap &= SZ_4K \| SZ_64K \| SZ_1M \| SZ_16M;
800	798
801		− /* TCR: T0SZ=0, disable TTBR1 */
802		− cfg->arm_v7s_cfg.tcr = ARM_V7S_TCR_PD1;
	799	+ /* TCR: T0SZ=0, EAE=0 (if applicable) */
	800	+ cfg->arm_v7s_cfg.tcr = 0;
803	801
804	802	/*
805	803	* TEX remap: the indices used map to the closest equivalent types
@@ -822,15 +820,13 @@ static struct io_pgtable arm_v7s_alloc_pgtable(struct io_pgtable_cfg cfg, · 16 unmodified lines
822	820	/* Ensure the empty pgd is visible before any actual TTBR write */
823	821	wmb();
824	822
825		− /* TTBRs */
826		− cfg->arm_v7s_cfg.ttbr[0] = virt_to_phys(data->pgd) \|
827		− ARM_V7S_TTBR_S \| ARM_V7S_TTBR_NOS \|
828		− (cfg->coherent_walk ?
829		− (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) \|
830		− ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) :
831		− (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) \|
832		− ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC)));
833		− cfg->arm_v7s_cfg.ttbr[1] = 0;
	823	+ /* TTBR */
	824	+ cfg->arm_v7s_cfg.ttbr = virt_to_phys(data->pgd) \| ARM_V7S_TTBR_S \|
	825	+ (cfg->coherent_walk ? (ARM_V7S_TTBR_NOS \|
	826	+ ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) \|
	827	+ ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) :
	828	+ (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) \|
	829	+ ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC)));
834	830	return &data->iop;
835	831
836	832	out_free_data:

drivers/iommu/io-pgtable-arm.c+77−87

@@ -100,40 +100,29 @@ · 99 unmodified lines
100	100	#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)
101	101
102	102	/* Register bits */
103		−#define ARM_32_LPAE_TCR_EAE (1 << 31)
104		−#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31)
	103	+#define ARM_LPAE_TCR_TG0_4K 0
	104	+#define ARM_LPAE_TCR_TG0_64K 1
	105	+#define ARM_LPAE_TCR_TG0_16K 2
105	106
106		−#define ARM_LPAE_TCR_EPD1 (1 << 23)
	107	+#define ARM_LPAE_TCR_TG1_16K 1
	108	+#define ARM_LPAE_TCR_TG1_4K 2
	109	+#define ARM_LPAE_TCR_TG1_64K 3
107	110
108		−#define ARM_LPAE_TCR_TG0_4K (0 << 14)
109		−#define ARM_LPAE_TCR_TG0_64K (1 << 14)
110		−#define ARM_LPAE_TCR_TG0_16K (2 << 14)
111		−
112		−#define ARM_LPAE_TCR_SH0_SHIFT 12
113		−#define ARM_LPAE_TCR_SH0_MASK 0x3
114	111	#define ARM_LPAE_TCR_SH_NS 0
115	112	#define ARM_LPAE_TCR_SH_OS 2
116	113	#define ARM_LPAE_TCR_SH_IS 3
117	114
118		−#define ARM_LPAE_TCR_ORGN0_SHIFT 10
119		−#define ARM_LPAE_TCR_IRGN0_SHIFT 8
120		−#define ARM_LPAE_TCR_RGN_MASK 0x3
121	115	#define ARM_LPAE_TCR_RGN_NC 0
122	116	#define ARM_LPAE_TCR_RGN_WBWA 1
123	117	#define ARM_LPAE_TCR_RGN_WT 2
124	118	#define ARM_LPAE_TCR_RGN_WB 3
125	119
126		−#define ARM_LPAE_TCR_SL0_SHIFT 6
127		−#define ARM_LPAE_TCR_SL0_MASK 0x3
	120	+#define ARM_LPAE_VTCR_SL0_MASK 0x3
128	121
129	122	#define ARM_LPAE_TCR_T0SZ_SHIFT 0
130		−#define ARM_LPAE_TCR_SZ_MASK 0xf
131		−
132		−#define ARM_LPAE_TCR_PS_SHIFT 16
133		−#define ARM_LPAE_TCR_PS_MASK 0x7
134	123
135		−#define ARM_LPAE_TCR_IPS_SHIFT 32
136		−#define ARM_LPAE_TCR_IPS_MASK 0x7
	124	+#define ARM_LPAE_VTCR_PS_SHIFT 16
	125	+#define ARM_LPAE_VTCR_PS_MASK 0x7
137	126
138	127	#define ARM_LPAE_TCR_PS_32_BIT 0x0ULL
139	128	#define ARM_LPAE_TCR_PS_36_BIT 0x1ULL
@@ -293,17 +282,11 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, · 153 unmodified lines
293	282	{
294	283	arm_lpae_iopte pte = prot;
295	284
296		− if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
297		− pte \|= ARM_LPAE_PTE_NS;
298		−
299	285	if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
300	286	pte \|= ARM_LPAE_PTE_TYPE_PAGE;
301	287	else
302	288	pte \|= ARM_LPAE_PTE_TYPE_BLOCK;
303	289
304		− if (data->iop.fmt != ARM_MALI_LPAE)
305		− pte \|= ARM_LPAE_PTE_AF;
306		− pte \|= ARM_LPAE_PTE_SH_IS;
307	290	pte \|= paddr_to_iopte(paddr, data);
308	291
309	292	__arm_lpae_set_pte(ptep, pte, &data->iop.cfg);
@@ -460,9 +443,20 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, · 150 unmodified lines
460	443	<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
461	444	}
462	445
	446	+ if (prot & IOMMU_CACHE)
	447	+ pte \|= ARM_LPAE_PTE_SH_IS;
	448	+ else
	449	+ pte \|= ARM_LPAE_PTE_SH_OS;
	450	+
463	451	if (prot & IOMMU_NOEXEC)
464	452	pte \|= ARM_LPAE_PTE_XN;
465	453
	454	+ if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
	455	+ pte \|= ARM_LPAE_PTE_NS;
	456	+
	457	+ if (data->iop.fmt != ARM_MALI_LPAE)
	458	+ pte \|= ARM_LPAE_PTE_AF;
	459	+
466	460	return pte;
467	461	}
468	462
@@ -474,6 +468,7 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, · 5 unmodified lines
474	468	arm_lpae_iopte *ptep = data->pgd;
475	469	int ret, lvl = data->start_level;
476	470	arm_lpae_iopte prot;
	471	+ long iaext = (long)iova >> cfg->ias;
477	472
478	473	/* If no access, then nothing to do */
479	474	if (!(iommu_prot & (IOMMU_READ \| IOMMU_WRITE)))
@@ -482,7 +477,9 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, · 2 unmodified lines
482	477	if (WARN_ON(!size \|\| (size & cfg->pgsize_bitmap) != size))
483	478	return -EINVAL;
484	479
485		− if (WARN_ON(iova >> data->iop.cfg.ias \|\| paddr >> data->iop.cfg.oas))
	480	+ if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
	481	+ iaext = ~iaext;
	482	+ if (WARN_ON(iaext \|\| paddr >> cfg->oas))
486	483	return -ERANGE;
487	484
488	485	prot = arm_lpae_prot_to_pte(data, iommu_prot);
@@ -648,11 +645,14 @@ static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, · 159 unmodified lines
648	645	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
649	646	struct io_pgtable_cfg *cfg = &data->iop.cfg;
650	647	arm_lpae_iopte *ptep = data->pgd;
	648	+ long iaext = (long)iova >> cfg->ias;
651	649
652	650	if (WARN_ON(!size \|\| (size & cfg->pgsize_bitmap) != size))
653	651	return 0;
654	652
655		− if (WARN_ON(iova >> data->iop.cfg.ias))
	653	+ if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
	654	+ iaext = ~iaext;
	655	+ if (WARN_ON(iaext))
656	656	return 0;
657	657
658	658	return __arm_lpae_unmap(data, gather, iova, size, data->start_level, ptep);
@@ -787,9 +787,12 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg cfg, void cookie) · 128 unmodified lines
787	787	{
788	788	u64 reg;
789	789	struct arm_lpae_io_pgtable *data;
	790	+ typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;
	791	+ bool tg1;
790	792
791	793	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS \|
792		− IO_PGTABLE_QUIRK_NON_STRICT))
	794	+ IO_PGTABLE_QUIRK_NON_STRICT \|
	795	+ IO_PGTABLE_QUIRK_ARM_TTBR1))
793	796	return NULL;
794	797
795	798	data = arm_lpae_alloc_pgtable(cfg);
@@ -798,58 +801,55 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg cfg, void cookie) · 2 unmodified lines
798	801
799	802	/* TCR */
800	803	if (cfg->coherent_walk) {
801		− reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) \|
802		− (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) \|
803		− (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
	804	+ tcr->sh = ARM_LPAE_TCR_SH_IS;
	805	+ tcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
	806	+ tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
804	807	} else {
805		− reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) \|
806		− (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) \|
807		− (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT);
	808	+ tcr->sh = ARM_LPAE_TCR_SH_OS;
	809	+ tcr->irgn = ARM_LPAE_TCR_RGN_NC;
	810	+ tcr->orgn = ARM_LPAE_TCR_RGN_NC;
808	811	}
809	812
	813	+ tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;
810	814	switch (ARM_LPAE_GRANULE(data)) {
811	815	case SZ_4K:
812		− reg \|= ARM_LPAE_TCR_TG0_4K;
	816	+ tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K;
813	817	break;
814	818	case SZ_16K:
815		− reg \|= ARM_LPAE_TCR_TG0_16K;
	819	+ tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K;
816	820	break;
817	821	case SZ_64K:
818		− reg \|= ARM_LPAE_TCR_TG0_64K;
	822	+ tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K;
819	823	break;
820	824	}
821	825
822	826	switch (cfg->oas) {
823	827	case 32:
824		− reg \|= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT);
	828	+ tcr->ips = ARM_LPAE_TCR_PS_32_BIT;
825	829	break;
826	830	case 36:
827		− reg \|= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT);
	831	+ tcr->ips = ARM_LPAE_TCR_PS_36_BIT;
828	832	break;
829	833	case 40:
830		− reg \|= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT);
	834	+ tcr->ips = ARM_LPAE_TCR_PS_40_BIT;
831	835	break;
832	836	case 42:
833		− reg \|= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT);
	837	+ tcr->ips = ARM_LPAE_TCR_PS_42_BIT;
834	838	break;
835	839	case 44:
836		− reg \|= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT);
	840	+ tcr->ips = ARM_LPAE_TCR_PS_44_BIT;
837	841	break;
838	842	case 48:
839		− reg \|= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT);
	843	+ tcr->ips = ARM_LPAE_TCR_PS_48_BIT;
840	844	break;
841	845	case 52:
842		− reg \|= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_IPS_SHIFT);
	846	+ tcr->ips = ARM_LPAE_TCR_PS_52_BIT;
843	847	break;
844	848	default:
845	849	goto out_free_data;
846	850	}
847	851
848		− reg \|= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
849		−
850		− /* Disable speculative walks through TTBR1 */
851		− reg \|= ARM_LPAE_TCR_EPD1;
852		− cfg->arm_lpae_s1_cfg.tcr = reg;
	852	+ tcr->tsz = 64ULL - cfg->ias;
853	853
854	854	/* MAIRs */
855	855	reg = (ARM_LPAE_MAIR_ATTR_NC
@@ -872,9 +872,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg cfg, void cookie) · 16 unmodified lines
872	872	/* Ensure the empty pgd is visible before any actual TTBR write */
873	873	wmb();
874	874
875		− /* TTBRs */
876		− cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
877		− cfg->arm_lpae_s1_cfg.ttbr[1] = 0;
	875	+ /* TTBR */
	876	+ cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
878	877	return &data->iop;
879	878
880	879	out_free_data:
@@ -885,8 +884,9 @@ out_free_data: · 4 unmodified lines
885	884	static struct io_pgtable *
886	885	arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg cfg, void cookie)
887	886	{
888		− u64 reg, sl;
	887	+ u64 sl;
889	888	struct arm_lpae_io_pgtable *data;
	889	+ typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;
890	890
891	891	/* The NS quirk doesn't apply at stage 2 */
892	892	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NON_STRICT))
@@ -911,55 +911,59 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg cfg, void cookie) · 18 unmodified lines
911	911	}
912	912
913	913	/* VTCR */
914		− reg = ARM_64_LPAE_S2_TCR_RES1 \|
915		− (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) \|
916		− (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) \|
917		− (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
	914	+ if (cfg->coherent_walk) {
	915	+ vtcr->sh = ARM_LPAE_TCR_SH_IS;
	916	+ vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
	917	+ vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
	918	+ } else {
	919	+ vtcr->sh = ARM_LPAE_TCR_SH_OS;
	920	+ vtcr->irgn = ARM_LPAE_TCR_RGN_NC;
	921	+ vtcr->orgn = ARM_LPAE_TCR_RGN_NC;
	922	+ }
918	923
919	924	sl = data->start_level;
920	925
921	926	switch (ARM_LPAE_GRANULE(data)) {
922	927	case SZ_4K:
923		− reg \|= ARM_LPAE_TCR_TG0_4K;
	928	+ vtcr->tg = ARM_LPAE_TCR_TG0_4K;
924	929	sl++; /* SL0 format is different for 4K granule size */
925	930	break;
926	931	case SZ_16K:
927		− reg \|= ARM_LPAE_TCR_TG0_16K;
	932	+ vtcr->tg = ARM_LPAE_TCR_TG0_16K;
928	933	break;
929	934	case SZ_64K:
930		− reg \|= ARM_LPAE_TCR_TG0_64K;
	935	+ vtcr->tg = ARM_LPAE_TCR_TG0_64K;
931	936	break;
932	937	}
933	938
934	939	switch (cfg->oas) {
935	940	case 32:
936		− reg \|= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_PS_SHIFT);
	941	+ vtcr->ps = ARM_LPAE_TCR_PS_32_BIT;
937	942	break;
938	943	case 36:
939		− reg \|= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_PS_SHIFT);
	944	+ vtcr->ps = ARM_LPAE_TCR_PS_36_BIT;
940	945	break;
941	946	case 40:
942		− reg \|= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_PS_SHIFT);
	947	+ vtcr->ps = ARM_LPAE_TCR_PS_40_BIT;
943	948	break;
944	949	case 42:
945		− reg \|= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_PS_SHIFT);
	950	+ vtcr->ps = ARM_LPAE_TCR_PS_42_BIT;
946	951	break;
947	952	case 44:
948		− reg \|= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_PS_SHIFT);
	953	+ vtcr->ps = ARM_LPAE_TCR_PS_44_BIT;
949	954	break;
950	955	case 48:
951		− reg \|= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT);
	956	+ vtcr->ps = ARM_LPAE_TCR_PS_48_BIT;
952	957	break;
953	958	case 52:
954		− reg \|= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_PS_SHIFT);
	959	+ vtcr->ps = ARM_LPAE_TCR_PS_52_BIT;
955	960	break;
956	961	default:
957	962	goto out_free_data;
958	963	}
959	964
960		− reg \|= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
961		− reg \|= (~sl & ARM_LPAE_TCR_SL0_MASK) << ARM_LPAE_TCR_SL0_SHIFT;
962		− cfg->arm_lpae_s2_cfg.vtcr = reg;
	965	+ vtcr->tsz = 64ULL - cfg->ias;
	966	+ vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK;
963	967
964	968	/* Allocate pgd pages */
965	969	data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
@@ -982,35 +986,21 @@ out_free_data: · 16 unmodified lines
982	986	static struct io_pgtable *
983	987	arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg cfg, void cookie)
984	988	{
985		− struct io_pgtable *iop;
986		−
987	989	if (cfg->ias > 32 \|\| cfg->oas > 40)
988	990	return NULL;
989	991
990	992	cfg->pgsize_bitmap &= (SZ_4K \| SZ_2M \| SZ_1G);
991		− iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
992		− if (iop) {
993		− cfg->arm_lpae_s1_cfg.tcr \|= ARM_32_LPAE_TCR_EAE;
994		− cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff;
995		− }
996		−
997		− return iop;
	993	+ return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
998	994	}
999	995
1000	996	static struct io_pgtable *
1001	997	arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg cfg, void cookie)
1002	998	{
1003		− struct io_pgtable *iop;
1004		−
1005	999	if (cfg->ias > 40 \|\| cfg->oas > 40)
1006	1000	return NULL;
1007	1001
1008	1002	cfg->pgsize_bitmap &= (SZ_4K \| SZ_2M \| SZ_1G);
1009		− iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
1010		− if (iop)
1011		− cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff;
1012		−
1013		− return iop;
	1003	+ return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
1014	1004	}
1015	1005
1016	1006	static struct io_pgtable *

drivers/iommu/io-pgtable.c+1−1

@@ -63,7 +63,7 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) · 62 unmodified lines
63	63	if (!ops)
64	64	return;
65	65
66		− iop = container_of(ops, struct io_pgtable, ops);
	66	+ iop = io_pgtable_ops_to_pgtable(ops);
67	67	io_pgtable_tlb_flush_all(iop);
68	68	io_pgtable_init_table[iop->fmt]->free(iop);
69	69	}

drivers/iommu/iommu-sysfs.c+5−0

@@ -87,6 +87,7 @@ error: · 86 unmodified lines
87	87	put_device(iommu->dev);
88	88	return ret;
89	89	}
	90	+EXPORT_SYMBOL_GPL(iommu_device_sysfs_add);
90	91
91	92	void iommu_device_sysfs_remove(struct iommu_device *iommu)
92	93	{
@@ -94,6 +95,8 @@ void iommu_device_sysfs_remove(struct iommu_device *iommu) · 1 unmodified line
94	95	device_unregister(iommu->dev);
95	96	iommu->dev = NULL;
96	97	}
	98	+EXPORT_SYMBOL_GPL(iommu_device_sysfs_remove);
	99	+
97	100	/*
98	101	* IOMMU drivers can indicate a device is managed by a given IOMMU using
99	102	* this interface. A link to the device will be created in the "devices"
@@ -119,6 +122,7 @@ int iommu_device_link(struct iommu_device iommu, struct device link) · 19 unmodified lines
119	122
120	123	return ret;
121	124	}
	125	+EXPORT_SYMBOL_GPL(iommu_device_link);
122	126
123	127	void iommu_device_unlink(struct iommu_device iommu, struct device link)
124	128	{
@@ -128,3 +132,4 @@ void iommu_device_unlink(struct iommu_device iommu, struct device link) · 3 unmodified lines
128	132	sysfs_remove_link(&link->kobj, "iommu");
129	133	sysfs_remove_link_from_group(&iommu->dev->kobj, "devices", dev_name(link));
130	134	}
	135	+EXPORT_SYMBOL_GPL(iommu_device_unlink);

drivers/iommu/iommu.c+49−2

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	#include <linux/bitops.h>
23	23	#include <linux/property.h>
24	24	#include <linux/fsl/mc.h>
	25	+#include <linux/module.h>
25	26	#include <trace/events/iommu.h>
26	27
27	28	static struct kset *iommu_group_kset;
@@ -141,6 +142,7 @@ int iommu_device_register(struct iommu_device *iommu) · 113 unmodified lines
141	142	spin_unlock(&iommu_device_lock);
142	143	return 0;
143	144	}
	145	+EXPORT_SYMBOL_GPL(iommu_device_register);
144	146
145	147	void iommu_device_unregister(struct iommu_device *iommu)
146	148	{
@@ -148,6 +150,7 @@ void iommu_device_unregister(struct iommu_device *iommu) · 1 unmodified line
148	150	list_del(&iommu->list);
149	151	spin_unlock(&iommu_device_lock);
150	152	}
	153	+EXPORT_SYMBOL_GPL(iommu_device_unregister);
151	154
152	155	static struct iommu_param iommu_get_dev_param(struct device dev)
153	156	{
@@ -183,10 +186,21 @@ int iommu_probe_device(struct device *dev) · 29 unmodified lines
183	186	if (!iommu_get_dev_param(dev))
184	187	return -ENOMEM;
185	188
	189	+ if (!try_module_get(ops->owner)) {
	190	+ ret = -EINVAL;
	191	+ goto err_free_dev_param;
	192	+ }
	193	+
186	194	ret = ops->add_device(dev);
187	195	if (ret)
188		− iommu_free_dev_param(dev);
	196	+ goto err_module_put;
	197	+
	198	+ return 0;
189	199
	200	+err_module_put:
	201	+ module_put(ops->owner);
	202	+err_free_dev_param:
	203	+ iommu_free_dev_param(dev);
190	204	return ret;
191	205	}
192	206
@@ -197,7 +211,10 @@ void iommu_release_device(struct device *dev) · 4 unmodified lines
197	211	if (dev->iommu_group)
198	212	ops->remove_device(dev);
199	213
200		− iommu_free_dev_param(dev);
	214	+ if (dev->iommu_param) {
	215	+ module_put(ops->owner);
	216	+ iommu_free_dev_param(dev);
	217	+ }
201	218	}
202	219
203	220	static struct iommu_domain __iommu_domain_alloc(struct bus_type bus,
@@ -887,6 +904,7 @@ struct iommu_group iommu_group_ref_get(struct iommu_group group) · 683 unmodified lines
887	904	kobject_get(group->devices_kobj);
888	905	return group;
889	906	}
	907	+EXPORT_SYMBOL_GPL(iommu_group_ref_get);
890	908
891	909	/**
892	910	* iommu_group_put - Decrement group reference
@@ -1260,6 +1278,7 @@ struct iommu_group generic_device_group(struct device dev) · 367 unmodified lines
1260	1278	{
1261	1279	return iommu_group_alloc();
1262	1280	}
	1281	+EXPORT_SYMBOL_GPL(generic_device_group);
1263	1282
1264	1283	/*
1265	1284	* Use standard PCI bus topology, isolation features, and DMA alias quirks
@@ -1327,6 +1346,7 @@ struct iommu_group pci_device_group(struct device dev) · 61 unmodified lines
1327	1346	/* No shared group found, allocate new */
1328	1347	return iommu_group_alloc();
1329	1348	}
	1349	+EXPORT_SYMBOL_GPL(pci_device_group);
1330	1350
1331	1351	/* Get the IOMMU group for device on fsl-mc bus */
1332	1352	struct iommu_group fsl_mc_device_group(struct device dev)
@@ -1339,6 +1359,7 @@ struct iommu_group fsl_mc_device_group(struct device dev) · 6 unmodified lines
1339	1359	group = iommu_group_alloc();
1340	1360	return group;
1341	1361	}
	1362	+EXPORT_SYMBOL_GPL(fsl_mc_device_group);
1342	1363
1343	1364	/**
1344	1365	* iommu_group_get_for_dev - Find or create the IOMMU group for a device
@@ -1407,6 +1428,7 @@ struct iommu_group iommu_group_get_for_dev(struct device dev) · 62 unmodified lines
1407	1428
1408	1429	return group;
1409	1430	}
	1431	+EXPORT_SYMBOL(iommu_group_get_for_dev);
1410	1432
1411	1433	struct iommu_domain iommu_group_default_domain(struct iommu_group group)
1412	1434	{
@@ -1537,6 +1559,11 @@ int bus_set_iommu(struct bus_type bus, const struct iommu_ops ops) · 124 unmodified lines
1537	1559	{
1538	1560	int err;
1539	1561
	1562	+ if (ops == NULL) {
	1563	+ bus->iommu_ops = NULL;
	1564	+ return 0;
	1565	+ }
	1566	+
1540	1567	if (bus->iommu_ops != NULL)
1541	1568	return -EBUSY;
1542	1569
@@ -2230,6 +2257,25 @@ void iommu_put_resv_regions(struct device dev, struct list_head list) · 687 unmodified lines
2230	2257	ops->put_resv_regions(dev, list);
2231	2258	}
2232	2259
	2260	+/**
	2261	+ * generic_iommu_put_resv_regions - Reserved region driver helper
	2262	+ * @dev: device for which to free reserved regions
	2263	+ * @list: reserved region list for device
	2264	+ *
	2265	+ * IOMMU drivers can use this to implement their .put_resv_regions() callback
	2266	+ * for simple reservations. Memory allocated for each reserved region will be
	2267	+ * freed. If an IOMMU driver allocates additional resources per region, it is
	2268	+ * going to have to implement a custom callback.
	2269	+ */
	2270	+void generic_iommu_put_resv_regions(struct device dev, struct list_head list)
	2271	+{
	2272	+ struct iommu_resv_region entry, next;
	2273	+
	2274	+ list_for_each_entry_safe(entry, next, list, list)
	2275	+ kfree(entry);
	2276	+}
	2277	+EXPORT_SYMBOL(generic_iommu_put_resv_regions);
	2278	+
2233	2279	struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
2234	2280	size_t length, int prot,
2235	2281	enum iommu_resv_type type)
@@ -2247,6 +2293,7 @@ struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, · 11 unmodified lines
2247	2293	region->type = type;
2248	2294	return region;
2249	2295	}
	2296	+EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
2250	2297
2251	2298	static int
2252	2299	request_default_domain_for_dev(struct device *dev, unsigned long type)

drivers/iommu/iova.c+1−1

@@ -233,7 +233,7 @@ static DEFINE_MUTEX(iova_cache_mutex); · 232 unmodified lines
233	233
234	234	struct iova *alloc_iova_mem(void)
235	235	{
236		− return kmem_cache_zalloc(iova_cache, GFP_ATOMIC);
	236	+ return kmem_cache_zalloc(iova_cache, GFP_ATOMIC \| __GFP_NOWARN);
237	237	}
238	238	EXPORT_SYMBOL(alloc_iova_mem);
239	239

drivers/iommu/ipmmu-vmsa.c+1−1

@@ -374,7 +374,7 @@ static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain) · 373 unmodified lines
374	374	u32 tmp;
375	375
376	376	/* TTBR0 */
377		− ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0];
	377	+ ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr;
378	378	ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr);
379	379	ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32);
380	380

drivers/iommu/msm_iommu.c+2−2

@@ -279,8 +279,8 @@ static void __program_context(void __iomem *base, int ctx, · 278 unmodified lines
279	279	SET_V2PCFG(base, ctx, 0x3);
280	280
281	281	SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr);
282		− SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]);
283		− SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]);
	282	+ SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr);
	283	+ SET_TTBR1(base, ctx, 0);
284	284
285	285	/* Set prrr and nmrr */
286	286	SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr);

drivers/iommu/mtk_iommu.c+2−2

@@ -367,7 +367,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, · 366 unmodified lines
367	367	/* Update the pgtable base address register of the M4U HW */
368	368	if (!data->m4u_dom) {
369	369	data->m4u_dom = dom;
370		− writel(dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK,
	370	+ writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK,
371	371	data->base + REG_MMU_PT_BASE_ADDR);
372	372	}
373	373
@@ -765,7 +765,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) · 391 unmodified lines
765	765	writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
766	766	writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);
767	767	if (m4u_dom)
768		− writel(m4u_dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK,
	768	+ writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK,
769	769	base + REG_MMU_PT_BASE_ADDR);
770	770	return 0;
771	771	}

drivers/iommu/of_iommu.c+18−7

@@ -8,11 +8,12 @@ · 7 unmodified lines
8	8	#include <linux/export.h>
9	9	#include <linux/iommu.h>
10	10	#include <linux/limits.h>
11		−#include <linux/pci.h>
	11	+#include <linux/module.h>
12	12	#include <linux/msi.h>
13	13	#include <linux/of.h>
14	14	#include <linux/of_iommu.h>
15	15	#include <linux/of_pci.h>
	16	+#include <linux/pci.h>
16	17	#include <linux/slab.h>
17	18	#include <linux/fsl/mc.h>
18	19
@@ -91,16 +92,16 @@ static int of_iommu_xlate(struct device *dev, · 72 unmodified lines
91	92	{
92	93	const struct iommu_ops *ops;
93	94	struct fwnode_handle *fwnode = &iommu_spec->np->fwnode;
94		− int err;
	95	+ int ret;
95	96
96	97	ops = iommu_ops_from_fwnode(fwnode);
97	98	if ((ops && !ops->of_xlate) \|\|
98	99	!of_device_is_available(iommu_spec->np))
99	100	return NO_IOMMU;
100	101
101		− err = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops);
102		− if (err)
103		− return err;
	102	+ ret = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops);
	103	+ if (ret)
	104	+ return ret;
104	105	/*
105	106	* The otherwise-empty fwspec handily serves to indicate the specific
106	107	* IOMMU device we're waiting for, which will be useful if we ever get
@@ -109,7 +110,12 @@ static int of_iommu_xlate(struct device *dev, · 2 unmodified lines
109	110	if (!ops)
110	111	return driver_deferred_probe_check_state(dev);
111	112
112		− return ops->of_xlate(dev, iommu_spec);
	113	+ if (!try_module_get(ops->owner))
	114	+ return -ENODEV;
	115	+
	116	+ ret = ops->of_xlate(dev, iommu_spec);
	117	+ module_put(ops->owner);
	118	+ return ret;
113	119	}
114	120
115	121	struct of_pci_iommu_alias_info {
@@ -179,6 +185,7 @@ const struct iommu_ops of_iommu_configure(struct device dev, · 63 unmodified lines
179	185	.np = master_np,
180	186	};
181	187
	188	+ pci_request_acs();
182	189	err = pci_for_each_dma_alias(to_pci_dev(dev),
183	190	of_pci_iommu_init, &info);
184	191	} else if (dev_is_fsl_mc(dev)) {
@@ -196,8 +203,12 @@ const struct iommu_ops of_iommu_configure(struct device dev, · 11 unmodified lines
196	203	if (err)
197	204	break;
198	205	}
199		− }
200	206
	207	+ fwspec = dev_iommu_fwspec_get(dev);
	208	+ if (!err && fwspec)
	209	+ of_property_read_u32(master_np, "pasid-num-bits",
	210	+ &fwspec->num_pasid_bits);
	211	+ }
201	212
202	213	/*
203	214	* Two success conditions can be represented by non-negative err here:

drivers/iommu/qcom_iommu.c+12−13

@@ -201,7 +201,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) · 200 unmodified lines
201	201
202	202	fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
203	203
204		− if (!(fsr & FSR_FAULT))
	204	+ if (!(fsr & ARM_SMMU_FSR_FAULT))
205	205	return IRQ_NONE;
206	206
207	207	fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
@@ -215,7 +215,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) · 7 unmodified lines
215	215	}
216	216
217	217	iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);
218		− iommu_writel(ctx, ARM_SMMU_CB_RESUME, RESUME_TERMINATE);
	218	+ iommu_writel(ctx, ARM_SMMU_CB_RESUME, ARM_SMMU_RESUME_TERMINATE);
219	219
220	220	return IRQ_HANDLED;
221	221	}
@@ -269,18 +269,15 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain, · 47 unmodified lines
269	269
270	270	/* TTBRs */
271	271	iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
272		− pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] \|
273		− FIELD_PREP(TTBRn_ASID, ctx->asid));
274		− iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
275		− pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] \|
276		− FIELD_PREP(TTBRn_ASID, ctx->asid));
	272	+ pgtbl_cfg.arm_lpae_s1_cfg.ttbr \|
	273	+ FIELD_PREP(ARM_SMMU_TTBRn_ASID, ctx->asid));
	274	+ iommu_writeq(ctx, ARM_SMMU_CB_TTBR1, 0);
277	275
278	276	/* TCR */
279	277	iommu_writel(ctx, ARM_SMMU_CB_TCR2,
280		− (pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) \|
281		− FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM));
	278	+ arm_smmu_lpae_tcr2(&pgtbl_cfg));
282	279	iommu_writel(ctx, ARM_SMMU_CB_TCR,
283		− pgtbl_cfg.arm_lpae_s1_cfg.tcr);
	280	+ arm_smmu_lpae_tcr(&pgtbl_cfg) \| ARM_SMMU_TCR_EAE);
284	281
285	282	/* MAIRs (stage-1 only) */
286	283	iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
@@ -289,11 +286,13 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain, · 2 unmodified lines
289	286	pgtbl_cfg.arm_lpae_s1_cfg.mair >> 32);
290	287
291	288	/* SCTLR */
292		− reg = SCTLR_CFIE \| SCTLR_CFRE \| SCTLR_AFE \| SCTLR_TRE \|
293		− SCTLR_M \| SCTLR_S1_ASIDPNE \| SCTLR_CFCFG;
	289	+ reg = ARM_SMMU_SCTLR_CFIE \| ARM_SMMU_SCTLR_CFRE \|
	290	+ ARM_SMMU_SCTLR_AFE \| ARM_SMMU_SCTLR_TRE \|
	291	+ ARM_SMMU_SCTLR_M \| ARM_SMMU_SCTLR_S1_ASIDPNE \|
	292	+ ARM_SMMU_SCTLR_CFCFG;
294	293
295	294	if (IS_ENABLED(CONFIG_BIG_ENDIAN))
296		− reg \|= SCTLR_E;
	295	+ reg \|= ARM_SMMU_SCTLR_E;
297	296
298	297	iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
299	298

drivers/iommu/virtio-iommu.c+3−11

@@ -837,14 +837,6 @@ static void viommu_get_resv_regions(struct device dev, struct list_head head) · 836 unmodified lines
837	837	iommu_dma_get_resv_regions(dev, head);
838	838	}
839	839
840		−static void viommu_put_resv_regions(struct device dev, struct list_head head)
841		−{
842		− struct iommu_resv_region entry, next;
843		−
844		− list_for_each_entry_safe(entry, next, head, list)
845		− kfree(entry);
846		−}
847		−
848	840	static struct iommu_ops viommu_ops;
849	841	static struct virtio_driver virtio_iommu_drv;
850	842
@@ -914,7 +906,7 @@ static int viommu_add_device(struct device *dev) · 63 unmodified lines
914	906	err_unlink_dev:
915	907	iommu_device_unlink(&viommu->iommu, dev);
916	908	err_free_dev:
917		− viommu_put_resv_regions(dev, &vdev->resv_regions);
	909	+ generic_iommu_put_resv_regions(dev, &vdev->resv_regions);
918	910	kfree(vdev);
919	911
920	912	return ret;
@@ -932,7 +924,7 @@ static void viommu_remove_device(struct device *dev) · 11 unmodified lines
932	924
933	925	iommu_group_remove_device(dev);
934	926	iommu_device_unlink(&vdev->viommu->iommu, dev);
935		− viommu_put_resv_regions(dev, &vdev->resv_regions);
	927	+ generic_iommu_put_resv_regions(dev, &vdev->resv_regions);
936	928	kfree(vdev);
937	929	}
938	930
@@ -961,7 +953,7 @@ static struct iommu_ops viommu_ops = { · 22 unmodified lines
961	953	.remove_device = viommu_remove_device,
962	954	.device_group = viommu_device_group,
963	955	.get_resv_regions = viommu_get_resv_regions,
964		− .put_resv_regions = viommu_put_resv_regions,
	956	+ .put_resv_regions = generic_iommu_put_resv_regions,
965	957	.of_xlate = viommu_of_xlate,
966	958	};
967	959

drivers/pci/ats.c+2−0

@@ -69,6 +69,7 @@ int pci_enable_ats(struct pci_dev *dev, int ps) · 68 unmodified lines
69	69	dev->ats_enabled = 1;
70	70	return 0;
71	71	}
	72	+EXPORT_SYMBOL_GPL(pci_enable_ats);
72	73
73	74	/**
74	75	* pci_disable_ats - disable the ATS capability
@@ -87,6 +88,7 @@ void pci_disable_ats(struct pci_dev *dev) · 12 unmodified lines
87	88
88	89	dev->ats_enabled = 0;
89	90	}
	91	+EXPORT_SYMBOL_GPL(pci_disable_ats);
90	92
91	93	void pci_restore_ats_state(struct pci_dev *dev)
92	94	{

drivers/pci/pci.c+1−0

@@ -131,6 +131,7 @@ bool pci_ats_disabled(void) · 130 unmodified lines
131	131	{
132	132	return pcie_ats_disabled;
133	133	}
	134	+EXPORT_SYMBOL_GPL(pci_ats_disabled);
134	135
135	136	/* Disable bridge_d3 for all PCIe ports */
136	137	static bool pci_bridge_d3_disable;

include/linux/intel-iommu.h+18−7

@@ -34,10 +34,13 @@ · 33 unmodified lines
34	34	#define VTD_STRIDE_SHIFT (9)
35	35	#define VTD_STRIDE_MASK (((u64)-1) << VTD_STRIDE_SHIFT)
36	36
37		−#define DMA_PTE_READ (1)
38		−#define DMA_PTE_WRITE (2)
39		−#define DMA_PTE_LARGE_PAGE (1 << 7)
40		−#define DMA_PTE_SNP (1 << 11)
	37	+#define DMA_PTE_READ BIT_ULL(0)
	38	+#define DMA_PTE_WRITE BIT_ULL(1)
	39	+#define DMA_PTE_LARGE_PAGE BIT_ULL(7)
	40	+#define DMA_PTE_SNP BIT_ULL(11)
	41	+
	42	+#define DMA_FL_PTE_PRESENT BIT_ULL(0)
	43	+#define DMA_FL_PTE_XD BIT_ULL(63)
41	44
42	45	#define CONTEXT_TT_MULTI_LEVEL 0
43	46	#define CONTEXT_TT_DEV_IOTLB 1
@@ -435,8 +438,10 @@ enum { · 391 unmodified lines
435	438
436	439	#define VTD_FLAG_TRANS_PRE_ENABLED (1 << 0)
437	440	#define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1)
	441	+#define VTD_FLAG_SVM_CAPABLE (1 << 2)
438	442
439	443	extern int intel_iommu_sm;
	444	+extern spinlock_t device_domain_lock;
440	445
441	446	#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))
442	447	#define pasid_supported(iommu) (sm_supported(iommu) && \
@@ -609,10 +614,11 @@ static inline void dma_clear_pte(struct dma_pte *pte) · 166 unmodified lines
609	614	static inline u64 dma_pte_addr(struct dma_pte *pte)
610	615	{
611	616	#ifdef CONFIG_64BIT
612		− return pte->val & VTD_PAGE_MASK;
	617	+ return pte->val & VTD_PAGE_MASK & (~DMA_FL_PTE_XD);
613	618	#else
614	619	/* Must have a full atomic 64-bit read */
615		− return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
	620	+ return __cmpxchg64(&pte->val, 0ULL, 0ULL) &
	621	+ VTD_PAGE_MASK & (~DMA_FL_PTE_XD);
616	622	#endif
617	623	}
618	624
@@ -645,6 +651,8 @@ extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, · 26 unmodified lines
645	651	unsigned int size_order, u64 type);
646	652	extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
647	653	u16 qdep, u64 addr, unsigned mask);
	654	+void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
	655	+ unsigned long npages, bool ih);
648	656	extern int qi_submit_sync(struct qi_desc desc, struct intel_iommu iommu);
649	657
650	658	extern int dmar_ir_support(void);
@@ -656,9 +664,10 @@ int for_each_device_domain(int (fn)(struct device_domain_info info, · 5 unmodified lines
656	664	void data), void data);
657	665	void iommu_flush_write_buffer(struct intel_iommu *iommu);
658	666	int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev);
	667	+struct dmar_domain find_domain(struct device dev);
659	668
660	669	#ifdef CONFIG_INTEL_IOMMU_SVM
661		−int intel_svm_init(struct intel_iommu *iommu);
	670	+extern void intel_svm_check(struct intel_iommu *iommu);
662	671	extern int intel_svm_enable_prq(struct intel_iommu *iommu);
663	672	extern int intel_svm_finish_prq(struct intel_iommu *iommu);
664	673
@@ -686,6 +695,8 @@ struct intel_svm { · 21 unmodified lines
686	695	};
687	696
688	697	extern struct intel_iommu intel_svm_device_to_iommu(struct device dev);
	698	+#else
	699	+static inline void intel_svm_check(struct intel_iommu *iommu) {}
689	700	#endif
690	701
691	702	#ifdef CONFIG_INTEL_IOMMU_DEBUGFS

include/linux/io-pgtable.h+23−4

@@ -83,12 +83,16 @@ struct io_pgtable_cfg { · 82 unmodified lines
83	83	* IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
84	84	* on unmap, for DMA domains using the flush queue mechanism for
85	85	* delayed invalidation.
	86	+ *
	87	+ * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table
	88	+ * for use in the upper half of a split address space.
86	89	*/
87	90	#define IO_PGTABLE_QUIRK_ARM_NS BIT(0)
88	91	#define IO_PGTABLE_QUIRK_NO_PERMS BIT(1)
89	92	#define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2)
90	93	#define IO_PGTABLE_QUIRK_ARM_MTK_EXT BIT(3)
91	94	#define IO_PGTABLE_QUIRK_NON_STRICT BIT(4)
	95	+ #define IO_PGTABLE_QUIRK_ARM_TTBR1 BIT(5)
92	96	unsigned long quirks;
93	97	unsigned long pgsize_bitmap;
94	98	unsigned int ias;
@@ -100,18 +104,33 @@ struct io_pgtable_cfg { · 5 unmodified lines
100	104	/* Low-level data specific to the table format */
101	105	union {
102	106	struct {
103		− u64 ttbr[2];
104		− u64 tcr;
	107	+ u64 ttbr;
	108	+ struct {
	109	+ u32 ips:3;
	110	+ u32 tg:2;
	111	+ u32 sh:2;
	112	+ u32 orgn:2;
	113	+ u32 irgn:2;
	114	+ u32 tsz:6;
	115	+ } tcr;
105	116	u64 mair;
106	117	} arm_lpae_s1_cfg;
107	118
108	119	struct {
109	120	u64 vttbr;
110		− u64 vtcr;
	121	+ struct {
	122	+ u32 ps:3;
	123	+ u32 tg:2;
	124	+ u32 sh:2;
	125	+ u32 orgn:2;
	126	+ u32 irgn:2;
	127	+ u32 sl:2;
	128	+ u32 tsz:6;
	129	+ } vtcr;
111	130	} arm_lpae_s2_cfg;
112	131
113	132	struct {
114		− u32 ttbr[2];
	133	+ u32 ttbr;
115	134	u32 tcr;
116	135	u32 nmrr;
117	136	u32 prrr;

include/linux/iommu.h+16−3

@@ -246,9 +246,10 @@ struct iommu_iotlb_gather { · 245 unmodified lines
246	246	* @sva_get_pasid: Get PASID associated to a SVA handle
247	247	* @page_response: handle page request response
248	248	* @cache_invalidate: invalidate translation caches
249		− * @pgsize_bitmap: bitmap of all possible supported page sizes
250	249	* @sva_bind_gpasid: bind guest pasid and mm
251	250	* @sva_unbind_gpasid: unbind guest pasid and mm
	251	+ * @pgsize_bitmap: bitmap of all possible supported page sizes
	252	+ * @owner: Driver module providing these ops
252	253	*/
253	254	struct iommu_ops {
254	255	bool (*capable)(enum iommu_cap);
@@ -318,6 +319,7 @@ struct iommu_ops { · 63 unmodified lines
318	319	int (sva_unbind_gpasid)(struct device dev, int pasid);
319	320
320	321	unsigned long pgsize_bitmap;
	322	+ struct module *owner;
321	323	};
322	324
323	325	/**
@@ -386,12 +388,19 @@ void iommu_device_sysfs_remove(struct iommu_device *iommu); · 62 unmodified lines
386	388	int iommu_device_link(struct iommu_device iommu, struct device link);
387	389	void iommu_device_unlink(struct iommu_device iommu, struct device link);
388	390
389		−static inline void iommu_device_set_ops(struct iommu_device *iommu,
390		− const struct iommu_ops *ops)
	391	+static inline void __iommu_device_set_ops(struct iommu_device *iommu,
	392	+ const struct iommu_ops *ops)
391	393	{
392	394	iommu->ops = ops;
393	395	}
394	396
	397	+#define iommu_device_set_ops(iommu, ops) \
	398	+do { \
	399	+ struct iommu_ops __ops = (struct iommu_ops )(ops); \
	400	+ __ops->owner = THIS_MODULE; \
	401	+ __iommu_device_set_ops(iommu, __ops); \
	402	+} while (0)
	403	+
395	404	static inline void iommu_device_set_fwnode(struct iommu_device *iommu,
396	405	struct fwnode_handle *fwnode)
397	406	{
@@ -456,6 +465,8 @@ extern void iommu_set_fault_handler(struct iommu_domain *domain, · 58 unmodified lines
456	465
457	466	extern void iommu_get_resv_regions(struct device dev, struct list_head list);
458	467	extern void iommu_put_resv_regions(struct device dev, struct list_head list);
	468	+extern void generic_iommu_put_resv_regions(struct device *dev,
	469	+ struct list_head *list);
459	470	extern int iommu_request_dm_for_dev(struct device *dev);
460	471	extern int iommu_request_dma_domain_for_dev(struct device *dev);
461	472	extern void iommu_set_default_passthrough(bool cmd_line);
@@ -570,6 +581,7 @@ struct iommu_group fsl_mc_device_group(struct device dev); · 108 unmodified lines
570	581	* @ops: ops for this device's IOMMU
571	582	* @iommu_fwnode: firmware handle for this device's IOMMU
572	583	* @iommu_priv: IOMMU driver private data for this device
	584	+ * @num_pasid_bits: number of PASID bits supported by this device
573	585	* @num_ids: number of associated device IDs
574	586	* @ids: IDs which this device may present to the IOMMU
575	587	*/
@@ -578,6 +590,7 @@ struct iommu_fwspec { · 2 unmodified lines
578	590	struct fwnode_handle *iommu_fwnode;
579	591	void *iommu_priv;
580	592	u32 flags;
	593	+ u32 num_pasid_bits;
581	594	unsigned int num_ids;
582	595	u32 ids[1];
583	596	};

include/linux/pci-ats.h+3−0

@@ -33,6 +33,9 @@ void pci_disable_pasid(struct pci_dev *pdev); · 32 unmodified lines
33	33	int pci_pasid_features(struct pci_dev *pdev);
34	34	int pci_max_pasids(struct pci_dev *pdev);
35	35	#else /* CONFIG_PCI_PASID */
	36	+static inline int pci_enable_pasid(struct pci_dev *pdev, int features)
	37	+{ return -EINVAL; }
	38	+static inline void pci_disable_pasid(struct pci_dev *pdev) { }
36	39	static inline int pci_pasid_features(struct pci_dev *pdev)
37	40	{ return -EINVAL; }
38	41	static inline int pci_max_pasids(struct pci_dev *pdev)

include/trace/events/intel_iommu.h+42−6

@@ -49,12 +49,6 @@ DEFINE_EVENT(dma_map, map_single, · 48 unmodified lines
49	49	TP_ARGS(dev, dev_addr, phys_addr, size)
50	50	);
51	51
52		−DEFINE_EVENT(dma_map, map_sg,
53		− TP_PROTO(struct device *dev, dma_addr_t dev_addr, phys_addr_t phys_addr,
54		− size_t size),
55		− TP_ARGS(dev, dev_addr, phys_addr, size)
56		−);
57		−
58	52	DEFINE_EVENT(dma_map, bounce_map_single,
59	53	TP_PROTO(struct device *dev, dma_addr_t dev_addr, phys_addr_t phys_addr,
60	54	size_t size),
@@ -99,6 +93,48 @@ DEFINE_EVENT(dma_unmap, bounce_unmap_single, · 38 unmodified lines
99	93	TP_ARGS(dev, dev_addr, size)
100	94	);
101	95
	96	+DECLARE_EVENT_CLASS(dma_map_sg,
	97	+ TP_PROTO(struct device *dev, int index, int total,
	98	+ struct scatterlist *sg),
	99	+
	100	+ TP_ARGS(dev, index, total, sg),
	101	+
	102	+ TP_STRUCT__entry(
	103	+ __string(dev_name, dev_name(dev))
	104	+ __field(dma_addr_t, dev_addr)
	105	+ __field(phys_addr_t, phys_addr)
	106	+ __field(size_t, size)
	107	+ __field(int, index)
	108	+ __field(int, total)
	109	+ ),
	110	+
	111	+ TP_fast_assign(
	112	+ __assign_str(dev_name, dev_name(dev));
	113	+ __entry->dev_addr = sg->dma_address;
	114	+ __entry->phys_addr = sg_phys(sg);
	115	+ __entry->size = sg->dma_length;
	116	+ __entry->index = index;
	117	+ __entry->total = total;
	118	+ ),
	119	+
	120	+ TP_printk("dev=%s [%d/%d] dev_addr=0x%llx phys_addr=0x%llx size=%zu",
	121	+ __get_str(dev_name), __entry->index, __entry->total,
	122	+ (unsigned long long)__entry->dev_addr,
	123	+ (unsigned long long)__entry->phys_addr,
	124	+ __entry->size)
	125	+);
	126	+
	127	+DEFINE_EVENT(dma_map_sg, map_sg,
	128	+ TP_PROTO(struct device *dev, int index, int total,
	129	+ struct scatterlist *sg),
	130	+ TP_ARGS(dev, index, total, sg)
	131	+);
	132	+
	133	+DEFINE_EVENT(dma_map_sg, bounce_map_sg,
	134	+ TP_PROTO(struct device *dev, int index, int total,
	135	+ struct scatterlist *sg),
	136	+ TP_ARGS(dev, index, total, sg)
	137	+);
102	138	#endif /* _TRACE_INTEL_IOMMU_H */
103	139
104	140	/* This part must be outside protection */

Merge tag 'iommu-updates-v5.6' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code