lael-labs/linux

Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml+4−0

               - renesas,ipmmu-r8a77990 # R-Car E3
               - renesas,ipmmu-r8a77995 # R-Car D3
               - renesas,ipmmu-r8a779a0 # R-Car V3U
+      - items:
+          - enum:
+              - renesas,ipmmu-r8a779f0 # R-Car S4-8
+          - const: renesas,rcar-gen4-ipmmu-vmsa  # R-Car Gen4
   reg:
     maxItems: 1

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c+1−1

@@ -133,7 +133,7 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev) · 132 unmodified lines
133	* or equal to the system's PAGE_SIZE, with a preference if	133	* or equal to the system's PAGE_SIZE, with a preference if
134	* both are equal.	134	* both are equal.
135	*/	135	*/
136	pgsize_bitmap = tdev->iommu.domain->ops->pgsize_bitmap;	136	pgsize_bitmap = tdev->iommu.domain->pgsize_bitmap;
137	if (pgsize_bitmap & PAGE_SIZE) {	137	if (pgsize_bitmap & PAGE_SIZE) {
138	tdev->iommu.pgshift = PAGE_SHIFT;	138	tdev->iommu.pgshift = PAGE_SHIFT;
139	} else {	139	} else {

drivers/iommu/amd/amd_iommu.h+1−3

 extern irqreturn_t amd_iommu_int_handler(int irq, void *data);
 extern void amd_iommu_apply_erratum_63(u16 devid);
 extern void amd_iommu_restart_event_logging(struct amd_iommu *iommu);
-extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu);
 extern int amd_iommu_init_devices(void);
 extern void amd_iommu_uninit_devices(void);
 extern void amd_iommu_init_notifier(void);
 extern bool translation_pre_enabled(struct amd_iommu *iommu);
-extern bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
+extern bool amd_iommu_is_attach_deferred(struct device *dev);
-					 struct device *dev);
 extern int __init add_special_device(u8 type, u8 id, u16 *devid,
 				     bool cmd_line);

drivers/iommu/amd/init.c+14−4

@@ -671,7 +671,7 @@ void amd_iommu_restart_event_logging(struct amd_iommu *iommu) · 670 unmodified lines
671	* This function resets the command buffer if the IOMMU stopped fetching	671	* This function resets the command buffer if the IOMMU stopped fetching
672	* commands from it.	672	* commands from it.
673	*/	673	*/
674	void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)	674	static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
675	{	675	{
676	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);	676	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
677		677
@@ -990,6 +990,7 @@ static bool copy_device_table(void) · 312 unmodified lines
990	get_order(dev_table_size));	990	get_order(dev_table_size));
991	if (old_dev_tbl_cpy == NULL) {	991	if (old_dev_tbl_cpy == NULL) {
992	pr_err("Failed to allocate memory for copying old device table!\n");	992	pr_err("Failed to allocate memory for copying old device table!\n");
		993	memunmap(old_devtb);
993	return false;	994	return false;
994	}	995	}
995		996
@@ -1020,6 +1021,7 @@ static bool copy_device_table(void) · 24 unmodified lines
1020	if ((int_ctl != DTE_IRQ_REMAP_INTCTL) \|\|	1021	if ((int_ctl != DTE_IRQ_REMAP_INTCTL) \|\|
1021	(int_tab_len != DTE_INTTABLEN)) {	1022	(int_tab_len != DTE_INTTABLEN)) {
1022	pr_err("Wrong old irq remapping flag: %#x\n", devid);	1023	pr_err("Wrong old irq remapping flag: %#x\n", devid);
		1024	memunmap(old_devtb);
1023	return false;	1025	return false;
1024	}	1026	}
1025		1027
@@ -1953,9 +1955,11 @@ static int __init amd_iommu_init_pci(void) · 927 unmodified lines
1953		1955
1954	for_each_iommu(iommu) {	1956	for_each_iommu(iommu) {
1955	ret = iommu_init_pci(iommu);	1957	ret = iommu_init_pci(iommu);
1956	if (ret)	1958	if (ret) {
1957	break;	1959	pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n",
1958		1960	iommu->index, ret);
		1961	goto out;
		1962	}
1959	/* Need to setup range after PCI init */	1963	/* Need to setup range after PCI init */
1960	iommu_set_cwwb_range(iommu);	1964	iommu_set_cwwb_range(iommu);
1961	}	1965	}
@@ -1971,6 +1975,11 @@ static int __init amd_iommu_init_pci(void) · 9 unmodified lines
1971	* active.	1975	* active.
1972	*/	1976	*/
1973	ret = amd_iommu_init_api();	1977	ret = amd_iommu_init_api();
		1978	if (ret) {
		1979	pr_err("IOMMU: Failed to initialize IOMMU-API interface (error=%d)!\n",
		1980	ret);
		1981	goto out;
		1982	}
1974		1983
1975	init_device_table_dma();	1984	init_device_table_dma();
1976		1985
@@ -1980,6 +1989,7 @@ static int __init amd_iommu_init_pci(void) · 3 unmodified lines
1980	if (!ret)	1989	if (!ret)
1981	print_iommu_info();	1990	print_iommu_info();
1982		1991
		1992	out:
1983	return ret;	1993	return ret;
1984	}	1994	}
1985		1995

drivers/iommu/amd/iommu.c+12−11

 	list_add_tail(&region->list, head);
 }
-bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
+bool amd_iommu_is_attach_deferred(struct device *dev)
-				  struct device *dev)
 {
 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
 const struct iommu_ops amd_iommu_ops = {
 	.capable = amd_iommu_capable,
 	.domain_alloc = amd_iommu_domain_alloc,
-	.domain_free  = amd_iommu_domain_free,
-	.attach_dev = amd_iommu_attach_device,
-	.detach_dev = amd_iommu_detach_device,
-	.map = amd_iommu_map,
-	.iotlb_sync_map	= amd_iommu_iotlb_sync_map,
-	.unmap = amd_iommu_unmap,
-	.iova_to_phys = amd_iommu_iova_to_phys,
 	.probe_device = amd_iommu_probe_device,
 	.release_device = amd_iommu_release_device,
 	.probe_finalize = amd_iommu_probe_finalize,
 	.put_resv_regions = generic_iommu_put_resv_regions,
 	.is_attach_deferred = amd_iommu_is_attach_deferred,
 	.pgsize_bitmap	= AMD_IOMMU_PGSIZES,
-	.flush_iotlb_all = amd_iommu_flush_iotlb_all,
-	.iotlb_sync = amd_iommu_iotlb_sync,
 	.def_domain_type = amd_iommu_def_domain_type,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= amd_iommu_attach_device,
+		.detach_dev	= amd_iommu_detach_device,
+		.map		= amd_iommu_map,
+		.unmap		= amd_iommu_unmap,
+		.iotlb_sync_map	= amd_iommu_iotlb_sync_map,
+		.iova_to_phys	= amd_iommu_iova_to_phys,
+		.flush_iotlb_all = amd_iommu_flush_iotlb_all,
+		.iotlb_sync	= amd_iommu_iotlb_sync,
+		.free		= amd_iommu_domain_free,
+	}
 };
 /*****************************************************************************

drivers/iommu/amd/iommu_v2.c+18−19

 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Joerg Roedel <jroedel@suse.de>");
-#define MAX_DEVICES		0x10000
 #define PRI_QUEUE_SIZE		512
 struct pri_queue {
 	struct pasid_state *state;
 	struct mm_struct *mm;
 	u64 address;
-	u16 devid;
 	u32 pasid;
 	u16 tag;
 	u16 finish;
 {
 	struct iommu_group *group;
+	/* Get rid of any remaining pasid states */
+	free_pasid_states(dev_state);
+	/*
+	 * Wait until the last reference is dropped before freeing
+	 * the device state.
+	 */
+	wait_event(dev_state->wq, !atomic_read(&dev_state->count));
 	/*
 	 * First detach device from domain - No more PRI requests will arrive
 	 * from that device after it is unbound from the IOMMUv2 domain.
 	ret = NOTIFY_DONE;
 	/* In kdump kernel pci dev is not initialized yet -> send INVALID */
-	if (amd_iommu_is_attach_deferred(NULL, &pdev->dev)) {
+	if (amd_iommu_is_attach_deferred(&pdev->dev)) {
 		amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
 				       PPR_INVALID, tag);
 		goto out;
 	spin_unlock_irqrestore(&state_lock, flags);
-	/* Get rid of any remaining pasid states */
-	free_pasid_states(dev_state);
 	put_device_state(dev_state);
-	/*
-	 * Wait until the last reference is dropped before freeing
-	 * the device state.
-	 */
-	wait_event(dev_state->wq, !atomic_read(&dev_state->count));
 	free_device_state(dev_state);
 }
 EXPORT_SYMBOL(amd_iommu_free_device);
 static void __exit amd_iommu_v2_exit(void)
 {
-	struct device_state *dev_state;
+	struct device_state *dev_state, *next;
-	int i;
+	unsigned long flags;
 	if (!amd_iommu_v2_supported())
 		return;
 	 * The loop below might call flush_workqueue(), so call
 	 * destroy_workqueue() after it
 	 */
-	for (i = 0; i < MAX_DEVICES; ++i) {
+	spin_lock_irqsave(&state_lock, flags);
-		dev_state = get_device_state(i);
-		if (dev_state == NULL)
-			continue;
+	list_for_each_entry_safe(dev_state, next, &state_list, list) {
 		WARN_ON_ONCE(1);
 		put_device_state(dev_state);
-		amd_iommu_free_device(dev_state->pdev);
+		list_del(&dev_state->list);
+		free_device_state(dev_state);
 	}
+	spin_unlock_irqrestore(&state_lock, flags);
 	destroy_workqueue(iommu_wq);
 }

drivers/iommu/apple-dart.c+11−9

 static const struct iommu_ops apple_dart_iommu_ops = {
 	.domain_alloc = apple_dart_domain_alloc,
-	.domain_free = apple_dart_domain_free,
-	.attach_dev = apple_dart_attach_dev,
-	.detach_dev = apple_dart_detach_dev,
-	.map_pages = apple_dart_map_pages,
-	.unmap_pages = apple_dart_unmap_pages,
-	.flush_iotlb_all = apple_dart_flush_iotlb_all,
-	.iotlb_sync = apple_dart_iotlb_sync,
-	.iotlb_sync_map = apple_dart_iotlb_sync_map,
-	.iova_to_phys = apple_dart_iova_to_phys,
 	.probe_device = apple_dart_probe_device,
 	.release_device = apple_dart_release_device,
 	.device_group = apple_dart_device_group,
 	.get_resv_regions = apple_dart_get_resv_regions,
 	.put_resv_regions = generic_iommu_put_resv_regions,
 	.pgsize_bitmap = -1UL, /* Restricted during dart probe */
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= apple_dart_attach_dev,
+		.detach_dev	= apple_dart_detach_dev,
+		.map_pages	= apple_dart_map_pages,
+		.unmap_pages	= apple_dart_unmap_pages,
+		.flush_iotlb_all = apple_dart_flush_iotlb_all,
+		.iotlb_sync	= apple_dart_iotlb_sync,
+		.iotlb_sync_map	= apple_dart_iotlb_sync_map,
+		.iova_to_phys	= apple_dart_iova_to_phys,
+		.free		= apple_dart_domain_free,
+	}
 };
 static irqreturn_t apple_dart_irq(int irq, void *dev)

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c+18−27

 				dev_info(smmu->dev, "\t0x%016llx\n",
 					 (unsigned long long)evt[i]);
+			cond_resched();
 		}
 		/*
 static struct iommu_ops arm_smmu_ops = {
 	.capable		= arm_smmu_capable,
 	.domain_alloc		= arm_smmu_domain_alloc,
-	.domain_free		= arm_smmu_domain_free,
-	.attach_dev		= arm_smmu_attach_dev,
-	.map_pages		= arm_smmu_map_pages,
-	.unmap_pages		= arm_smmu_unmap_pages,
-	.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
-	.iotlb_sync		= arm_smmu_iotlb_sync,
-	.iova_to_phys		= arm_smmu_iova_to_phys,
 	.probe_device		= arm_smmu_probe_device,
 	.release_device		= arm_smmu_release_device,
 	.device_group		= arm_smmu_device_group,
-	.enable_nesting		= arm_smmu_enable_nesting,
 	.of_xlate		= arm_smmu_of_xlate,
 	.get_resv_regions	= arm_smmu_get_resv_regions,
 	.put_resv_regions	= generic_iommu_put_resv_regions,
 	.page_response		= arm_smmu_page_response,
 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
 	.owner			= THIS_MODULE,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev		= arm_smmu_attach_dev,
+		.map_pages		= arm_smmu_map_pages,
+		.unmap_pages		= arm_smmu_unmap_pages,
+		.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
+		.iotlb_sync		= arm_smmu_iotlb_sync,
+		.iova_to_phys		= arm_smmu_iova_to_phys,
+		.enable_nesting		= arm_smmu_enable_nesting,
+		.free			= arm_smmu_domain_free,
+	}
 };
 /* Probing and initialisation functions */
 	return 0;
 }
-static void arm_smmu_cmdq_free_bitmap(void *data)
-{
-	unsigned long *bitmap = data;
-	bitmap_free(bitmap);
-}
 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
 {
-	int ret = 0;
 	struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
 	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
-	atomic_long_t *bitmap;
 	atomic_set(&cmdq->owner_prod, 0);
 	atomic_set(&cmdq->lock, 0);
-	bitmap = (atomic_long_t *)bitmap_zalloc(nents, GFP_KERNEL);
+	cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
-	if (!bitmap) {
+							      GFP_KERNEL);
-		dev_err(smmu->dev, "failed to allocate cmdq bitmap\n");
+	if (!cmdq->valid_map)
-		ret = -ENOMEM;
+		return -ENOMEM;
-	} else {
-		cmdq->valid_map = bitmap;
-		devm_add_action(smmu->dev, arm_smmu_cmdq_free_bitmap, bitmap);
-	}
-	return ret;
+	return 0;
 }
 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
 {
 	unsigned int i;
 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
-	size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
 	void *strtab = smmu->strtab_cfg.strtab;
-	cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
+	cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents,
+				    sizeof(*cfg->l1_desc), GFP_KERNEL);
 	if (!cfg->l1_desc)
 		return -ENOMEM;

drivers/iommu/arm/arm-smmu/arm-smmu.c+55−58

@@ -807,7 +807,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, · 806 unmodified lines
807	* Request context fault interrupt. Do this last to avoid the	807	* Request context fault interrupt. Do this last to avoid the
808	* handler seeing a half-initialised domain state.	808	* handler seeing a half-initialised domain state.
809	*/	809	*/
810	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];	810	irq = smmu->irqs[cfg->irptndx];
811		811
812	if (smmu->impl && smmu->impl->context_fault)	812	if (smmu->impl && smmu->impl->context_fault)
813	context_fault = smmu->impl->context_fault;	813	context_fault = smmu->impl->context_fault;
@@ -858,7 +858,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) · 44 unmodified lines
858	arm_smmu_write_context_bank(smmu, cfg->cbndx);	858	arm_smmu_write_context_bank(smmu, cfg->cbndx);
859		859
860	if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {	860	if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
861	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];	861	irq = smmu->irqs[cfg->irptndx];
862	devm_free_irq(smmu->dev, irq, domain);	862	devm_free_irq(smmu->dev, irq, domain);
863	}	863	}
864		864
@@ -1583,25 +1583,27 @@ static int arm_smmu_def_domain_type(struct device *dev) · 718 unmodified lines
1583	static struct iommu_ops arm_smmu_ops = {	1583	static struct iommu_ops arm_smmu_ops = {
1584	.capable = arm_smmu_capable,	1584	.capable = arm_smmu_capable,
1585	.domain_alloc = arm_smmu_domain_alloc,	1585	.domain_alloc = arm_smmu_domain_alloc,
1586	.domain_free = arm_smmu_domain_free,
1587	.attach_dev = arm_smmu_attach_dev,
1588	.map_pages = arm_smmu_map_pages,
1589	.unmap_pages = arm_smmu_unmap_pages,
1590	.flush_iotlb_all = arm_smmu_flush_iotlb_all,
1591	.iotlb_sync = arm_smmu_iotlb_sync,
1592	.iova_to_phys = arm_smmu_iova_to_phys,
1593	.probe_device = arm_smmu_probe_device,	1586	.probe_device = arm_smmu_probe_device,
1594	.release_device = arm_smmu_release_device,	1587	.release_device = arm_smmu_release_device,
1595	.probe_finalize = arm_smmu_probe_finalize,	1588	.probe_finalize = arm_smmu_probe_finalize,
1596	.device_group = arm_smmu_device_group,	1589	.device_group = arm_smmu_device_group,
1597	.enable_nesting = arm_smmu_enable_nesting,
1598	.set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1599	.of_xlate = arm_smmu_of_xlate,	1590	.of_xlate = arm_smmu_of_xlate,
1600	.get_resv_regions = arm_smmu_get_resv_regions,	1591	.get_resv_regions = arm_smmu_get_resv_regions,
1601	.put_resv_regions = generic_iommu_put_resv_regions,	1592	.put_resv_regions = generic_iommu_put_resv_regions,
1602	.def_domain_type = arm_smmu_def_domain_type,	1593	.def_domain_type = arm_smmu_def_domain_type,
1603	.pgsize_bitmap = -1UL, /* Restricted during device attach */	1594	.pgsize_bitmap = -1UL, /* Restricted during device attach */
1604	.owner = THIS_MODULE,	1595	.owner = THIS_MODULE,
		1596	.default_domain_ops = &(const struct iommu_domain_ops) {
		1597	.attach_dev = arm_smmu_attach_dev,
		1598	.map_pages = arm_smmu_map_pages,
		1599	.unmap_pages = arm_smmu_unmap_pages,
		1600	.flush_iotlb_all = arm_smmu_flush_iotlb_all,
		1601	.iotlb_sync = arm_smmu_iotlb_sync,
		1602	.iova_to_phys = arm_smmu_iova_to_phys,
		1603	.enable_nesting = arm_smmu_enable_nesting,
		1604	.set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
		1605	.free = arm_smmu_domain_free,
		1606	}
1605	};	1607	};
1606		1608
1607	static void arm_smmu_device_reset(struct arm_smmu_device *smmu)	1609	static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
@@ -1951,8 +1953,8 @@ static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) · 343 unmodified lines
1951	return ret;	1953	return ret;
1952	}	1954	}
1953		1955
1954	static int arm_smmu_device_acpi_probe(struct platform_device *pdev,	1956	static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1955	struct arm_smmu_device *smmu)	1957	u32 global_irqs, u32 pmu_irqs)
1956	{	1958	{
1957	struct device *dev = smmu->dev;	1959	struct device *dev = smmu->dev;
1958	struct acpi_iort_node *node =	1960	struct acpi_iort_node *node =
@@ -1968,7 +1970,8 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev, · 9 unmodified lines
1968	return ret;	1970	return ret;
1969		1971
1970	/* Ignore the configuration access interrupt */	1972	/* Ignore the configuration access interrupt */
1971	smmu->num_global_irqs = 1;	1973	*global_irqs = 1;
		1974	*pmu_irqs = 0;
1972		1975
1973	if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)	1976	if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1974	smmu->features \|= ARM_SMMU_FEAT_COHERENT_WALK;	1977	smmu->features \|= ARM_SMMU_FEAT_COHERENT_WALK;
@@ -1976,25 +1979,24 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev, · 1 unmodified line
1976	return 0;	1979	return 0;
1977	}	1980	}
1978	#else	1981	#else
1979	static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,	1982	static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1980	struct arm_smmu_device *smmu)	1983	u32 global_irqs, u32 pmu_irqs)
1981	{	1984	{
1982	return -ENODEV;	1985	return -ENODEV;
1983	}	1986	}
1984	#endif	1987	#endif
1985		1988
1986	static int arm_smmu_device_dt_probe(struct platform_device *pdev,	1989	static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
1987	struct arm_smmu_device *smmu)	1990	u32 global_irqs, u32 pmu_irqs)
1988	{	1991	{
1989	const struct arm_smmu_match_data *data;	1992	const struct arm_smmu_match_data *data;
1990	struct device *dev = &pdev->dev;	1993	struct device *dev = smmu->dev;
1991	bool legacy_binding;	1994	bool legacy_binding;
1992		1995
1993	if (of_property_read_u32(dev->of_node, "#global-interrupts",	1996	if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
1994	&smmu->num_global_irqs)) {	1997	return dev_err_probe(dev, -ENODEV,
1995	dev_err(dev, "missing #global-interrupts property\n");	1998	"missing #global-interrupts property\n");
1996	return -ENODEV;	1999	*pmu_irqs = 0;
1997	}
1998		2000
1999	data = of_device_get_match_data(dev);	2001	data = of_device_get_match_data(dev);
2000	smmu->version = data->version;	2002	smmu->version = data->version;
@@ -2073,6 +2075,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 72 unmodified lines
2073	struct arm_smmu_device *smmu;	2075	struct arm_smmu_device *smmu;
2074	struct device *dev = &pdev->dev;	2076	struct device *dev = &pdev->dev;
2075	int num_irqs, i, err;	2077	int num_irqs, i, err;
		2078	u32 global_irqs, pmu_irqs;
2076	irqreturn_t (global_fault)(int irq, void dev);	2079	irqreturn_t (global_fault)(int irq, void dev);
2077		2080
2078	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);	2081	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
@@ -2083,10 +2086,9 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 4 unmodified lines
2083	smmu->dev = dev;	2086	smmu->dev = dev;
2084		2087
2085	if (dev->of_node)	2088	if (dev->of_node)
2086	err = arm_smmu_device_dt_probe(pdev, smmu);	2089	err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2087	else	2090	else
2088	err = arm_smmu_device_acpi_probe(pdev, smmu);	2091	err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2089
2090	if (err)	2092	if (err)
2091	return err;	2093	return err;
2092		2094
@@ -2105,31 +2107,25 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 12 unmodified lines
2105	if (IS_ERR(smmu))	2107	if (IS_ERR(smmu))
2106	return PTR_ERR(smmu);	2108	return PTR_ERR(smmu);
2107		2109
2108	num_irqs = 0;	2110	num_irqs = platform_irq_count(pdev);
2109	while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2110	num_irqs++;
2111	if (num_irqs > smmu->num_global_irqs)
2112	smmu->num_context_irqs++;
2113	}
2114		2111
2115	if (!smmu->num_context_irqs) {	2112	smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2116	dev_err(dev, "found %d interrupts but expected at least %d\n",	2113	if (smmu->num_context_irqs <= 0)
2117	num_irqs, smmu->num_global_irqs + 1);	2114	return dev_err_probe(dev, -ENODEV,
2118	return -ENODEV;	2115	"found %d interrupts but expected at least %d\n",
2119	}	2116	num_irqs, global_irqs + pmu_irqs + 1);
2120		2117
2121	smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),	2118	smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2122	GFP_KERNEL);	2119	sizeof(*smmu->irqs), GFP_KERNEL);
2123	if (!smmu->irqs) {	2120	if (!smmu->irqs)
2124	dev_err(dev, "failed to allocate %d irqs\n", num_irqs);	2121	return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2125	return -ENOMEM;	2122	smmu->num_context_irqs);
2126	}
2127		2123
2128	for (i = 0; i < num_irqs; ++i) {	2124	for (i = 0; i < smmu->num_context_irqs; i++) {
2129	int irq = platform_get_irq(pdev, i);	2125	int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2130		2126
2131	if (irq < 0)	2127	if (irq < 0)
2132	return -ENODEV;	2128	return irq;
2133	smmu->irqs[i] = irq;	2129	smmu->irqs[i] = irq;
2134	}	2130	}
2135		2131
@@ -2165,17 +2161,18 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 29 unmodified lines
2165	else	2161	else
2166	global_fault = arm_smmu_global_fault;	2162	global_fault = arm_smmu_global_fault;
2167		2163
2168	for (i = 0; i < smmu->num_global_irqs; ++i) {	2164	for (i = 0; i < global_irqs; i++) {
2169	err = devm_request_irq(smmu->dev, smmu->irqs[i],	2165	int irq = platform_get_irq(pdev, i);
2170	global_fault,	2166
2171	IRQF_SHARED,	2167	if (irq < 0)
2172	"arm-smmu global fault",	2168	return irq;
2173	smmu);	2169
2174	if (err) {	2170	err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2175	dev_err(dev, "failed to request global IRQ %d (%u)\n",	2171	"arm-smmu global fault", smmu);
2176	i, smmu->irqs[i]);	2172	if (err)
2177	return err;	2173	return dev_err_probe(dev, err,
2178	}	2174	"failed to request global IRQ %d (%u)\n",
		2175	i, irq);
2179	}	2176	}
2180		2177
2181	err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,	2178	err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,

drivers/iommu/arm/arm-smmu/arm-smmu.h+2−3

 	unsigned long			pa_size;
 	unsigned long			pgsize_bitmap;
-	u32				num_global_irqs;
+	int				num_context_irqs;
-	u32				num_context_irqs;
+	int				num_clks;
 	unsigned int			*irqs;
 	struct clk_bulk_data		*clks;
-	int				num_clks;
 	spinlock_t			global_sync_lock;

drivers/iommu/arm/arm-smmu/qcom_iommu.c+17−11

 static const struct iommu_ops qcom_iommu_ops = {
 	.capable	= qcom_iommu_capable,
 	.domain_alloc	= qcom_iommu_domain_alloc,
-	.domain_free	= qcom_iommu_domain_free,
-	.attach_dev	= qcom_iommu_attach_dev,
-	.detach_dev	= qcom_iommu_detach_dev,
-	.map		= qcom_iommu_map,
-	.unmap		= qcom_iommu_unmap,
-	.flush_iotlb_all = qcom_iommu_flush_iotlb_all,
-	.iotlb_sync	= qcom_iommu_iotlb_sync,
-	.iova_to_phys	= qcom_iommu_iova_to_phys,
 	.probe_device	= qcom_iommu_probe_device,
 	.release_device	= qcom_iommu_release_device,
 	.device_group	= generic_device_group,
 	.of_xlate	= qcom_iommu_of_xlate,
 	.pgsize_bitmap	= SZ_4K | SZ_64K | SZ_1M | SZ_16M,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= qcom_iommu_attach_dev,
+		.detach_dev	= qcom_iommu_detach_dev,
+		.map		= qcom_iommu_map,
+		.unmap		= qcom_iommu_unmap,
+		.flush_iotlb_all = qcom_iommu_flush_iotlb_all,
+		.iotlb_sync	= qcom_iommu_iotlb_sync,
+		.iova_to_phys	= qcom_iommu_iova_to_phys,
+		.free		= qcom_iommu_domain_free,
+	}
 };
 static int qcom_iommu_sec_ptbl_init(struct device *dev)
 	ret = devm_of_platform_populate(dev);
 	if (ret) {
 		dev_err(dev, "Failed to populate iommu contexts\n");
-		return ret;
+		goto err_pm_disable;
 	}
 	ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
 				     dev_name(dev));
 	if (ret) {
 		dev_err(dev, "Failed to register iommu in sysfs\n");
-		return ret;
+		goto err_pm_disable;
 	}
 	ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);
 	if (ret) {
 		dev_err(dev, "Failed to register iommu\n");
-		return ret;
+		goto err_pm_disable;
 	}
 	bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
 	}
 	return 0;
+err_pm_disable:
+	pm_runtime_disable(dev);
+	return ret;
 }
 static int qcom_iommu_device_remove(struct platform_device *pdev)

drivers/iommu/dma-iommu.c+4−0

 	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 	unsigned long order, base_pfn;
 	struct iova_domain *iovad;
+	int ret;
 	if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
 		return -EINVAL;
 	}
 	init_iova_domain(iovad, 1UL << order, base_pfn);
+	ret = iova_domain_init_rcaches(iovad);
+	if (ret)
+		return ret;
 	/* If the FQ fails we can simply fall back to strict mode */
 	if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain))

drivers/iommu/exynos-iommu.c+8−6

 static const struct iommu_ops exynos_iommu_ops = {
 	.domain_alloc = exynos_iommu_domain_alloc,
-	.domain_free = exynos_iommu_domain_free,
-	.attach_dev = exynos_iommu_attach_device,
-	.detach_dev = exynos_iommu_detach_device,
-	.map = exynos_iommu_map,
-	.unmap = exynos_iommu_unmap,
-	.iova_to_phys = exynos_iommu_iova_to_phys,
 	.device_group = generic_device_group,
 	.probe_device = exynos_iommu_probe_device,
 	.release_device = exynos_iommu_release_device,
 	.pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
 	.of_xlate = exynos_iommu_of_xlate,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= exynos_iommu_attach_device,
+		.detach_dev	= exynos_iommu_detach_device,
+		.map		= exynos_iommu_map,
+		.unmap		= exynos_iommu_unmap,
+		.iova_to_phys	= exynos_iommu_iova_to_phys,
+		.free		= exynos_iommu_domain_free,
+	}
 };
 static int __init exynos_iommu_init(void)

drivers/iommu/fsl_pamu_domain.c+6−4

 static const struct iommu_ops fsl_pamu_ops = {
 	.capable	= fsl_pamu_capable,
 	.domain_alloc	= fsl_pamu_domain_alloc,
-	.domain_free    = fsl_pamu_domain_free,
-	.attach_dev	= fsl_pamu_attach_device,
-	.detach_dev	= fsl_pamu_detach_device,
-	.iova_to_phys	= fsl_pamu_iova_to_phys,
 	.probe_device	= fsl_pamu_probe_device,
 	.release_device	= fsl_pamu_release_device,
 	.device_group   = fsl_pamu_device_group,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= fsl_pamu_attach_device,
+		.detach_dev	= fsl_pamu_detach_device,
+		.iova_to_phys	= fsl_pamu_iova_to_phys,
+		.free		= fsl_pamu_domain_free,
+	}
 };
 int __init pamu_domain_init(void)

drivers/iommu/intel/debugfs.c+3−3

 static int show_device_domain_translation(struct device *dev, void *data)
 {
-	struct dmar_domain *domain = find_domain(dev);
+	struct device_domain_info *info = dev_iommu_priv_get(dev);
+	struct dmar_domain *domain = info->domain;
 	struct seq_file *m = data;
 	u64 path[6] = { 0 };
 	if (!domain)
 		return 0;
-	seq_printf(m, "Device %s with pasid %d @0x%llx\n",
+	seq_printf(m, "Device %s @0x%llx\n", dev_name(dev),
-		   dev_name(dev), domain->default_pasid,
 		   (u64)virt_to_phys(domain->pgd));
 	seq_puts(m, "IOVA_PFN\t\tPML5E\t\t\tPML4E\t\t\tPDPE\t\t\tPDE\t\t\tPTE\n");

drivers/iommu/intel/dmar.c+2−3

 static int alloc_iommu(struct dmar_drhd_unit *drhd);
 static void free_iommu(struct intel_iommu *iommu);
-extern const struct iommu_ops intel_iommu_ops;
 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
 {
 	/*
 				       andd->device_name);
 				continue;
 			}
-			if (acpi_bus_get_device(h, &adev)) {
+			adev = acpi_fetch_acpi_dev(h);
+			if (!adev) {
 				pr_err("Failed to get device for ACPI object %s\n",
 				       andd->device_name);
 				continue;

drivers/iommu/intel/iommu.c+133−866

@@ -13,38 +13,18 @@ · 12 unmodified lines
13	#define pr_fmt(fmt) "DMAR: " fmt	13	#define pr_fmt(fmt) "DMAR: " fmt
14	#define dev_fmt(fmt) pr_fmt(fmt)	14	#define dev_fmt(fmt) pr_fmt(fmt)
15		15
16	#include <linux/init.h>	16	#include <linux/crash_dump.h>
17	#include <linux/bitmap.h>	17	#include <linux/dma-direct.h>
18	#include <linux/debugfs.h>
19	#include <linux/export.h>
20	#include <linux/slab.h>
21	#include <linux/irq.h>
22	#include <linux/interrupt.h>
23	#include <linux/spinlock.h>
24	#include <linux/pci.h>
25	#include <linux/dmar.h>
26	#include <linux/dma-map-ops.h>
27	#include <linux/mempool.h>
28	#include <linux/memory.h>
29	#include <linux/cpu.h>
30	#include <linux/timer.h>
31	#include <linux/io.h>
32	#include <linux/iova.h>
33	#include <linux/iommu.h>
34	#include <linux/dma-iommu.h>	18	#include <linux/dma-iommu.h>
		19	#include <linux/dmi.h>
35	#include <linux/intel-iommu.h>	20	#include <linux/intel-iommu.h>
36	#include <linux/intel-svm.h>	21	#include <linux/intel-svm.h>
		22	#include <linux/memory.h>
		23	#include <linux/pci.h>
		24	#include <linux/pci-ats.h>
		25	#include <linux/spinlock.h>
37	#include <linux/syscore_ops.h>	26	#include <linux/syscore_ops.h>
38	#include <linux/tboot.h>	27	#include <linux/tboot.h>
39	#include <linux/dmi.h>
40	#include <linux/pci-ats.h>
41	#include <linux/memblock.h>
42	#include <linux/dma-direct.h>
43	#include <linux/crash_dump.h>
44	#include <linux/numa.h>
45	#include <asm/irq_remapping.h>
46	#include <asm/cacheflush.h>
47	#include <asm/iommu.h>
48		28
49	#include "../irq_remapping.h"	29	#include "../irq_remapping.h"
50	#include "../iommu-sva-lib.h"	30	#include "../iommu-sva-lib.h"
@@ -316,14 +296,9 @@ static LIST_HEAD(dmar_satc_units); · 265 unmodified lines
316	/* bitmap for indexing intel_iommus */	296	/* bitmap for indexing intel_iommus */
317	static int g_num_of_iommus;	297	static int g_num_of_iommus;
318		298
319	static void domain_exit(struct dmar_domain *domain);
320	static void domain_remove_dev_info(struct dmar_domain *domain);	299	static void domain_remove_dev_info(struct dmar_domain *domain);
321	static void dmar_remove_one_dev_info(struct device *dev);	300	static void dmar_remove_one_dev_info(struct device *dev);
322	static void __dmar_remove_one_dev_info(struct device_domain_info *info);	301	static void __dmar_remove_one_dev_info(struct device_domain_info *info);
323	static int intel_iommu_attach_device(struct iommu_domain *domain,
324	struct device *dev);
325	static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
326	dma_addr_t iova);
327		302
328	int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);	303	int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
329	int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);	304	int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
@@ -342,21 +317,6 @@ static int iommu_skip_te_disable; · 12 unmodified lines
342	int intel_iommu_gfx_mapped;	317	int intel_iommu_gfx_mapped;
343	EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);	318	EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
344		319
345	#define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
346	struct device_domain_info get_domain_info(struct device dev)
347	{
348	struct device_domain_info *info;
349
350	if (!dev)
351	return NULL;
352
353	info = dev_iommu_priv_get(dev);
354	if (unlikely(info == DEFER_DEVICE_DOMAIN_INFO))
355	return NULL;
356
357	return info;
358	}
359
360	DEFINE_SPINLOCK(device_domain_lock);	320	DEFINE_SPINLOCK(device_domain_lock);
361	static LIST_HEAD(device_domain_list);	321	static LIST_HEAD(device_domain_list);
362		322
@@ -452,39 +412,6 @@ static int __init intel_iommu_setup(char *str) · 89 unmodified lines
452	}	412	}
453	__setup("intel_iommu=", intel_iommu_setup);	413	__setup("intel_iommu=", intel_iommu_setup);
454		414
455	static struct kmem_cache *iommu_domain_cache;
456	static struct kmem_cache *iommu_devinfo_cache;
457
458	static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
459	{
460	struct dmar_domain **domains;
461	int idx = did >> 8;
462
463	domains = iommu->domains[idx];
464	if (!domains)
465	return NULL;
466
467	return domains[did & 0xff];
468	}
469
470	static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
471	struct dmar_domain *domain)
472	{
473	struct dmar_domain **domains;
474	int idx = did >> 8;
475
476	if (!iommu->domains[idx]) {
477	size_t size = 256 * sizeof(struct dmar_domain *);
478	iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
479	}
480
481	domains = iommu->domains[idx];
482	if (WARN_ON(!domains))
483	return;
484	else
485	domains[did & 0xff] = domain;
486	}
487
488	void *alloc_pgtable_page(int node)	415	void *alloc_pgtable_page(int node)
489	{	416	{
490	struct page *page;	417	struct page *page;
@@ -501,26 +428,6 @@ void free_pgtable_page(void *vaddr) · 10 unmodified lines
501	free_page((unsigned long)vaddr);	428	free_page((unsigned long)vaddr);
502	}	429	}
503		430
504	static inline void *alloc_domain_mem(void)
505	{
506	return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
507	}
508
509	static void free_domain_mem(void *vaddr)
510	{
511	kmem_cache_free(iommu_domain_cache, vaddr);
512	}
513
514	static inline void * alloc_devinfo_mem(void)
515	{
516	return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
517	}
518
519	static inline void free_devinfo_mem(void *vaddr)
520	{
521	kmem_cache_free(iommu_devinfo_cache, vaddr);
522	}
523
524	static inline int domain_type_is_si(struct dmar_domain *domain)	431	static inline int domain_type_is_si(struct dmar_domain *domain)
525	{	432	{
526	return domain->domain.type == IOMMU_DOMAIN_IDENTITY;	433	return domain->domain.type == IOMMU_DOMAIN_IDENTITY;
@@ -794,11 +701,6 @@ struct context_entry iommu_context_addr(struct intel_iommu iommu, u8 bus, · 267 unmodified lines
794	return &context[devfn];	701	return &context[devfn];
795	}	702	}
796		703
797	static bool attach_deferred(struct device *dev)
798	{
799	return dev_iommu_priv_get(dev) == DEFER_DEVICE_DOMAIN_INFO;
800	}
801
802	/**	704	/**
803	* is_downstream_to_pci_bridge - test if a device belongs to the PCI	705	* is_downstream_to_pci_bridge - test if a device belongs to the PCI
804	* sub-hierarchy of a candidate PCI-PCI bridge	706	* sub-hierarchy of a candidate PCI-PCI bridge
@@ -925,7 +827,7 @@ struct intel_iommu device_to_iommu(struct device dev, u8 bus, u8 devfn) · 120 unmodified lines
925	}	827	}
926		828
927	if (pdev && drhd->include_all) {	829	if (pdev && drhd->include_all) {
928	got_pdev:	830	got_pdev:
929	if (bus && devfn) {	831	if (bus && devfn) {
930	*bus = pdev->bus->number;	832	*bus = pdev->bus->number;
931	*devfn = pdev->devfn;	833	*devfn = pdev->devfn;
@@ -934,7 +836,7 @@ struct intel_iommu device_to_iommu(struct device dev, u8 bus, u8 devfn) · 2 unmodified lines
934	}	836	}
935	}	837	}
936	iommu = NULL;	838	iommu = NULL;
937	out:	839	out:
938	if (iommu_is_dummy(iommu, dev))	840	if (iommu_is_dummy(iommu, dev))
939	iommu = NULL;	841	iommu = NULL;
940		842
@@ -1573,18 +1475,6 @@ static void domain_update_iotlb(struct dmar_domain *domain) · 632 unmodified lines
1573	break;	1475	break;
1574	}	1476	}
1575		1477
1576	if (!has_iotlb_device) {
1577	struct subdev_domain_info *sinfo;
1578
1579	list_for_each_entry(sinfo, &domain->subdevices, link_domain) {
1580	info = get_domain_info(sinfo->pdev);
1581	if (info && info->ats_enabled) {
1582	has_iotlb_device = true;
1583	break;
1584	}
1585	}
1586	}
1587
1588	domain->has_iotlb_device = has_iotlb_device;	1478	domain->has_iotlb_device = has_iotlb_device;
1589	}	1479	}
1590		1480
@@ -1682,7 +1572,6 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, · 91 unmodified lines
1682	{	1572	{
1683	unsigned long flags;	1573	unsigned long flags;
1684	struct device_domain_info *info;	1574	struct device_domain_info *info;
1685	struct subdev_domain_info *sinfo;
1686		1575
1687	if (!domain->has_iotlb_device)	1576	if (!domain->has_iotlb_device)
1688	return;	1577	return;
@@ -1691,27 +1580,9 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, · 2 unmodified lines
1691	list_for_each_entry(info, &domain->devices, link)	1580	list_for_each_entry(info, &domain->devices, link)
1692	__iommu_flush_dev_iotlb(info, addr, mask);	1581	__iommu_flush_dev_iotlb(info, addr, mask);
1693		1582
1694	list_for_each_entry(sinfo, &domain->subdevices, link_domain) {
1695	info = get_domain_info(sinfo->pdev);
1696	__iommu_flush_dev_iotlb(info, addr, mask);
1697	}
1698	spin_unlock_irqrestore(&device_domain_lock, flags);	1583	spin_unlock_irqrestore(&device_domain_lock, flags);
1699	}	1584	}
1700		1585
1701	static void domain_flush_piotlb(struct intel_iommu *iommu,
1702	struct dmar_domain *domain,
1703	u64 addr, unsigned long npages, bool ih)
1704	{
1705	u16 did = domain->iommu_did[iommu->seq_id];
1706
1707	if (domain->default_pasid)
1708	qi_flush_piotlb(iommu, did, domain->default_pasid,
1709	addr, npages, ih);
1710
1711	if (!list_empty(&domain->devices))
1712	qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, npages, ih);
1713	}
1714
1715	static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,	1586	static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1716	struct dmar_domain *domain,	1587	struct dmar_domain *domain,
1717	unsigned long pfn, unsigned int pages,	1588	unsigned long pfn, unsigned int pages,
@@ -1727,7 +1598,7 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, · 9 unmodified lines
1727	ih = 1 << 6;	1598	ih = 1 << 6;
1728		1599
1729	if (domain_use_first_level(domain)) {	1600	if (domain_use_first_level(domain)) {
1730	domain_flush_piotlb(iommu, domain, addr, pages, ih);	1601	qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, pages, ih);
1731	} else {	1602	} else {
1732	/*	1603	/*
1733	* Fallback to domain selective flush if no PSI support or	1604	* Fallback to domain selective flush if no PSI support or
@@ -1776,14 +1647,13 @@ static void intel_flush_iotlb_all(struct iommu_domain *domain) · 42 unmodified lines
1776	u16 did = dmar_domain->iommu_did[iommu->seq_id];	1647	u16 did = dmar_domain->iommu_did[iommu->seq_id];
1777		1648
1778	if (domain_use_first_level(dmar_domain))	1649	if (domain_use_first_level(dmar_domain))
1779	domain_flush_piotlb(iommu, dmar_domain, 0, -1, 0);	1650	qi_flush_piotlb(iommu, did, PASID_RID2PASID, 0, -1, 0);
1780	else	1651	else
1781	iommu->flush.flush_iotlb(iommu, did, 0, 0,	1652	iommu->flush.flush_iotlb(iommu, did, 0, 0,
1782	DMA_TLB_DSI_FLUSH);	1653	DMA_TLB_DSI_FLUSH);
1783		1654
1784	if (!cap_caching_mode(iommu->cap))	1655	if (!cap_caching_mode(iommu->cap))
1785	iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),	1656	iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH);
1786	0, MAX_AGAW_PFN_WIDTH);
1787	}	1657	}
1788	}	1658	}
1789		1659
@@ -1846,7 +1716,6 @@ static void iommu_disable_translation(struct intel_iommu *iommu) · 56 unmodified lines
1846	static int iommu_init_domains(struct intel_iommu *iommu)	1716	static int iommu_init_domains(struct intel_iommu *iommu)
1847	{	1717	{
1848	u32 ndomains;	1718	u32 ndomains;
1849	size_t size;
1850		1719
1851	ndomains = cap_ndoms(iommu->cap);	1720	ndomains = cap_ndoms(iommu->cap);
1852	pr_debug("%s: Number of Domains supported <%d>\n",	1721	pr_debug("%s: Number of Domains supported <%d>\n",
@@ -1858,24 +1727,6 @@ static int iommu_init_domains(struct intel_iommu *iommu) · 5 unmodified lines
1858	if (!iommu->domain_ids)	1727	if (!iommu->domain_ids)
1859	return -ENOMEM;	1728	return -ENOMEM;
1860		1729
1861	size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1862	iommu->domains = kzalloc(size, GFP_KERNEL);
1863
1864	if (iommu->domains) {
1865	size = 256 * sizeof(struct dmar_domain *);
1866	iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1867	}
1868
1869	if (!iommu->domains \|\| !iommu->domains[0]) {
1870	pr_err("%s: Allocating domain array failed\n",
1871	iommu->name);
1872	bitmap_free(iommu->domain_ids);
1873	kfree(iommu->domains);
1874	iommu->domain_ids = NULL;
1875	iommu->domains = NULL;
1876	return -ENOMEM;
1877	}
1878
1879	/*	1730	/*
1880	* If Caching mode is set, then invalid translations are tagged	1731	* If Caching mode is set, then invalid translations are tagged
1881	* with domain-id 0, hence we need to pre-allocate it. We also	1732	* with domain-id 0, hence we need to pre-allocate it. We also
@@ -1902,7 +1753,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) · 20 unmodified lines
1902	struct device_domain_info info, tmp;	1753	struct device_domain_info info, tmp;
1903	unsigned long flags;	1754	unsigned long flags;
1904		1755
1905	if (!iommu->domains \|\| !iommu->domain_ids)	1756	if (!iommu->domain_ids)
1906	return;	1757	return;
1907		1758
1908	spin_lock_irqsave(&device_domain_lock, flags);	1759	spin_lock_irqsave(&device_domain_lock, flags);
@@ -1923,15 +1774,8 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) · 14 unmodified lines
1923		1774
1924	static void free_dmar_iommu(struct intel_iommu *iommu)	1775	static void free_dmar_iommu(struct intel_iommu *iommu)
1925	{	1776	{
1926	if ((iommu->domains) && (iommu->domain_ids)) {	1777	if (iommu->domain_ids) {
1927	int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1928	int i;
1929
1930	for (i = 0; i < elems; i++)
1931	kfree(iommu->domains[i]);
1932	kfree(iommu->domains);
1933	bitmap_free(iommu->domain_ids);	1778	bitmap_free(iommu->domain_ids);
1934	iommu->domains = NULL;
1935	iommu->domain_ids = NULL;	1779	iommu->domain_ids = NULL;
1936	}	1780	}
1937		1781
@@ -1973,17 +1817,15 @@ static struct dmar_domain *alloc_domain(unsigned int type) · 35 unmodified lines
1973	{	1817	{
1974	struct dmar_domain *domain;	1818	struct dmar_domain *domain;
1975		1819
1976	domain = alloc_domain_mem();	1820	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
1977	if (!domain)	1821	if (!domain)
1978	return NULL;	1822	return NULL;
1979		1823
1980	memset(domain, 0, sizeof(*domain));
1981	domain->nid = NUMA_NO_NODE;	1824	domain->nid = NUMA_NO_NODE;
1982	if (first_level_by_default(type))	1825	if (first_level_by_default(type))
1983	domain->flags \|= DOMAIN_FLAG_USE_FIRST_LEVEL;	1826	domain->flags \|= DOMAIN_FLAG_USE_FIRST_LEVEL;
1984	domain->has_iotlb_device = false;	1827	domain->has_iotlb_device = false;
1985	INIT_LIST_HEAD(&domain->devices);	1828	INIT_LIST_HEAD(&domain->devices);
1986	INIT_LIST_HEAD(&domain->subdevices);
1987		1829
1988	return domain;	1830	return domain;
1989	}	1831	}
@@ -2010,11 +1852,8 @@ static int domain_attach_iommu(struct dmar_domain *domain, · 20 unmodified lines
2010	}	1852	}
2011		1853
2012	set_bit(num, iommu->domain_ids);	1854	set_bit(num, iommu->domain_ids);
2013	set_iommu_domain(iommu, num, domain);
2014
2015	domain->iommu_did[iommu->seq_id] = num;	1855	domain->iommu_did[iommu->seq_id] = num;
2016	domain->nid = iommu->node;	1856	domain->nid = iommu->node;
2017
2018	domain_update_iommu_cap(domain);	1857	domain_update_iommu_cap(domain);
2019	}	1858	}
2020		1859
@@ -2033,8 +1872,6 @@ static void domain_detach_iommu(struct dmar_domain *domain, · 12 unmodified lines
2033	if (domain->iommu_refcnt[iommu->seq_id] == 0) {	1872	if (domain->iommu_refcnt[iommu->seq_id] == 0) {
2034	num = domain->iommu_did[iommu->seq_id];	1873	num = domain->iommu_did[iommu->seq_id];
2035	clear_bit(num, iommu->domain_ids);	1874	clear_bit(num, iommu->domain_ids);
2036	set_iommu_domain(iommu, num, NULL);
2037
2038	domain_update_iommu_cap(domain);	1875	domain_update_iommu_cap(domain);
2039	domain->iommu_did[iommu->seq_id] = 0;	1876	domain->iommu_did[iommu->seq_id] = 0;
2040	}	1877	}
@@ -2067,7 +1904,7 @@ static void domain_exit(struct dmar_domain *domain) · 26 unmodified lines
2067	put_pages_list(&freelist);	1904	put_pages_list(&freelist);
2068	}	1905	}
2069		1906
2070	free_domain_mem(domain);	1907	kfree(domain);
2071	}	1908	}
2072		1909
2073	/*	1910	/*
@@ -2550,15 +2387,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 · 476 unmodified lines
2550	__iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);	2387	__iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);
2551	}	2388	}
2552		2389
2553	static inline void unlink_domain_info(struct device_domain_info *info)
2554	{
2555	assert_spin_locked(&device_domain_lock);
2556	list_del(&info->link);
2557	list_del(&info->global);
2558	if (info->dev)
2559	dev_iommu_priv_set(info->dev, NULL);
2560	}
2561
2562	static void domain_remove_dev_info(struct dmar_domain *domain)	2390	static void domain_remove_dev_info(struct dmar_domain *domain)
2563	{	2391	{
2564	struct device_domain_info info, tmp;	2392	struct device_domain_info info, tmp;
@@ -2570,24 +2398,6 @@ static void domain_remove_dev_info(struct dmar_domain *domain) · 5 unmodified lines
2570	spin_unlock_irqrestore(&device_domain_lock, flags);	2398	spin_unlock_irqrestore(&device_domain_lock, flags);
2571	}	2399	}
2572		2400
2573	struct dmar_domain find_domain(struct device dev)
2574	{
2575	struct device_domain_info *info;
2576
2577	if (unlikely(!dev \|\| !dev->iommu))
2578	return NULL;
2579
2580	if (unlikely(attach_deferred(dev)))
2581	return NULL;
2582
2583	/* No lock here, assumes no domain exit in normal case */
2584	info = get_domain_info(dev);
2585	if (likely(info))
2586	return info->domain;
2587
2588	return NULL;
2589	}
2590
2591	static inline struct device_domain_info *	2401	static inline struct device_domain_info *
2592	dmar_search_domain_by_dev_info(int segment, int bus, int devfn)	2402	dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2593	{	2403	{
@@ -2648,93 +2458,20 @@ static struct dmar_domain dmar_insert_one_dev_info(struct intel_iommu iommu, · 54 unmodified lines
2648	struct device *dev,	2458	struct device *dev,
2649	struct dmar_domain *domain)	2459	struct dmar_domain *domain)
2650	{	2460	{
2651	struct dmar_domain *found = NULL;	2461	struct device_domain_info *info = dev_iommu_priv_get(dev);
2652	struct device_domain_info *info;
2653	unsigned long flags;	2462	unsigned long flags;
2654	int ret;	2463	int ret;
2655		2464
2656	info = alloc_devinfo_mem();
2657	if (!info)
2658	return NULL;
2659
2660	if (!dev_is_real_dma_subdevice(dev)) {
2661	info->bus = bus;
2662	info->devfn = devfn;
2663	info->segment = iommu->segment;
2664	} else {
2665	struct pci_dev *pdev = to_pci_dev(dev);
2666
2667	info->bus = pdev->bus->number;
2668	info->devfn = pdev->devfn;
2669	info->segment = pci_domain_nr(pdev->bus);
2670	}
2671
2672	info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2673	info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2674	info->ats_qdep = 0;
2675	info->dev = dev;
2676	info->domain = domain;
2677	info->iommu = iommu;
2678	info->pasid_table = NULL;
2679	info->auxd_enabled = 0;
2680	INIT_LIST_HEAD(&info->subdevices);
2681
2682	if (dev && dev_is_pci(dev)) {
2683	struct pci_dev *pdev = to_pci_dev(info->dev);
2684
2685	if (ecap_dev_iotlb_support(iommu->ecap) &&
2686	pci_ats_supported(pdev) &&
2687	dmar_find_matched_atsr_unit(pdev))
2688	info->ats_supported = 1;
2689
2690	if (sm_supported(iommu)) {
2691	if (pasid_supported(iommu)) {
2692	int features = pci_pasid_features(pdev);
2693	if (features >= 0)
2694	info->pasid_supported = features \| 1;
2695	}
2696
2697	if (info->ats_supported && ecap_prs(iommu->ecap) &&
2698	pci_pri_supported(pdev))
2699	info->pri_supported = 1;
2700	}
2701	}
2702
2703	spin_lock_irqsave(&device_domain_lock, flags);	2465	spin_lock_irqsave(&device_domain_lock, flags);
2704	if (dev)	2466	info->domain = domain;
2705	found = find_domain(dev);
2706
2707	if (!found) {
2708	struct device_domain_info *info2;
2709	info2 = dmar_search_domain_by_dev_info(info->segment, info->bus,
2710	info->devfn);
2711	if (info2) {
2712	found = info2->domain;
2713	info2->dev = dev;
2714	}
2715	}
2716
2717	if (found) {
2718	spin_unlock_irqrestore(&device_domain_lock, flags);
2719	free_devinfo_mem(info);
2720	/* Caller must free the original domain */
2721	return found;
2722	}
2723
2724	spin_lock(&iommu->lock);	2467	spin_lock(&iommu->lock);
2725	ret = domain_attach_iommu(domain, iommu);	2468	ret = domain_attach_iommu(domain, iommu);
2726	spin_unlock(&iommu->lock);	2469	spin_unlock(&iommu->lock);
2727
2728	if (ret) {	2470	if (ret) {
2729	spin_unlock_irqrestore(&device_domain_lock, flags);	2471	spin_unlock_irqrestore(&device_domain_lock, flags);
2730	free_devinfo_mem(info);
2731	return NULL;	2472	return NULL;
2732	}	2473	}
2733
2734	list_add(&info->link, &domain->devices);	2474	list_add(&info->link, &domain->devices);
2735	list_add(&info->global, &device_domain_list);
2736	if (dev)
2737	dev_iommu_priv_set(dev, info);
2738	spin_unlock_irqrestore(&device_domain_lock, flags);	2475	spin_unlock_irqrestore(&device_domain_lock, flags);
2739		2476
2740	/* PASID table is mandatory for a PCI device in scalable mode. */	2477	/* PASID table is mandatory for a PCI device in scalable mode. */
@@ -3460,70 +3197,6 @@ error: · 719 unmodified lines
3460	return ret;	3197	return ret;
3461	}	3198	}
3462		3199
3463	static inline int iommu_domain_cache_init(void)
3464	{
3465	int ret = 0;
3466
3467	iommu_domain_cache = kmem_cache_create("iommu_domain",
3468	sizeof(struct dmar_domain),
3469	0,
3470	SLAB_HWCACHE_ALIGN,
3471
3472	NULL);
3473	if (!iommu_domain_cache) {
3474	pr_err("Couldn't create iommu_domain cache\n");
3475	ret = -ENOMEM;
3476	}
3477
3478	return ret;
3479	}
3480
3481	static inline int iommu_devinfo_cache_init(void)
3482	{
3483	int ret = 0;
3484
3485	iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3486	sizeof(struct device_domain_info),
3487	0,
3488	SLAB_HWCACHE_ALIGN,
3489	NULL);
3490	if (!iommu_devinfo_cache) {
3491	pr_err("Couldn't create devinfo cache\n");
3492	ret = -ENOMEM;
3493	}
3494
3495	return ret;
3496	}
3497
3498	static int __init iommu_init_mempool(void)
3499	{
3500	int ret;
3501	ret = iova_cache_get();
3502	if (ret)
3503	return ret;
3504
3505	ret = iommu_domain_cache_init();
3506	if (ret)
3507	goto domain_error;
3508
3509	ret = iommu_devinfo_cache_init();
3510	if (!ret)
3511	return ret;
3512
3513	kmem_cache_destroy(iommu_domain_cache);
3514	domain_error:
3515	iova_cache_put();
3516
3517	return -ENOMEM;
3518	}
3519
3520	static void __init iommu_exit_mempool(void)
3521	{
3522	kmem_cache_destroy(iommu_devinfo_cache);
3523	kmem_cache_destroy(iommu_domain_cache);
3524	iova_cache_put();
3525	}
3526
3527	static void __init init_no_remapping_devices(void)	3200	static void __init init_no_remapping_devices(void)
3528	{	3201	{
3529	struct dmar_drhd_unit *drhd;	3202	struct dmar_drhd_unit *drhd;
@@ -3691,7 +3364,7 @@ static void __init init_iommu_pm_ops(void) · 161 unmodified lines
3691	static inline void init_iommu_pm_ops(void) {}	3364	static inline void init_iommu_pm_ops(void) {}
3692	#endif /* CONFIG_PM */	3365	#endif /* CONFIG_PM */
3693		3366
3694	static int rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)	3367	static int __init rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
3695	{	3368	{
3696	if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) \|\|	3369	if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) \|\|
3697	!IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) \|\|	3370	!IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) \|\|
@@ -4020,7 +3693,31 @@ static void intel_iommu_free_dmars(void) · 322 unmodified lines
4020	}	3693	}
4021	}	3694	}
4022		3695
4023	int dmar_find_matched_atsr_unit(struct pci_dev *dev)	3696	static struct dmar_satc_unit dmar_find_matched_satc_unit(struct pci_dev dev)
		3697	{
		3698	struct dmar_satc_unit *satcu;
		3699	struct acpi_dmar_satc *satc;
		3700	struct device *tmp;
		3701	int i;
		3702
		3703	dev = pci_physfn(dev);
		3704	rcu_read_lock();
		3705
		3706	list_for_each_entry_rcu(satcu, &dmar_satc_units, list) {
		3707	satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
		3708	if (satc->segment != pci_domain_nr(dev->bus))
		3709	continue;
		3710	for_each_dev_scope(satcu->devices, satcu->devices_cnt, i, tmp)
		3711	if (to_pci_dev(tmp) == dev)
		3712	goto out;
		3713	}
		3714	satcu = NULL;
		3715	out:
		3716	rcu_read_unlock();
		3717	return satcu;
		3718	}
		3719
		3720	static int dmar_ats_supported(struct pci_dev dev, struct intel_iommu iommu)
4024	{	3721	{
4025	int i, ret = 1;	3722	int i, ret = 1;
4026	struct pci_bus *bus;	3723	struct pci_bus *bus;
@@ -4028,8 +3725,20 @@ int dmar_find_matched_atsr_unit(struct pci_dev *dev) · 1 unmodified line
4028	struct device *tmp;	3725	struct device *tmp;
4029	struct acpi_dmar_atsr *atsr;	3726	struct acpi_dmar_atsr *atsr;
4030	struct dmar_atsr_unit *atsru;	3727	struct dmar_atsr_unit *atsru;
		3728	struct dmar_satc_unit *satcu;
4031		3729
4032	dev = pci_physfn(dev);	3730	dev = pci_physfn(dev);
		3731	satcu = dmar_find_matched_satc_unit(dev);
		3732	if (satcu)
		3733	/*
		3734	* This device supports ATS as it is in SATC table.
		3735	* When IOMMU is in legacy mode, enabling ATS is done
		3736	* automatically by HW for the device that requires
		3737	* ATS, hence OS should not enable this device ATS
		3738	* to avoid duplicated TLB invalidation.
		3739	*/
		3740	return !(satcu->atc_required && !sm_supported(iommu));
		3741
4033	for (bus = dev->bus; bus; bus = bus->parent) {	3742	for (bus = dev->bus; bus; bus = bus->parent) {
4034	bridge = bus->self;	3743	bridge = bus->self;
4035	/* If it's an integrated device, allow ATS */	3744	/* If it's an integrated device, allow ATS */
@@ -4375,12 +4084,6 @@ int __init intel_iommu_init(void) · 339 unmodified lines
4375	force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) \|\|	4084	force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) \|\|
4376	platform_optin_force_iommu();	4085	platform_optin_force_iommu();
4377		4086
4378	if (iommu_init_mempool()) {
4379	if (force_on)
4380	panic("tboot: Failed to initialize iommu memory\n");
4381	return -ENOMEM;
4382	}
4383
4384	down_write(&dmar_global_lock);	4087	down_write(&dmar_global_lock);
4385	if (dmar_table_init()) {	4088	if (dmar_table_init()) {
4386	if (force_on)	4089	if (force_on)
@@ -4501,7 +4204,6 @@ int __init intel_iommu_init(void) · 114 unmodified lines
4501	out_free_dmar:	4204	out_free_dmar:
4502	intel_iommu_free_dmars();	4205	intel_iommu_free_dmars();
4503	up_write(&dmar_global_lock);	4206	up_write(&dmar_global_lock);
4504	iommu_exit_mempool();
4505	return ret;	4207	return ret;
4506	}	4208	}
4507		4209
@@ -4552,13 +4254,11 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info) · 44 unmodified lines
4552	intel_pasid_free_table(info->dev);	4254	intel_pasid_free_table(info->dev);
4553	}	4255	}
4554		4256
4555	unlink_domain_info(info);	4257	list_del(&info->link);
4556		4258
4557	spin_lock_irqsave(&iommu->lock, flags);	4259	spin_lock_irqsave(&iommu->lock, flags);
4558	domain_detach_iommu(domain, iommu);	4260	domain_detach_iommu(domain, iommu);
4559	spin_unlock_irqrestore(&iommu->lock, flags);	4261	spin_unlock_irqrestore(&iommu->lock, flags);
4560
4561	free_devinfo_mem(info);
4562	}	4262	}
4563		4263
4564	static void dmar_remove_one_dev_info(struct device *dev)	4264	static void dmar_remove_one_dev_info(struct device *dev)
@@ -4567,7 +4267,7 @@ static void dmar_remove_one_dev_info(struct device *dev) · 2 unmodified lines
4567	unsigned long flags;	4267	unsigned long flags;
4568		4268
4569	spin_lock_irqsave(&device_domain_lock, flags);	4269	spin_lock_irqsave(&device_domain_lock, flags);
4570	info = get_domain_info(dev);	4270	info = dev_iommu_priv_get(dev);
4571	if (info)	4271	if (info)
4572	__dmar_remove_one_dev_info(info);	4272	__dmar_remove_one_dev_info(info);
4573	spin_unlock_irqrestore(&device_domain_lock, flags);	4273	spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -4637,183 +4337,6 @@ static void intel_iommu_domain_free(struct iommu_domain *domain) · 63 unmodified lines
4637	domain_exit(to_dmar_domain(domain));	4337	domain_exit(to_dmar_domain(domain));
4638	}	4338	}
4639		4339
4640	/*
4641	* Check whether a @domain could be attached to the @dev through the
4642	* aux-domain attach/detach APIs.
4643	*/
4644	static inline bool
4645	is_aux_domain(struct device dev, struct iommu_domain domain)
4646	{
4647	struct device_domain_info *info = get_domain_info(dev);
4648
4649	return info && info->auxd_enabled &&
4650	domain->type == IOMMU_DOMAIN_UNMANAGED;
4651	}
4652
4653	static inline struct subdev_domain_info *
4654	lookup_subdev_info(struct dmar_domain domain, struct device dev)
4655	{
4656	struct subdev_domain_info *sinfo;
4657
4658	if (!list_empty(&domain->subdevices)) {
4659	list_for_each_entry(sinfo, &domain->subdevices, link_domain) {
4660	if (sinfo->pdev == dev)
4661	return sinfo;
4662	}
4663	}
4664
4665	return NULL;
4666	}
4667
4668	static int auxiliary_link_device(struct dmar_domain *domain,
4669	struct device *dev)
4670	{
4671	struct device_domain_info *info = get_domain_info(dev);
4672	struct subdev_domain_info *sinfo = lookup_subdev_info(domain, dev);
4673
4674	assert_spin_locked(&device_domain_lock);
4675	if (WARN_ON(!info))
4676	return -EINVAL;
4677
4678	if (!sinfo) {
4679	sinfo = kzalloc(sizeof(*sinfo), GFP_ATOMIC);
4680	if (!sinfo)
4681	return -ENOMEM;
4682	sinfo->domain = domain;
4683	sinfo->pdev = dev;
4684	list_add(&sinfo->link_phys, &info->subdevices);
4685	list_add(&sinfo->link_domain, &domain->subdevices);
4686	}
4687
4688	return ++sinfo->users;
4689	}
4690
4691	static int auxiliary_unlink_device(struct dmar_domain *domain,
4692	struct device *dev)
4693	{
4694	struct device_domain_info *info = get_domain_info(dev);
4695	struct subdev_domain_info *sinfo = lookup_subdev_info(domain, dev);
4696	int ret;
4697
4698	assert_spin_locked(&device_domain_lock);
4699	if (WARN_ON(!info \|\| !sinfo \|\| sinfo->users <= 0))
4700	return -EINVAL;
4701
4702	ret = --sinfo->users;
4703	if (!ret) {
4704	list_del(&sinfo->link_phys);
4705	list_del(&sinfo->link_domain);
4706	kfree(sinfo);
4707	}
4708
4709	return ret;
4710	}
4711
4712	static int aux_domain_add_dev(struct dmar_domain *domain,
4713	struct device *dev)
4714	{
4715	int ret;
4716	unsigned long flags;
4717	struct intel_iommu *iommu;
4718
4719	iommu = device_to_iommu(dev, NULL, NULL);
4720	if (!iommu)
4721	return -ENODEV;
4722
4723	if (domain->default_pasid <= 0) {
4724	u32 pasid;
4725
4726	/* No private data needed for the default pasid */
4727	pasid = ioasid_alloc(NULL, PASID_MIN,
4728	pci_max_pasids(to_pci_dev(dev)) - 1,
4729	NULL);
4730	if (pasid == INVALID_IOASID) {
4731	pr_err("Can't allocate default pasid\n");
4732	return -ENODEV;
4733	}
4734	domain->default_pasid = pasid;
4735	}
4736
4737	spin_lock_irqsave(&device_domain_lock, flags);
4738	ret = auxiliary_link_device(domain, dev);
4739	if (ret <= 0)
4740	goto link_failed;
4741
4742	/*
4743	* Subdevices from the same physical device can be attached to the
4744	* same domain. For such cases, only the first subdevice attachment
4745	* needs to go through the full steps in this function. So if ret >
4746	* 1, just goto out.
4747	*/
4748	if (ret > 1)
4749	goto out;
4750
4751	/*
4752	* iommu->lock must be held to attach domain to iommu and setup the
4753	* pasid entry for second level translation.
4754	*/
4755	spin_lock(&iommu->lock);
4756	ret = domain_attach_iommu(domain, iommu);
4757	if (ret)
4758	goto attach_failed;
4759
4760	/* Setup the PASID entry for mediated devices: */
4761	if (domain_use_first_level(domain))
4762	ret = domain_setup_first_level(iommu, domain, dev,
4763	domain->default_pasid);
4764	else
4765	ret = intel_pasid_setup_second_level(iommu, domain, dev,
4766	domain->default_pasid);
4767	if (ret)
4768	goto table_failed;
4769
4770	spin_unlock(&iommu->lock);
4771	out:
4772	spin_unlock_irqrestore(&device_domain_lock, flags);
4773
4774	return 0;
4775
4776	table_failed:
4777	domain_detach_iommu(domain, iommu);
4778	attach_failed:
4779	spin_unlock(&iommu->lock);
4780	auxiliary_unlink_device(domain, dev);
4781	link_failed:
4782	spin_unlock_irqrestore(&device_domain_lock, flags);
4783	if (list_empty(&domain->subdevices) && domain->default_pasid > 0)
4784	ioasid_free(domain->default_pasid);
4785
4786	return ret;
4787	}
4788
4789	static void aux_domain_remove_dev(struct dmar_domain *domain,
4790	struct device *dev)
4791	{
4792	struct device_domain_info *info;
4793	struct intel_iommu *iommu;
4794	unsigned long flags;
4795
4796	if (!is_aux_domain(dev, &domain->domain))
4797	return;
4798
4799	spin_lock_irqsave(&device_domain_lock, flags);
4800	info = get_domain_info(dev);
4801	iommu = info->iommu;
4802
4803	if (!auxiliary_unlink_device(domain, dev)) {
4804	spin_lock(&iommu->lock);
4805	intel_pasid_tear_down_entry(iommu, dev,
4806	domain->default_pasid, false);
4807	domain_detach_iommu(domain, iommu);
4808	spin_unlock(&iommu->lock);
4809	}
4810
4811	spin_unlock_irqrestore(&device_domain_lock, flags);
4812
4813	if (list_empty(&domain->subdevices) && domain->default_pasid > 0)
4814	ioasid_free(domain->default_pasid);
4815	}
4816
4817	static int prepare_domain_attach_device(struct iommu_domain *domain,	4340	static int prepare_domain_attach_device(struct iommu_domain *domain,
4818	struct device *dev)	4341	struct device *dev)
4819	{	4342	{
@@ -4825,13 +4348,6 @@ static int prepare_domain_attach_device(struct iommu_domain *domain, · 5 unmodified lines
4825	if (!iommu)	4348	if (!iommu)
4826	return -ENODEV;	4349	return -ENODEV;
4827		4350
4828	if ((dmar_domain->flags & DOMAIN_FLAG_NESTING_MODE) &&
4829	!ecap_nest(iommu->ecap)) {
4830	dev_err(dev, "%s: iommu not support nested translation\n",
4831	iommu->name);
4832	return -EINVAL;
4833	}
4834
4835	/* check if this iommu agaw is sufficient for max mapped address */	4351	/* check if this iommu agaw is sufficient for max mapped address */
4836	addr_width = agaw_to_width(iommu->agaw);	4352	addr_width = agaw_to_width(iommu->agaw);
4837	if (addr_width > cap_mgaw(iommu->cap))	4353	if (addr_width > cap_mgaw(iommu->cap))
@@ -4873,15 +4389,11 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, · 35 unmodified lines
4873	return -EPERM;	4389	return -EPERM;
4874	}	4390	}
4875		4391
4876	if (is_aux_domain(dev, domain))
4877	return -EPERM;
4878
4879	/* normally dev is not mapped */	4392	/* normally dev is not mapped */
4880	if (unlikely(domain_context_mapped(dev))) {	4393	if (unlikely(domain_context_mapped(dev))) {
4881	struct dmar_domain *old_domain;	4394	struct device_domain_info *info = dev_iommu_priv_get(dev);
4882		4395
4883	old_domain = find_domain(dev);	4396	if (info->domain)
4884	if (old_domain)
4885	dmar_remove_one_dev_info(dev);	4397	dmar_remove_one_dev_info(dev);
4886	}	4398	}
4887		4399
@@ -4892,212 +4404,12 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, · 4 unmodified lines
4892	return domain_add_dev_info(to_dmar_domain(domain), dev);	4404	return domain_add_dev_info(to_dmar_domain(domain), dev);
4893	}	4405	}
4894		4406
4895	static int intel_iommu_aux_attach_device(struct iommu_domain *domain,
4896	struct device *dev)
4897	{
4898	int ret;
4899
4900	if (!is_aux_domain(dev, domain))
4901	return -EPERM;
4902
4903	ret = prepare_domain_attach_device(domain, dev);
4904	if (ret)
4905	return ret;
4906
4907	return aux_domain_add_dev(to_dmar_domain(domain), dev);
4908	}
4909
4910	static void intel_iommu_detach_device(struct iommu_domain *domain,	4407	static void intel_iommu_detach_device(struct iommu_domain *domain,
4911	struct device *dev)	4408	struct device *dev)
4912	{	4409	{
4913	dmar_remove_one_dev_info(dev);	4410	dmar_remove_one_dev_info(dev);
4914	}	4411	}
4915		4412
4916	static void intel_iommu_aux_detach_device(struct iommu_domain *domain,
4917	struct device *dev)
4918	{
4919	aux_domain_remove_dev(to_dmar_domain(domain), dev);
4920	}
4921
4922	#ifdef CONFIG_INTEL_IOMMU_SVM
4923	/*
4924	* 2D array for converting and sanitizing IOMMU generic TLB granularity to
4925	* VT-d granularity. Invalidation is typically included in the unmap operation
4926	* as a result of DMA or VFIO unmap. However, for assigned devices guest
4927	* owns the first level page tables. Invalidations of translation caches in the
4928	* guest are trapped and passed down to the host.
4929	*
4930	* vIOMMU in the guest will only expose first level page tables, therefore
4931	* we do not support IOTLB granularity for request without PASID (second level).
4932	*
4933	* For example, to find the VT-d granularity encoding for IOTLB
4934	* type and page selective granularity within PASID:
4935	* X: indexed by iommu cache type
4936	* Y: indexed by enum iommu_inv_granularity
4937	* [IOMMU_CACHE_INV_TYPE_IOTLB][IOMMU_INV_GRANU_ADDR]
4938	*/
4939
4940	static const int
4941	inv_type_granu_table[IOMMU_CACHE_INV_TYPE_NR][IOMMU_INV_GRANU_NR] = {
4942	/*
4943	* PASID based IOTLB invalidation: PASID selective (per PASID),
4944	* page selective (address granularity)
4945	*/
4946	{-EINVAL, QI_GRAN_NONG_PASID, QI_GRAN_PSI_PASID},
4947	/* PASID based dev TLBs */
4948	{-EINVAL, -EINVAL, QI_DEV_IOTLB_GRAN_PASID_SEL},
4949	/* PASID cache */
4950	{-EINVAL, -EINVAL, -EINVAL}
4951	};
4952
4953	static inline int to_vtd_granularity(int type, int granu)
4954	{
4955	return inv_type_granu_table[type][granu];
4956	}
4957
4958	static inline u64 to_vtd_size(u64 granu_size, u64 nr_granules)
4959	{
4960	u64 nr_pages = (granu_size * nr_granules) >> VTD_PAGE_SHIFT;
4961
4962	/* VT-d size is encoded as 2^size of 4K pages, 0 for 4k, 9 for 2MB, etc.
4963	* IOMMU cache invalidate API passes granu_size in bytes, and number of
4964	* granu size in contiguous memory.
4965	*/
4966	return order_base_2(nr_pages);
4967	}
4968
4969	static int
4970	intel_iommu_sva_invalidate(struct iommu_domain domain, struct device dev,
4971	struct iommu_cache_invalidate_info *inv_info)
4972	{
4973	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4974	struct device_domain_info *info;
4975	struct intel_iommu *iommu;
4976	unsigned long flags;
4977	int cache_type;
4978	u8 bus, devfn;
4979	u16 did, sid;
4980	int ret = 0;
4981	u64 size = 0;
4982
4983	if (!inv_info \|\| !dmar_domain)
4984	return -EINVAL;
4985
4986	if (!dev \|\| !dev_is_pci(dev))
4987	return -ENODEV;
4988
4989	iommu = device_to_iommu(dev, &bus, &devfn);
4990	if (!iommu)
4991	return -ENODEV;
4992
4993	if (!(dmar_domain->flags & DOMAIN_FLAG_NESTING_MODE))
4994	return -EINVAL;
4995
4996	spin_lock_irqsave(&device_domain_lock, flags);
4997	spin_lock(&iommu->lock);
4998	info = get_domain_info(dev);
4999	if (!info) {
5000	ret = -EINVAL;
5001	goto out_unlock;
5002	}
5003	did = dmar_domain->iommu_did[iommu->seq_id];
5004	sid = PCI_DEVID(bus, devfn);
5005
5006	/* Size is only valid in address selective invalidation */
5007	if (inv_info->granularity == IOMMU_INV_GRANU_ADDR)
5008	size = to_vtd_size(inv_info->granu.addr_info.granule_size,
5009	inv_info->granu.addr_info.nb_granules);
5010
5011	for_each_set_bit(cache_type,
5012	(unsigned long *)&inv_info->cache,
5013	IOMMU_CACHE_INV_TYPE_NR) {
5014	int granu = 0;
5015	u64 pasid = 0;
5016	u64 addr = 0;
5017
5018	granu = to_vtd_granularity(cache_type, inv_info->granularity);
5019	if (granu == -EINVAL) {
5020	pr_err_ratelimited("Invalid cache type and granu combination %d/%d\n",
5021	cache_type, inv_info->granularity);
5022	break;
5023	}
5024
5025	/*
5026	* PASID is stored in different locations based on the
5027	* granularity.
5028	*/
5029	if (inv_info->granularity == IOMMU_INV_GRANU_PASID &&
5030	(inv_info->granu.pasid_info.flags & IOMMU_INV_PASID_FLAGS_PASID))
5031	pasid = inv_info->granu.pasid_info.pasid;
5032	else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR &&
5033	(inv_info->granu.addr_info.flags & IOMMU_INV_ADDR_FLAGS_PASID))
5034	pasid = inv_info->granu.addr_info.pasid;
5035
5036	switch (BIT(cache_type)) {
5037	case IOMMU_CACHE_INV_TYPE_IOTLB:
5038	/* HW will ignore LSB bits based on address mask */
5039	if (inv_info->granularity == IOMMU_INV_GRANU_ADDR &&
5040	size &&
5041	(inv_info->granu.addr_info.addr & ((BIT(VTD_PAGE_SHIFT + size)) - 1))) {
5042	pr_err_ratelimited("User address not aligned, 0x%llx, size order %llu\n",
5043	inv_info->granu.addr_info.addr, size);
5044	}
5045
5046	/*
5047	* If granu is PASID-selective, address is ignored.
5048	* We use npages = -1 to indicate that.
5049	*/
5050	qi_flush_piotlb(iommu, did, pasid,
5051	mm_to_dma_pfn(inv_info->granu.addr_info.addr),
5052	(granu == QI_GRAN_NONG_PASID) ? -1 : 1 << size,
5053	inv_info->granu.addr_info.flags & IOMMU_INV_ADDR_FLAGS_LEAF);
5054
5055	if (!info->ats_enabled)
5056	break;
5057	/*
5058	* Always flush device IOTLB if ATS is enabled. vIOMMU
5059	* in the guest may assume IOTLB flush is inclusive,
5060	* which is more efficient.
5061	*/
5062	fallthrough;
5063	case IOMMU_CACHE_INV_TYPE_DEV_IOTLB:
5064	/*
5065	* PASID based device TLB invalidation does not support
5066	* IOMMU_INV_GRANU_PASID granularity but only supports
5067	* IOMMU_INV_GRANU_ADDR.
5068	* The equivalent of that is we set the size to be the
5069	* entire range of 64 bit. User only provides PASID info
5070	* without address info. So we set addr to 0.
5071	*/
5072	if (inv_info->granularity == IOMMU_INV_GRANU_PASID) {
5073	size = 64 - VTD_PAGE_SHIFT;
5074	addr = 0;
5075	} else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR) {
5076	addr = inv_info->granu.addr_info.addr;
5077	}
5078
5079	if (info->ats_enabled)
5080	qi_flush_dev_iotlb_pasid(iommu, sid,
5081	info->pfsid, pasid,
5082	info->ats_qdep, addr,
5083	size);
5084	else
5085	pr_warn_ratelimited("Passdown device IOTLB flush w/o ATS!\n");
5086	break;
5087	default:
5088	dev_err_ratelimited(dev, "Unsupported IOMMU invalidation type %d\n",
5089	cache_type);
5090	ret = -EINVAL;
5091	}
5092	}
5093	out_unlock:
5094	spin_unlock(&iommu->lock);
5095	spin_unlock_irqrestore(&device_domain_lock, flags);
5096
5097	return ret;
5098	}
5099	#endif
5100
5101	static int intel_iommu_map(struct iommu_domain *domain,	4413	static int intel_iommu_map(struct iommu_domain *domain,
5102	unsigned long iova, phys_addr_t hpa,	4414	unsigned long iova, phys_addr_t hpa,
5103	size_t size, int iommu_prot, gfp_t gfp)	4415	size_t size, int iommu_prot, gfp_t gfp)
@@ -5245,28 +4557,73 @@ static bool intel_iommu_capable(enum iommu_cap cap) · 141 unmodified lines
5245		4557
5246	static struct iommu_device intel_iommu_probe_device(struct device dev)	4558	static struct iommu_device intel_iommu_probe_device(struct device dev)
5247	{	4559	{
		4560	struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
		4561	struct device_domain_info *info;
5248	struct intel_iommu *iommu;	4562	struct intel_iommu *iommu;
		4563	unsigned long flags;
		4564	u8 bus, devfn;
5249		4565
5250	iommu = device_to_iommu(dev, NULL, NULL);	4566	iommu = device_to_iommu(dev, &bus, &devfn);
5251	if (!iommu)	4567	if (!iommu)
5252	return ERR_PTR(-ENODEV);	4568	return ERR_PTR(-ENODEV);
5253		4569
5254	if (translation_pre_enabled(iommu))	4570	info = kzalloc(sizeof(*info), GFP_KERNEL);
5255	dev_iommu_priv_set(dev, DEFER_DEVICE_DOMAIN_INFO);	4571	if (!info)
		4572	return ERR_PTR(-ENOMEM);
		4573
		4574	if (dev_is_real_dma_subdevice(dev)) {
		4575	info->bus = pdev->bus->number;
		4576	info->devfn = pdev->devfn;
		4577	info->segment = pci_domain_nr(pdev->bus);
		4578	} else {
		4579	info->bus = bus;
		4580	info->devfn = devfn;
		4581	info->segment = iommu->segment;
		4582	}
		4583
		4584	info->dev = dev;
		4585	info->iommu = iommu;
		4586	if (dev_is_pci(dev)) {
		4587	if (ecap_dev_iotlb_support(iommu->ecap) &&
		4588	pci_ats_supported(pdev) &&
		4589	dmar_ats_supported(pdev, iommu))
		4590	info->ats_supported = 1;
		4591
		4592	if (sm_supported(iommu)) {
		4593	if (pasid_supported(iommu)) {
		4594	int features = pci_pasid_features(pdev);
		4595
		4596	if (features >= 0)
		4597	info->pasid_supported = features \| 1;
		4598	}
		4599
		4600	if (info->ats_supported && ecap_prs(iommu->ecap) &&
		4601	pci_pri_supported(pdev))
		4602	info->pri_supported = 1;
		4603	}
		4604	}
		4605
		4606	spin_lock_irqsave(&device_domain_lock, flags);
		4607	list_add(&info->global, &device_domain_list);
		4608	dev_iommu_priv_set(dev, info);
		4609	spin_unlock_irqrestore(&device_domain_lock, flags);
5256		4610
5257	return &iommu->iommu;	4611	return &iommu->iommu;
5258	}	4612	}
5259		4613
5260	static void intel_iommu_release_device(struct device *dev)	4614	static void intel_iommu_release_device(struct device *dev)
5261	{	4615	{
5262	struct intel_iommu *iommu;	4616	struct device_domain_info *info = dev_iommu_priv_get(dev);
5263		4617	unsigned long flags;
5264	iommu = device_to_iommu(dev, NULL, NULL);
5265	if (!iommu)
5266	return;
5267		4618
5268	dmar_remove_one_dev_info(dev);	4619	dmar_remove_one_dev_info(dev);
5269		4620
		4621	spin_lock_irqsave(&device_domain_lock, flags);
		4622	dev_iommu_priv_set(dev, NULL);
		4623	list_del(&info->global);
		4624	spin_unlock_irqrestore(&device_domain_lock, flags);
		4625
		4626	kfree(info);
5270	set_dma_ops(dev, NULL);	4627	set_dma_ops(dev, NULL);
5271	}	4628	}
5272		4629
@@ -5335,14 +4692,14 @@ static void intel_iommu_get_resv_regions(struct device *device, · 62 unmodified lines
5335		4692
5336	int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev)	4693	int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev)
5337	{	4694	{
5338	struct device_domain_info *info;	4695	struct device_domain_info *info = dev_iommu_priv_get(dev);
5339	struct context_entry *context;	4696	struct context_entry *context;
5340	struct dmar_domain *domain;	4697	struct dmar_domain *domain;
5341	unsigned long flags;	4698	unsigned long flags;
5342	u64 ctx_lo;	4699	u64 ctx_lo;
5343	int ret;	4700	int ret;
5344		4701
5345	domain = find_domain(dev);	4702	domain = info->domain;
5346	if (!domain)	4703	if (!domain)
5347	return -EINVAL;	4704	return -EINVAL;
5348		4705
@@ -5350,8 +4707,7 @@ int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev) · 1 unmodified line
5350	spin_lock(&iommu->lock);	4707	spin_lock(&iommu->lock);
5351		4708
5352	ret = -EINVAL;	4709	ret = -EINVAL;
5353	info = get_domain_info(dev);	4710	if (!info->pasid_supported)
5354	if (!info \|\| !info->pasid_supported)
5355	goto out;	4711	goto out;
5356		4712
5357	context = iommu_context_addr(iommu, info->bus, info->devfn, 0);	4713	context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
@@ -5391,49 +4747,9 @@ static struct iommu_group intel_iommu_device_group(struct device dev) · 33 unmodified lines
5391	return generic_device_group(dev);	4747	return generic_device_group(dev);
5392	}	4748	}
5393		4749
5394	static int intel_iommu_enable_auxd(struct device *dev)
5395	{
5396	struct device_domain_info *info;
5397	struct intel_iommu *iommu;
5398	unsigned long flags;
5399	int ret;
5400
5401	iommu = device_to_iommu(dev, NULL, NULL);
5402	if (!iommu \|\| dmar_disabled)
5403	return -EINVAL;
5404
5405	if (!sm_supported(iommu) \|\| !pasid_supported(iommu))
5406	return -EINVAL;
5407
5408	ret = intel_iommu_enable_pasid(iommu, dev);
5409	if (ret)
5410	return -ENODEV;
5411
5412	spin_lock_irqsave(&device_domain_lock, flags);
5413	info = get_domain_info(dev);
5414	info->auxd_enabled = 1;
5415	spin_unlock_irqrestore(&device_domain_lock, flags);
5416
5417	return 0;
5418	}
5419
5420	static int intel_iommu_disable_auxd(struct device *dev)
5421	{
5422	struct device_domain_info *info;
5423	unsigned long flags;
5424
5425	spin_lock_irqsave(&device_domain_lock, flags);
5426	info = get_domain_info(dev);
5427	if (!WARN_ON(!info))
5428	info->auxd_enabled = 0;
5429	spin_unlock_irqrestore(&device_domain_lock, flags);
5430
5431	return 0;
5432	}
5433
5434	static int intel_iommu_enable_sva(struct device *dev)	4750	static int intel_iommu_enable_sva(struct device *dev)
5435	{	4751	{
5436	struct device_domain_info *info = get_domain_info(dev);	4752	struct device_domain_info *info = dev_iommu_priv_get(dev);
5437	struct intel_iommu *iommu;	4753	struct intel_iommu *iommu;
5438	int ret;	4754	int ret;
5439		4755
@@ -5462,7 +4778,7 @@ static int intel_iommu_enable_sva(struct device *dev) · 22 unmodified lines
5462		4778
5463	static int intel_iommu_disable_sva(struct device *dev)	4779	static int intel_iommu_disable_sva(struct device *dev)
5464	{	4780	{
5465	struct device_domain_info *info = get_domain_info(dev);	4781	struct device_domain_info *info = dev_iommu_priv_get(dev);
5466	struct intel_iommu *iommu = info->iommu;	4782	struct intel_iommu *iommu = info->iommu;
5467	int ret;	4783	int ret;
5468		4784
@@ -5475,7 +4791,7 @@ static int intel_iommu_disable_sva(struct device *dev) · 6 unmodified lines
5475		4791
5476	static int intel_iommu_enable_iopf(struct device *dev)	4792	static int intel_iommu_enable_iopf(struct device *dev)
5477	{	4793	{
5478	struct device_domain_info *info = get_domain_info(dev);	4794	struct device_domain_info *info = dev_iommu_priv_get(dev);
5479		4795
5480	if (info && info->pri_supported)	4796	if (info && info->pri_supported)
5481	return 0;	4797	return 0;
@@ -5487,9 +4803,6 @@ static int · 5 unmodified lines
5487	intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)	4803	intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
5488	{	4804	{
5489	switch (feat) {	4805	switch (feat) {
5490	case IOMMU_DEV_FEAT_AUX:
5491	return intel_iommu_enable_auxd(dev);
5492
5493	case IOMMU_DEV_FEAT_IOPF:	4806	case IOMMU_DEV_FEAT_IOPF:
5494	return intel_iommu_enable_iopf(dev);	4807	return intel_iommu_enable_iopf(dev);
5495		4808
@@ -5505,9 +4818,6 @@ static int · 9 unmodified lines
5505	intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)	4818	intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
5506	{	4819	{
5507	switch (feat) {	4820	switch (feat) {
5508	case IOMMU_DEV_FEAT_AUX:
5509	return intel_iommu_disable_auxd(dev);
5510
5511	case IOMMU_DEV_FEAT_IOPF:	4821	case IOMMU_DEV_FEAT_IOPF:
5512	return 0;	4822	return 0;
5513		4823
@@ -5519,48 +4829,11 @@ intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) · 5 unmodified lines
5519	}	4829	}
5520	}	4830	}
5521		4831
5522	static bool	4832	static bool intel_iommu_is_attach_deferred(struct device *dev)
5523	intel_iommu_dev_feat_enabled(struct device *dev, enum iommu_dev_features feat)
5524	{
5525	struct device_domain_info *info = get_domain_info(dev);
5526
5527	if (feat == IOMMU_DEV_FEAT_AUX)
5528	return scalable_mode_support() && info && info->auxd_enabled;
5529
5530	return false;
5531	}
5532
5533	static int
5534	intel_iommu_aux_get_pasid(struct iommu_domain domain, struct device dev)
5535	{	4833	{
5536	struct dmar_domain *dmar_domain = to_dmar_domain(domain);	4834	struct device_domain_info *info = dev_iommu_priv_get(dev);
5537		4835
5538	return dmar_domain->default_pasid > 0 ?	4836	return translation_pre_enabled(info->iommu) && !info->domain;
5539	dmar_domain->default_pasid : -EINVAL;
5540	}
5541
5542	static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain,
5543	struct device *dev)
5544	{
5545	return attach_deferred(dev);
5546	}
5547
5548	static int
5549	intel_iommu_enable_nesting(struct iommu_domain *domain)
5550	{
5551	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5552	unsigned long flags;
5553	int ret = -ENODEV;
5554
5555	spin_lock_irqsave(&device_domain_lock, flags);
5556	if (list_empty(&dmar_domain->devices)) {
5557	dmar_domain->flags \|= DOMAIN_FLAG_NESTING_MODE;
5558	dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL;
5559	ret = 0;
5560	}
5561	spin_unlock_irqrestore(&device_domain_lock, flags);
5562
5563	return ret;
5564	}	4837	}
5565		4838
5566	/*	4839	/*
@@ -5598,40 +4871,34 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain, · 31 unmodified lines
5598	const struct iommu_ops intel_iommu_ops = {	4871	const struct iommu_ops intel_iommu_ops = {
5599	.capable = intel_iommu_capable,	4872	.capable = intel_iommu_capable,
5600	.domain_alloc = intel_iommu_domain_alloc,	4873	.domain_alloc = intel_iommu_domain_alloc,
5601	.domain_free = intel_iommu_domain_free,
5602	.enable_nesting = intel_iommu_enable_nesting,
5603	.attach_dev = intel_iommu_attach_device,
5604	.detach_dev = intel_iommu_detach_device,
5605	.aux_attach_dev = intel_iommu_aux_attach_device,
5606	.aux_detach_dev = intel_iommu_aux_detach_device,
5607	.aux_get_pasid = intel_iommu_aux_get_pasid,
5608	.map_pages = intel_iommu_map_pages,
5609	.unmap_pages = intel_iommu_unmap_pages,
5610	.iotlb_sync_map = intel_iommu_iotlb_sync_map,
5611	.flush_iotlb_all = intel_flush_iotlb_all,
5612	.iotlb_sync = intel_iommu_tlb_sync,
5613	.iova_to_phys = intel_iommu_iova_to_phys,
5614	.probe_device = intel_iommu_probe_device,	4874	.probe_device = intel_iommu_probe_device,
5615	.probe_finalize = intel_iommu_probe_finalize,	4875	.probe_finalize = intel_iommu_probe_finalize,
5616	.release_device = intel_iommu_release_device,	4876	.release_device = intel_iommu_release_device,
5617	.get_resv_regions = intel_iommu_get_resv_regions,	4877	.get_resv_regions = intel_iommu_get_resv_regions,
5618	.put_resv_regions = generic_iommu_put_resv_regions,	4878	.put_resv_regions = generic_iommu_put_resv_regions,
5619	.device_group = intel_iommu_device_group,	4879	.device_group = intel_iommu_device_group,
5620	.dev_feat_enabled = intel_iommu_dev_feat_enabled,
5621	.dev_enable_feat = intel_iommu_dev_enable_feat,	4880	.dev_enable_feat = intel_iommu_dev_enable_feat,
5622	.dev_disable_feat = intel_iommu_dev_disable_feat,	4881	.dev_disable_feat = intel_iommu_dev_disable_feat,
5623	.is_attach_deferred = intel_iommu_is_attach_deferred,	4882	.is_attach_deferred = intel_iommu_is_attach_deferred,
5624	.def_domain_type = device_def_domain_type,	4883	.def_domain_type = device_def_domain_type,
5625	.pgsize_bitmap = SZ_4K,	4884	.pgsize_bitmap = SZ_4K,
5626	#ifdef CONFIG_INTEL_IOMMU_SVM	4885	#ifdef CONFIG_INTEL_IOMMU_SVM
5627	.cache_invalidate = intel_iommu_sva_invalidate,
5628	.sva_bind_gpasid = intel_svm_bind_gpasid,
5629	.sva_unbind_gpasid = intel_svm_unbind_gpasid,
5630	.sva_bind = intel_svm_bind,	4886	.sva_bind = intel_svm_bind,
5631	.sva_unbind = intel_svm_unbind,	4887	.sva_unbind = intel_svm_unbind,
5632	.sva_get_pasid = intel_svm_get_pasid,	4888	.sva_get_pasid = intel_svm_get_pasid,
5633	.page_response = intel_svm_page_response,	4889	.page_response = intel_svm_page_response,
5634	#endif	4890	#endif
		4891	.default_domain_ops = &(const struct iommu_domain_ops) {
		4892	.attach_dev = intel_iommu_attach_device,
		4893	.detach_dev = intel_iommu_detach_device,
		4894	.map_pages = intel_iommu_map_pages,
		4895	.unmap_pages = intel_iommu_unmap_pages,
		4896	.iotlb_sync_map = intel_iommu_iotlb_sync_map,
		4897	.flush_iotlb_all = intel_flush_iotlb_all,
		4898	.iotlb_sync = intel_iommu_tlb_sync,
		4899	.iova_to_phys = intel_iommu_iova_to_phys,
		4900	.free = intel_iommu_domain_free,
		4901	}
5635	};	4902	};
5636		4903
5637	static void quirk_iommu_igfx(struct pci_dev *dev)	4904	static void quirk_iommu_igfx(struct pci_dev *dev)

drivers/iommu/intel/pasid.c+6−167

@@ -150,7 +150,7 @@ int intel_pasid_alloc_table(struct device *dev) · 149 unmodified lines
150	int size;	150	int size;
151		151
152	might_sleep();	152	might_sleep();
153	info = get_domain_info(dev);	153	info = dev_iommu_priv_get(dev);
154	if (WARN_ON(!info \|\| !dev_is_pci(dev) \|\| info->pasid_table))	154	if (WARN_ON(!info \|\| !dev_is_pci(dev) \|\| info->pasid_table))
155	return -EINVAL;	155	return -EINVAL;
156		156
@@ -197,7 +197,7 @@ void intel_pasid_free_table(struct device *dev) · 40 unmodified lines
197	struct pasid_entry *table;	197	struct pasid_entry *table;
198	int i, max_pde;	198	int i, max_pde;
199		199
200	info = get_domain_info(dev);	200	info = dev_iommu_priv_get(dev);
201	if (!info \|\| !dev_is_pci(dev) \|\| !info->pasid_table)	201	if (!info \|\| !dev_is_pci(dev) \|\| !info->pasid_table)
202	return;	202	return;
203		203
@@ -223,7 +223,7 @@ struct pasid_table intel_pasid_get_table(struct device dev) · 19 unmodified lines
223	{	223	{
224	struct device_domain_info *info;	224	struct device_domain_info *info;
225		225
226	info = get_domain_info(dev);	226	info = dev_iommu_priv_get(dev);
227	if (!info)	227	if (!info)
228	return NULL;	228	return NULL;
229		229
@@ -234,7 +234,7 @@ static int intel_pasid_get_dev_max_id(struct device *dev) · 4 unmodified lines
234	{	234	{
235	struct device_domain_info *info;	235	struct device_domain_info *info;
236		236
237	info = get_domain_info(dev);	237	info = dev_iommu_priv_get(dev);
238	if (!info \|\| !info->pasid_table)	238	if (!info \|\| !info->pasid_table)
239	return 0;	239	return 0;
240		240
@@ -254,7 +254,7 @@ static struct pasid_entry intel_pasid_get_entry(struct device dev, u32 pasid) · 13 unmodified lines
254	return NULL;	254	return NULL;
255		255
256	dir = pasid_table->table;	256	dir = pasid_table->table;
257	info = get_domain_info(dev);	257	info = dev_iommu_priv_get(dev);
258	dir_index = pasid >> PASID_PDE_SHIFT;	258	dir_index = pasid >> PASID_PDE_SHIFT;
259	index = pasid & PASID_PTE_MASK;	259	index = pasid & PASID_PTE_MASK;
260		260
@@ -487,7 +487,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, · 226 unmodified lines
487	struct device_domain_info *info;	487	struct device_domain_info *info;
488	u16 sid, qdep, pfsid;	488	u16 sid, qdep, pfsid;
489		489
490	info = get_domain_info(dev);	490	info = dev_iommu_priv_get(dev);
491	if (!info \|\| !info->ats_enabled)	491	if (!info \|\| !info->ats_enabled)
492	return;	492	return;
493		493
@@ -762,164 +762,3 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, · 268 unmodified lines
762		762
763	return 0;	763	return 0;
764	}	764	}
765
766	static int
767	intel_pasid_setup_bind_data(struct intel_iommu iommu, struct pasid_entry pte,
768	struct iommu_gpasid_bind_data_vtd *pasid_data)
769	{
770	/*
771	* Not all guest PASID table entry fields are passed down during bind,
772	* here we only set up the ones that are dependent on guest settings.
773	* Execution related bits such as NXE, SMEP are not supported.
774	* Other fields, such as snoop related, are set based on host needs
775	* regardless of guest settings.
776	*/
777	if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_SRE) {
778	if (!ecap_srs(iommu->ecap)) {
779	pr_err_ratelimited("No supervisor request support on %s\n",
780	iommu->name);
781	return -EINVAL;
782	}
783	pasid_set_sre(pte);
784	/* Enable write protect WP if guest requested */
785	if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_WPE)
786	pasid_set_wpe(pte);
787	}
788
789	if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_EAFE) {
790	if (!ecap_eafs(iommu->ecap)) {
791	pr_err_ratelimited("No extended access flag support on %s\n",
792	iommu->name);
793	return -EINVAL;
794	}
795	pasid_set_eafe(pte);
796	}
797
798	/*
799	* Memory type is only applicable to devices inside processor coherent
800	* domain. Will add MTS support once coherent devices are available.
801	*/
802	if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_MTS_MASK) {
803	pr_warn_ratelimited("No memory type support %s\n",
804	iommu->name);
805	return -EINVAL;
806	}
807
808	return 0;
809	}
810
811	/**
812	* intel_pasid_setup_nested() - Set up PASID entry for nested translation.
813	* This could be used for guest shared virtual address. In this case, the
814	* first level page tables are used for GVA-GPA translation in the guest,
815	* second level page tables are used for GPA-HPA translation.
816	*
817	* @iommu: IOMMU which the device belong to
818	* @dev: Device to be set up for translation
819	* @gpgd: FLPTPTR: First Level Page translation pointer in GPA
820	* @pasid: PASID to be programmed in the device PASID table
821	* @pasid_data: Additional PASID info from the guest bind request
822	* @domain: Domain info for setting up second level page tables
823	* @addr_width: Address width of the first level (guest)
824	*/
825	int intel_pasid_setup_nested(struct intel_iommu iommu, struct device dev,
826	pgd_t *gpgd, u32 pasid,
827	struct iommu_gpasid_bind_data_vtd *pasid_data,
828	struct dmar_domain *domain, int addr_width)
829	{
830	struct pasid_entry *pte;
831	struct dma_pte *pgd;
832	int ret = 0;
833	u64 pgd_val;
834	int agaw;
835	u16 did;
836
837	if (!ecap_nest(iommu->ecap)) {
838	pr_err_ratelimited("IOMMU: %s: No nested translation support\n",
839	iommu->name);
840	return -EINVAL;
841	}
842
843	if (!(domain->flags & DOMAIN_FLAG_NESTING_MODE)) {
844	pr_err_ratelimited("Domain is not in nesting mode, %x\n",
845	domain->flags);
846	return -EINVAL;
847	}
848
849	pte = intel_pasid_get_entry(dev, pasid);
850	if (WARN_ON(!pte))
851	return -EINVAL;
852
853	/*
854	* Caller must ensure PASID entry is not in use, i.e. not bind the
855	* same PASID to the same device twice.
856	*/
857	if (pasid_pte_is_present(pte))
858	return -EBUSY;
859
860	pasid_clear_entry(pte);
861
862	/* Sanity checking performed by caller to make sure address
863	* width matching in two dimensions:
864	* 1. CPU vs. IOMMU
865	* 2. Guest vs. Host.
866	*/
867	switch (addr_width) {
868	#ifdef CONFIG_X86
869	case ADDR_WIDTH_5LEVEL:
870	if (!cpu_feature_enabled(X86_FEATURE_LA57) \|\|
871	!cap_5lp_support(iommu->cap)) {
872	dev_err_ratelimited(dev,
873	"5-level paging not supported\n");
874	return -EINVAL;
875	}
876
877	pasid_set_flpm(pte, 1);
878	break;
879	#endif
880	case ADDR_WIDTH_4LEVEL:
881	pasid_set_flpm(pte, 0);
882	break;
883	default:
884	dev_err_ratelimited(dev, "Invalid guest address width %d\n",
885	addr_width);
886	return -EINVAL;
887	}
888
889	/* First level PGD is in GPA, must be supported by the second level */
890	if ((uintptr_t)gpgd > domain->max_addr) {
891	dev_err_ratelimited(dev,
892	"Guest PGD %lx not supported, max %llx\n",
893	(uintptr_t)gpgd, domain->max_addr);
894	return -EINVAL;
895	}
896	pasid_set_flptr(pte, (uintptr_t)gpgd);
897
898	ret = intel_pasid_setup_bind_data(iommu, pte, pasid_data);
899	if (ret)
900	return ret;
901
902	/* Setup the second level based on the given domain */
903	pgd = domain->pgd;
904
905	agaw = iommu_skip_agaw(domain, iommu, &pgd);
906	if (agaw < 0) {
907	dev_err_ratelimited(dev, "Invalid domain page table\n");
908	return -EINVAL;
909	}
910	pgd_val = virt_to_phys(pgd);
911	pasid_set_slptr(pte, pgd_val);
912	pasid_set_fault_enable(pte);
913
914	did = domain->iommu_did[iommu->seq_id];
915	pasid_set_domain_id(pte, did);
916
917	pasid_set_address_width(pte, agaw);
918	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
919
920	pasid_set_translation_type(pte, PASID_ENTRY_PGTT_NESTED);
921	pasid_set_present(pte);
922	pasid_flush_caches(iommu, pte, pasid, did);
923
924	return ret;
925	}

drivers/iommu/intel/pasid.h+0−4

 int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
 				   struct dmar_domain *domain,
 				   struct device *dev, u32 pasid);
-int intel_pasid_setup_nested(struct intel_iommu *iommu,
-			     struct device *dev, pgd_t *pgd, u32 pasid,
-			     struct iommu_gpasid_bind_data_vtd *pasid_data,
-			     struct dmar_domain *domain, int addr_width);
 void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
 				 struct device *dev, u32 pasid,
 				 bool fault_ignore);

drivers/iommu/intel/svm.c+3−217

@@ -168,11 +168,6 @@ int intel_svm_finish_prq(struct intel_iommu *iommu) · 167 unmodified lines
168	return 0;	168	return 0;
169	}	169	}
170		170
171	static inline bool intel_svm_capable(struct intel_iommu *iommu)
172	{
173	return iommu->flags & VTD_FLAG_SVM_CAPABLE;
174	}
175
176	void intel_svm_check(struct intel_iommu *iommu)	171	void intel_svm_check(struct intel_iommu *iommu)
177	{	172	{
178	if (!pasid_supported(iommu))	173	if (!pasid_supported(iommu))
@@ -200,7 +195,7 @@ static void __flush_svm_range_dev(struct intel_svm *svm, · 21 unmodified lines
200	unsigned long address,	195	unsigned long address,
201	unsigned long pages, int ih)	196	unsigned long pages, int ih)
202	{	197	{
203	struct device_domain_info *info = get_domain_info(sdev->dev);	198	struct device_domain_info *info = dev_iommu_priv_get(sdev->dev);
204		199
205	if (WARN_ON(!pages))	200	if (WARN_ON(!pages))
206	return;	201	return;
@@ -318,193 +313,6 @@ out: · 111 unmodified lines
318	return 0;	313	return 0;
319	}	314	}
320		315
321	int intel_svm_bind_gpasid(struct iommu_domain domain, struct device dev,
322	struct iommu_gpasid_bind_data *data)
323	{
324	struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
325	struct intel_svm_dev *sdev = NULL;
326	struct dmar_domain *dmar_domain;
327	struct device_domain_info *info;
328	struct intel_svm *svm = NULL;
329	unsigned long iflags;
330	int ret = 0;
331
332	if (WARN_ON(!iommu) \|\| !data)
333	return -EINVAL;
334
335	if (data->format != IOMMU_PASID_FORMAT_INTEL_VTD)
336	return -EINVAL;
337
338	/* IOMMU core ensures argsz is more than the start of the union */
339	if (data->argsz < offsetofend(struct iommu_gpasid_bind_data, vendor.vtd))
340	return -EINVAL;
341
342	/* Make sure no undefined flags are used in vendor data */
343	if (data->vendor.vtd.flags & ~(IOMMU_SVA_VTD_GPASID_LAST - 1))
344	return -EINVAL;
345
346	if (!dev_is_pci(dev))
347	return -ENOTSUPP;
348
349	/* VT-d supports devices with full 20 bit PASIDs only */
350	if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX)
351	return -EINVAL;
352
353	/*
354	* We only check host PASID range, we have no knowledge to check
355	* guest PASID range.
356	*/
357	if (data->hpasid <= 0 \|\| data->hpasid >= PASID_MAX)
358	return -EINVAL;
359
360	info = get_domain_info(dev);
361	if (!info)
362	return -EINVAL;
363
364	dmar_domain = to_dmar_domain(domain);
365
366	mutex_lock(&pasid_mutex);
367	ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev);
368	if (ret)
369	goto out;
370
371	if (sdev) {
372	/*
373	* Do not allow multiple bindings of the same device-PASID since
374	* there is only one SL page tables per PASID. We may revisit
375	* once sharing PGD across domains are supported.
376	*/
377	dev_warn_ratelimited(dev, "Already bound with PASID %u\n",
378	svm->pasid);
379	ret = -EBUSY;
380	goto out;
381	}
382
383	if (!svm) {
384	/* We come here when PASID has never been bond to a device. */
385	svm = kzalloc(sizeof(*svm), GFP_KERNEL);
386	if (!svm) {
387	ret = -ENOMEM;
388	goto out;
389	}
390	/* REVISIT: upper layer/VFIO can track host process that bind
391	* the PASID. ioasid_set = mm might be sufficient for vfio to
392	* check pasid VMM ownership. We can drop the following line
393	* once VFIO and IOASID set check is in place.
394	*/
395	svm->mm = get_task_mm(current);
396	svm->pasid = data->hpasid;
397	if (data->flags & IOMMU_SVA_GPASID_VAL) {
398	svm->gpasid = data->gpasid;
399	svm->flags \|= SVM_FLAG_GUEST_PASID;
400	}
401	pasid_private_add(data->hpasid, svm);
402	INIT_LIST_HEAD_RCU(&svm->devs);
403	mmput(svm->mm);
404	}
405	sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
406	if (!sdev) {
407	ret = -ENOMEM;
408	goto out;
409	}
410	sdev->dev = dev;
411	sdev->sid = PCI_DEVID(info->bus, info->devfn);
412	sdev->iommu = iommu;
413
414	/* Only count users if device has aux domains */
415	if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
416	sdev->users = 1;
417
418	/* Set up device context entry for PASID if not enabled already */
419	ret = intel_iommu_enable_pasid(iommu, sdev->dev);
420	if (ret) {
421	dev_err_ratelimited(dev, "Failed to enable PASID capability\n");
422	kfree(sdev);
423	goto out;
424	}
425
426	/*
427	* PASID table is per device for better security. Therefore, for
428	* each bind of a new device even with an existing PASID, we need to
429	* call the nested mode setup function here.
430	*/
431	spin_lock_irqsave(&iommu->lock, iflags);
432	ret = intel_pasid_setup_nested(iommu, dev,
433	(pgd_t *)(uintptr_t)data->gpgd,
434	data->hpasid, &data->vendor.vtd, dmar_domain,
435	data->addr_width);
436	spin_unlock_irqrestore(&iommu->lock, iflags);
437	if (ret) {
438	dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n",
439	data->hpasid, ret);
440	/*
441	* PASID entry should be in cleared state if nested mode
442	* set up failed. So we only need to clear IOASID tracking
443	* data such that free call will succeed.
444	*/
445	kfree(sdev);
446	goto out;
447	}
448
449	svm->flags \|= SVM_FLAG_GUEST_MODE;
450
451	init_rcu_head(&sdev->rcu);
452	list_add_rcu(&sdev->list, &svm->devs);
453	out:
454	if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) {
455	pasid_private_remove(data->hpasid);
456	kfree(svm);
457	}
458
459	mutex_unlock(&pasid_mutex);
460	return ret;
461	}
462
463	int intel_svm_unbind_gpasid(struct device *dev, u32 pasid)
464	{
465	struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
466	struct intel_svm_dev *sdev;
467	struct intel_svm *svm;
468	int ret;
469
470	if (WARN_ON(!iommu))
471	return -EINVAL;
472
473	mutex_lock(&pasid_mutex);
474	ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);
475	if (ret)
476	goto out;
477
478	if (sdev) {
479	if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
480	sdev->users--;
481	if (!sdev->users) {
482	list_del_rcu(&sdev->list);
483	intel_pasid_tear_down_entry(iommu, dev,
484	svm->pasid, false);
485	intel_svm_drain_prq(dev, svm->pasid);
486	kfree_rcu(sdev, rcu);
487
488	if (list_empty(&svm->devs)) {
489	/*
490	* We do not free the IOASID here in that
491	* IOMMU driver did not allocate it.
492	* Unlike native SVM, IOASID for guest use was
493	* allocated prior to the bind call.
494	* In any case, if the free call comes before
495	* the unbind, IOMMU driver will get notified
496	* and perform cleanup.
497	*/
498	pasid_private_remove(pasid);
499	kfree(svm);
500	}
501	}
502	}
503	out:
504	mutex_unlock(&pasid_mutex);
505	return ret;
506	}
507
508	static int intel_svm_alloc_pasid(struct device dev, struct mm_struct mm,	316	static int intel_svm_alloc_pasid(struct device dev, struct mm_struct mm,
509	unsigned int flags)	317	unsigned int flags)
510	{	318	{
@@ -519,7 +327,7 @@ static struct iommu_sva intel_svm_bind_mm(struct intel_iommu iommu, · 8 unmodified lines
519	struct mm_struct *mm,	327	struct mm_struct *mm,
520	unsigned int flags)	328	unsigned int flags)
521	{	329	{
522	struct device_domain_info *info = get_domain_info(dev);	330	struct device_domain_info *info = dev_iommu_priv_get(dev);
523	unsigned long iflags, sflags;	331	unsigned long iflags, sflags;
524	struct intel_svm_dev *sdev;	332	struct intel_svm_dev *sdev;
525	struct intel_svm *svm;	333	struct intel_svm *svm;
@@ -725,7 +533,7 @@ static void intel_svm_drain_prq(struct device *dev, u32 pasid) · 199 unmodified lines
725	u16 sid, did;	533	u16 sid, did;
726	int qdep;	534	int qdep;
727		535
728	info = get_domain_info(dev);	536	info = dev_iommu_priv_get(dev);
729	if (WARN_ON(!info \|\| !dev_is_pci(dev)))	537	if (WARN_ON(!info \|\| !dev_is_pci(dev)))
730	return;	538	return;
731		539
@@ -1116,28 +924,6 @@ int intel_svm_page_response(struct device *dev, · 384 unmodified lines
1116	goto out;	924	goto out;
1117	}	925	}
1118		926
1119	/*
1120	* For responses from userspace, need to make sure that the
1121	* pasid has been bound to its mm.
1122	*/
1123	if (svm->flags & SVM_FLAG_GUEST_MODE) {
1124	struct mm_struct *mm;
1125
1126	mm = get_task_mm(current);
1127	if (!mm) {
1128	ret = -EINVAL;
1129	goto out;
1130	}
1131
1132	if (mm != svm->mm) {
1133	ret = -ENODEV;
1134	mmput(mm);
1135	goto out;
1136	}
1137
1138	mmput(mm);
1139	}
1140
1141	/*	927	/*
1142	* Per VT-d spec. v3.0 ch7.7, system software must respond	928	* Per VT-d spec. v3.0 ch7.7, system software must respond
1143	* with page group response if private data is present (PDP)	929	* with page group response if private data is present (PDP)

drivers/iommu/iommu.c+39−300

@@ -323,13 +323,14 @@ err_out: · 322 unmodified lines
323		323
324	void iommu_release_device(struct device *dev)	324	void iommu_release_device(struct device *dev)
325	{	325	{
326	const struct iommu_ops *ops = dev->bus->iommu_ops;	326	const struct iommu_ops *ops;
327		327
328	if (!dev->iommu)	328	if (!dev->iommu)
329	return;	329	return;
330		330
331	iommu_device_unlink(dev->iommu->iommu_dev, dev);	331	iommu_device_unlink(dev->iommu->iommu_dev, dev);
332		332
		333	ops = dev_iommu_ops(dev);
333	ops->release_device(dev);	334	ops->release_device(dev);
334		335
335	iommu_group_remove_device(dev);	336	iommu_group_remove_device(dev);
@@ -790,9 +791,6 @@ static int iommu_create_device_direct_mappings(struct iommu_group *group, · 454 unmodified lines
790	dma_addr_t start, end, addr;	791	dma_addr_t start, end, addr;
791	size_t map_size = 0;	792	size_t map_size = 0;
792		793
793	if (domain->ops->apply_resv_region)
794	domain->ops->apply_resv_region(dev, domain, entry);
795
796	start = ALIGN(entry->start, pg_size);	794	start = ALIGN(entry->start, pg_size);
797	end = ALIGN(entry->start + entry->length, pg_size);	795	end = ALIGN(entry->start + entry->length, pg_size);
798		796
@@ -833,11 +831,12 @@ out: · 34 unmodified lines
833	return ret;	831	return ret;
834	}	832	}
835		833
836	static bool iommu_is_attach_deferred(struct iommu_domain *domain,	834	static bool iommu_is_attach_deferred(struct device *dev)
837	struct device *dev)
838	{	835	{
839	if (domain->ops->is_attach_deferred)	836	const struct iommu_ops *ops = dev_iommu_ops(dev);
840	return domain->ops->is_attach_deferred(domain, dev);	837
		838	if (ops->is_attach_deferred)
		839	return ops->is_attach_deferred(dev);
841		840
842	return false;	841	return false;
843	}	842	}
@@ -894,7 +893,7 @@ rename: · 50 unmodified lines
894		893
895	mutex_lock(&group->mutex);	894	mutex_lock(&group->mutex);
896	list_add_tail(&device->list, &group->devices);	895	list_add_tail(&device->list, &group->devices);
897	if (group->domain && !iommu_is_attach_deferred(group->domain, dev))	896	if (group->domain && !iommu_is_attach_deferred(dev))
898	ret = __iommu_attach_device(group->domain, dev);	897	ret = __iommu_attach_device(group->domain, dev);
899	mutex_unlock(&group->mutex);	898	mutex_unlock(&group->mutex);
900	if (ret)	899	if (ret)
@@ -1255,10 +1254,10 @@ int iommu_page_response(struct device *dev, · 354 unmodified lines
1255	struct iommu_fault_event *evt;	1254	struct iommu_fault_event *evt;
1256	struct iommu_fault_page_request *prm;	1255	struct iommu_fault_page_request *prm;
1257	struct dev_iommu *param = dev->iommu;	1256	struct dev_iommu *param = dev->iommu;
		1257	const struct iommu_ops *ops = dev_iommu_ops(dev);
1258	bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID;	1258	bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID;
1259	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1260		1259
1261	if (!domain \|\| !domain->ops->page_response)	1260	if (!ops->page_response)
1262	return -ENODEV;	1261	return -ENODEV;
1263		1262
1264	if (!param \|\| !param->fault_param)	1263	if (!param \|\| !param->fault_param)
@@ -1299,7 +1298,7 @@ int iommu_page_response(struct device *dev, · 34 unmodified lines
1299	msg->pasid = 0;	1298	msg->pasid = 0;
1300	}	1299	}
1301		1300
1302	ret = domain->ops->page_response(dev, evt, msg);	1301	ret = ops->page_response(dev, evt, msg);
1303	list_del(&evt->list);	1302	list_del(&evt->list);
1304	kfree(evt);	1303	kfree(evt);
1305	break;	1304	break;
@@ -1524,7 +1523,7 @@ EXPORT_SYMBOL_GPL(fsl_mc_device_group); · 218 unmodified lines
1524		1523
1525	static int iommu_get_def_domain_type(struct device *dev)	1524	static int iommu_get_def_domain_type(struct device *dev)
1526	{	1525	{
1527	const struct iommu_ops *ops = dev->bus->iommu_ops;	1526	const struct iommu_ops *ops = dev_iommu_ops(dev);
1528		1527
1529	if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)	1528	if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
1530	return IOMMU_DOMAIN_DMA;	1529	return IOMMU_DOMAIN_DMA;
@@ -1583,7 +1582,7 @@ static int iommu_alloc_default_domain(struct iommu_group *group, · 52 unmodified lines
1583	*/	1582	*/
1584	static struct iommu_group iommu_group_get_for_dev(struct device dev)	1583	static struct iommu_group iommu_group_get_for_dev(struct device dev)
1585	{	1584	{
1586	const struct iommu_ops *ops = dev->bus->iommu_ops;	1585	const struct iommu_ops *ops = dev_iommu_ops(dev);
1587	struct iommu_group *group;	1586	struct iommu_group *group;
1588	int ret;	1587	int ret;
1589		1588
@@ -1591,9 +1590,6 @@ static struct iommu_group iommu_group_get_for_dev(struct device dev) · 1 unmodified line
1591	if (group)	1590	if (group)
1592	return group;	1591	return group;
1593		1592
1594	if (!ops)
1595	return ERR_PTR(-EINVAL);
1596
1597	group = ops->device_group(dev);	1593	group = ops->device_group(dev);
1598	if (WARN_ON_ONCE(group == NULL))	1594	if (WARN_ON_ONCE(group == NULL))
1599	return ERR_PTR(-EINVAL);	1595	return ERR_PTR(-EINVAL);
@@ -1748,7 +1744,7 @@ static int iommu_group_do_dma_attach(struct device dev, void data) · 148 unmodified lines
1748	struct iommu_domain *domain = data;	1744	struct iommu_domain *domain = data;
1749	int ret = 0;	1745	int ret = 0;
1750		1746
1751	if (!iommu_is_attach_deferred(domain, dev))	1747	if (!iommu_is_attach_deferred(dev))
1752	ret = __iommu_attach_device(domain, dev);	1748	ret = __iommu_attach_device(domain, dev);
1753		1749
1754	return ret;	1750	return ret;
@@ -1762,10 +1758,10 @@ static int __iommu_group_dma_attach(struct iommu_group *group) · 7 unmodified lines
1762		1758
1763	static int iommu_group_do_probe_finalize(struct device dev, void data)	1759	static int iommu_group_do_probe_finalize(struct device dev, void data)
1764	{	1760	{
1765	struct iommu_domain *domain = data;	1761	const struct iommu_ops *ops = dev_iommu_ops(dev);
1766		1762
1767	if (domain->ops->probe_finalize)	1763	if (ops->probe_finalize)
1768	domain->ops->probe_finalize(dev);	1764	ops->probe_finalize(dev);
1769		1765
1770	return 0;	1766	return 0;
1771	}	1767	}
@@ -1954,10 +1950,11 @@ static struct iommu_domain __iommu_domain_alloc(struct bus_type bus, · 182 unmodified lines
1954	if (!domain)	1950	if (!domain)
1955	return NULL;	1951	return NULL;
1956		1952
1957	domain->ops = bus->iommu_ops;
1958	domain->type = type;	1953	domain->type = type;
1959	/* Assume all sizes by default; the driver may override this later */	1954	/* Assume all sizes by default; the driver may override this later */
1960	domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;	1955	domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
		1956	if (!domain->ops)
		1957	domain->ops = bus->iommu_ops->default_domain_ops;
1961		1958
1962	if (iommu_is_dma_domain(domain) && iommu_get_dma_cookie(domain)) {	1959	if (iommu_is_dma_domain(domain) && iommu_get_dma_cookie(domain)) {
1963	iommu_domain_free(domain);	1960	iommu_domain_free(domain);
@@ -1975,7 +1972,7 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc); · 11 unmodified lines
1975	void iommu_domain_free(struct iommu_domain *domain)	1972	void iommu_domain_free(struct iommu_domain *domain)
1976	{	1973	{
1977	iommu_put_dma_cookie(domain);	1974	iommu_put_dma_cookie(domain);
1978	domain->ops->domain_free(domain);	1975	domain->ops->free(domain);
1979	}	1976	}
1980	EXPORT_SYMBOL_GPL(iommu_domain_free);	1977	EXPORT_SYMBOL_GPL(iommu_domain_free);
1981		1978
@@ -2023,228 +2020,16 @@ EXPORT_SYMBOL_GPL(iommu_attach_device); · 41 unmodified lines
2023		2020
2024	int iommu_deferred_attach(struct device dev, struct iommu_domain domain)	2021	int iommu_deferred_attach(struct device dev, struct iommu_domain domain)
2025	{	2022	{
2026	const struct iommu_ops *ops = domain->ops;	2023	if (iommu_is_attach_deferred(dev))
2027
2028	if (ops->is_attach_deferred && ops->is_attach_deferred(domain, dev))
2029	return __iommu_attach_device(domain, dev);	2024	return __iommu_attach_device(domain, dev);
2030		2025
2031	return 0;	2026	return 0;
2032	}	2027	}
2033		2028
2034	/*
2035	* Check flags and other user provided data for valid combinations. We also
2036	* make sure no reserved fields or unused flags are set. This is to ensure
2037	* not breaking userspace in the future when these fields or flags are used.
2038	*/
2039	static int iommu_check_cache_invl_data(struct iommu_cache_invalidate_info *info)
2040	{
2041	u32 mask;
2042	int i;
2043
2044	if (info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1)
2045	return -EINVAL;
2046
2047	mask = (1 << IOMMU_CACHE_INV_TYPE_NR) - 1;
2048	if (info->cache & ~mask)
2049	return -EINVAL;
2050
2051	if (info->granularity >= IOMMU_INV_GRANU_NR)
2052	return -EINVAL;
2053
2054	switch (info->granularity) {
2055	case IOMMU_INV_GRANU_ADDR:
2056	if (info->cache & IOMMU_CACHE_INV_TYPE_PASID)
2057	return -EINVAL;
2058
2059	mask = IOMMU_INV_ADDR_FLAGS_PASID \|
2060	IOMMU_INV_ADDR_FLAGS_ARCHID \|
2061	IOMMU_INV_ADDR_FLAGS_LEAF;
2062
2063	if (info->granu.addr_info.flags & ~mask)
2064	return -EINVAL;
2065	break;
2066	case IOMMU_INV_GRANU_PASID:
2067	mask = IOMMU_INV_PASID_FLAGS_PASID \|
2068	IOMMU_INV_PASID_FLAGS_ARCHID;
2069	if (info->granu.pasid_info.flags & ~mask)
2070	return -EINVAL;
2071
2072	break;
2073	case IOMMU_INV_GRANU_DOMAIN:
2074	if (info->cache & IOMMU_CACHE_INV_TYPE_DEV_IOTLB)
2075	return -EINVAL;
2076	break;
2077	default:
2078	return -EINVAL;
2079	}
2080
2081	/* Check reserved padding fields */
2082	for (i = 0; i < sizeof(info->padding); i++) {
2083	if (info->padding[i])
2084	return -EINVAL;
2085	}
2086
2087	return 0;
2088	}
2089
2090	int iommu_uapi_cache_invalidate(struct iommu_domain domain, struct device dev,
2091	void __user *uinfo)
2092	{
2093	struct iommu_cache_invalidate_info inv_info = { 0 };
2094	u32 minsz;
2095	int ret;
2096
2097	if (unlikely(!domain->ops->cache_invalidate))
2098	return -ENODEV;
2099
2100	/*
2101	* No new spaces can be added before the variable sized union, the
2102	* minimum size is the offset to the union.
2103	*/
2104	minsz = offsetof(struct iommu_cache_invalidate_info, granu);
2105
2106	/* Copy minsz from user to get flags and argsz */
2107	if (copy_from_user(&inv_info, uinfo, minsz))
2108	return -EFAULT;
2109
2110	/* Fields before the variable size union are mandatory */
2111	if (inv_info.argsz < minsz)
2112	return -EINVAL;
2113
2114	/* PASID and address granu require additional info beyond minsz */
2115	if (inv_info.granularity == IOMMU_INV_GRANU_PASID &&
2116	inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.pasid_info))
2117	return -EINVAL;
2118
2119	if (inv_info.granularity == IOMMU_INV_GRANU_ADDR &&
2120	inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.addr_info))
2121	return -EINVAL;
2122
2123	/*
2124	* User might be using a newer UAPI header which has a larger data
2125	* size, we shall support the existing flags within the current
2126	* size. Copy the remaining user data _after_ minsz but not more
2127	* than the current kernel supported size.
2128	*/
2129	if (copy_from_user((void *)&inv_info + minsz, uinfo + minsz,
2130	min_t(u32, inv_info.argsz, sizeof(inv_info)) - minsz))
2131	return -EFAULT;
2132
2133	/* Now the argsz is validated, check the content */
2134	ret = iommu_check_cache_invl_data(&inv_info);
2135	if (ret)
2136	return ret;
2137
2138	return domain->ops->cache_invalidate(domain, dev, &inv_info);
2139	}
2140	EXPORT_SYMBOL_GPL(iommu_uapi_cache_invalidate);
2141
2142	static int iommu_check_bind_data(struct iommu_gpasid_bind_data *data)
2143	{
2144	u64 mask;
2145	int i;
2146
2147	if (data->version != IOMMU_GPASID_BIND_VERSION_1)
2148	return -EINVAL;
2149
2150	/* Check the range of supported formats */
2151	if (data->format >= IOMMU_PASID_FORMAT_LAST)
2152	return -EINVAL;
2153
2154	/* Check all flags */
2155	mask = IOMMU_SVA_GPASID_VAL;
2156	if (data->flags & ~mask)
2157	return -EINVAL;
2158
2159	/* Check reserved padding fields */
2160	for (i = 0; i < sizeof(data->padding); i++) {
2161	if (data->padding[i])
2162	return -EINVAL;
2163	}
2164
2165	return 0;
2166	}
2167
2168	static int iommu_sva_prepare_bind_data(void __user *udata,
2169	struct iommu_gpasid_bind_data *data)
2170	{
2171	u32 minsz;
2172
2173	/*
2174	* No new spaces can be added before the variable sized union, the
2175	* minimum size is the offset to the union.
2176	*/
2177	minsz = offsetof(struct iommu_gpasid_bind_data, vendor);
2178
2179	/* Copy minsz from user to get flags and argsz */
2180	if (copy_from_user(data, udata, minsz))
2181	return -EFAULT;
2182
2183	/* Fields before the variable size union are mandatory */
2184	if (data->argsz < minsz)
2185	return -EINVAL;
2186	/*
2187	* User might be using a newer UAPI header, we shall let IOMMU vendor
2188	* driver decide on what size it needs. Since the guest PASID bind data
2189	* can be vendor specific, larger argsz could be the result of extension
2190	* for one vendor but it should not affect another vendor.
2191	* Copy the remaining user data _after_ minsz
2192	*/
2193	if (copy_from_user((void *)data + minsz, udata + minsz,
2194	min_t(u32, data->argsz, sizeof(*data)) - minsz))
2195	return -EFAULT;
2196
2197	return iommu_check_bind_data(data);
2198	}
2199
2200	int iommu_uapi_sva_bind_gpasid(struct iommu_domain domain, struct device dev,
2201	void __user *udata)
2202	{
2203	struct iommu_gpasid_bind_data data = { 0 };
2204	int ret;
2205
2206	if (unlikely(!domain->ops->sva_bind_gpasid))
2207	return -ENODEV;
2208
2209	ret = iommu_sva_prepare_bind_data(udata, &data);
2210	if (ret)
2211	return ret;
2212
2213	return domain->ops->sva_bind_gpasid(domain, dev, &data);
2214	}
2215	EXPORT_SYMBOL_GPL(iommu_uapi_sva_bind_gpasid);
2216
2217	int iommu_sva_unbind_gpasid(struct iommu_domain domain, struct device dev,
2218	ioasid_t pasid)
2219	{
2220	if (unlikely(!domain->ops->sva_unbind_gpasid))
2221	return -ENODEV;
2222
2223	return domain->ops->sva_unbind_gpasid(dev, pasid);
2224	}
2225	EXPORT_SYMBOL_GPL(iommu_sva_unbind_gpasid);
2226
2227	int iommu_uapi_sva_unbind_gpasid(struct iommu_domain domain, struct device dev,
2228	void __user *udata)
2229	{
2230	struct iommu_gpasid_bind_data data = { 0 };
2231	int ret;
2232
2233	if (unlikely(!domain->ops->sva_bind_gpasid))
2234	return -ENODEV;
2235
2236	ret = iommu_sva_prepare_bind_data(udata, &data);
2237	if (ret)
2238	return ret;
2239
2240	return iommu_sva_unbind_gpasid(domain, dev, data.hpasid);
2241	}
2242	EXPORT_SYMBOL_GPL(iommu_uapi_sva_unbind_gpasid);
2243
2244	static void __iommu_detach_device(struct iommu_domain *domain,	2029	static void __iommu_detach_device(struct iommu_domain *domain,
2245	struct device *dev)	2030	struct device *dev)
2246	{	2031	{
2247	if (iommu_is_attach_deferred(domain, dev))	2032	if (iommu_is_attach_deferred(dev))
2248	return;	2033	return;
2249		2034
2250	if (unlikely(domain->ops->detach_dev == NULL))	2035	if (unlikely(domain->ops->detach_dev == NULL))
@@ -2458,7 +2243,7 @@ static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova, · 207 unmodified lines
2458	phys_addr_t paddr, size_t size, int prot,	2243	phys_addr_t paddr, size_t size, int prot,
2459	gfp_t gfp, size_t *mapped)	2244	gfp_t gfp, size_t *mapped)
2460	{	2245	{
2461	const struct iommu_ops *ops = domain->ops;	2246	const struct iommu_domain_ops *ops = domain->ops;
2462	size_t pgsize, count;	2247	size_t pgsize, count;
2463	int ret;	2248	int ret;
2464		2249
@@ -2481,7 +2266,7 @@ static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova, · 16 unmodified lines
2481	static int __iommu_map(struct iommu_domain *domain, unsigned long iova,	2266	static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
2482	phys_addr_t paddr, size_t size, int prot, gfp_t gfp)	2267	phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2483	{	2268	{
2484	const struct iommu_ops *ops = domain->ops;	2269	const struct iommu_domain_ops *ops = domain->ops;
2485	unsigned long orig_iova = iova;	2270	unsigned long orig_iova = iova;
2486	unsigned int min_pagesz;	2271	unsigned int min_pagesz;
2487	size_t orig_size = size;	2272	size_t orig_size = size;
@@ -2541,7 +2326,7 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova, · 53 unmodified lines
2541	static int _iommu_map(struct iommu_domain *domain, unsigned long iova,	2326	static int _iommu_map(struct iommu_domain *domain, unsigned long iova,
2542	phys_addr_t paddr, size_t size, int prot, gfp_t gfp)	2327	phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2543	{	2328	{
2544	const struct iommu_ops *ops = domain->ops;	2329	const struct iommu_domain_ops *ops = domain->ops;
2545	int ret;	2330	int ret;
2546		2331
2547	ret = __iommu_map(domain, iova, paddr, size, prot, gfp);	2332	ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
@@ -2570,7 +2355,7 @@ static size_t __iommu_unmap_pages(struct iommu_domain *domain, · 22 unmodified lines
2570	unsigned long iova, size_t size,	2355	unsigned long iova, size_t size,
2571	struct iommu_iotlb_gather *iotlb_gather)	2356	struct iommu_iotlb_gather *iotlb_gather)
2572	{	2357	{
2573	const struct iommu_ops *ops = domain->ops;	2358	const struct iommu_domain_ops *ops = domain->ops;
2574	size_t pgsize, count;	2359	size_t pgsize, count;
2575		2360
2576	pgsize = iommu_pgsize(domain, iova, iova, size, &count);	2361	pgsize = iommu_pgsize(domain, iova, iova, size, &count);
@@ -2583,7 +2368,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain, · 6 unmodified lines
2583	unsigned long iova, size_t size,	2368	unsigned long iova, size_t size,
2584	struct iommu_iotlb_gather *iotlb_gather)	2369	struct iommu_iotlb_gather *iotlb_gather)
2585	{	2370	{
2586	const struct iommu_ops *ops = domain->ops;	2371	const struct iommu_domain_ops *ops = domain->ops;
2587	size_t unmapped_page, unmapped = 0;	2372	size_t unmapped_page, unmapped = 0;
2588	unsigned long orig_iova = iova;	2373	unsigned long orig_iova = iova;
2589	unsigned int min_pagesz;	2374	unsigned int min_pagesz;
@@ -2659,7 +2444,7 @@ static ssize_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, · 69 unmodified lines
2659	struct scatterlist *sg, unsigned int nents, int prot,	2444	struct scatterlist *sg, unsigned int nents, int prot,
2660	gfp_t gfp)	2445	gfp_t gfp)
2661	{	2446	{
2662	const struct iommu_ops *ops = domain->ops;	2447	const struct iommu_domain_ops *ops = domain->ops;
2663	size_t len = 0, mapped = 0;	2448	size_t len = 0, mapped = 0;
2664	phys_addr_t start;	2449	phys_addr_t start;
2665	unsigned int i = 0;	2450	unsigned int i = 0;
@@ -2792,17 +2577,17 @@ EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks); · 126 unmodified lines
2792		2577
2793	void iommu_get_resv_regions(struct device dev, struct list_head list)	2578	void iommu_get_resv_regions(struct device dev, struct list_head list)
2794	{	2579	{
2795	const struct iommu_ops *ops = dev->bus->iommu_ops;	2580	const struct iommu_ops *ops = dev_iommu_ops(dev);
2796		2581
2797	if (ops && ops->get_resv_regions)	2582	if (ops->get_resv_regions)
2798	ops->get_resv_regions(dev, list);	2583	ops->get_resv_regions(dev, list);
2799	}	2584	}
2800		2585
2801	void iommu_put_resv_regions(struct device dev, struct list_head list)	2586	void iommu_put_resv_regions(struct device dev, struct list_head list)
2802	{	2587	{
2803	const struct iommu_ops *ops = dev->bus->iommu_ops;	2588	const struct iommu_ops *ops = dev_iommu_ops(dev);
2804		2589
2805	if (ops && ops->put_resv_regions)	2590	if (ops->put_resv_regions)
2806	ops->put_resv_regions(dev, list);	2591	ops->put_resv_regions(dev, list);
2807	}	2592	}
2808		2593
@@ -2959,8 +2744,6 @@ EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); · 150 unmodified lines
2959		2744
2960	/*	2745	/*
2961	* The device drivers should do the necessary cleanups before calling this.	2746	* The device drivers should do the necessary cleanups before calling this.
2962	* For example, before disabling the aux-domain feature, the device driver
2963	* should detach all aux-domains. Otherwise, this will return -EBUSY.
2964	*/	2747	*/
2965	int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)	2748	int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
2966	{	2749	{
@@ -2988,50 +2771,6 @@ bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) · 21 unmodified lines
2988	}	2771	}
2989	EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled);	2772	EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled);
2990		2773
2991	/*
2992	* Aux-domain specific attach/detach.
2993	*
2994	* Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns
2995	* true. Also, as long as domains are attached to a device through this
2996	* interface, any tries to call iommu_attach_device() should fail
2997	* (iommu_detach_device() can't fail, so we fail when trying to re-attach).
2998	* This should make us safe against a device being attached to a guest as a
2999	* whole while there are still pasid users on it (aux and sva).
3000	*/
3001	int iommu_aux_attach_device(struct iommu_domain domain, struct device dev)
3002	{
3003	int ret = -ENODEV;
3004
3005	if (domain->ops->aux_attach_dev)
3006	ret = domain->ops->aux_attach_dev(domain, dev);
3007
3008	if (!ret)
3009	trace_attach_device_to_domain(dev);
3010
3011	return ret;
3012	}
3013	EXPORT_SYMBOL_GPL(iommu_aux_attach_device);
3014
3015	void iommu_aux_detach_device(struct iommu_domain domain, struct device dev)
3016	{
3017	if (domain->ops->aux_detach_dev) {
3018	domain->ops->aux_detach_dev(domain, dev);
3019	trace_detach_device_from_domain(dev);
3020	}
3021	}
3022	EXPORT_SYMBOL_GPL(iommu_aux_detach_device);
3023
3024	int iommu_aux_get_pasid(struct iommu_domain domain, struct device dev)
3025	{
3026	int ret = -ENODEV;
3027
3028	if (domain->ops->aux_get_pasid)
3029	ret = domain->ops->aux_get_pasid(domain, dev);
3030
3031	return ret;
3032	}
3033	EXPORT_SYMBOL_GPL(iommu_aux_get_pasid);
3034
3035	/**	2774	/**
3036	* iommu_sva_bind_device() - Bind a process address space to a device	2775	* iommu_sva_bind_device() - Bind a process address space to a device
3037	* @dev: the device	2776	* @dev: the device
@@ -3053,9 +2792,9 @@ iommu_sva_bind_device(struct device dev, struct mm_struct mm, void *drvdata) · 15 unmodified lines
3053	{	2792	{
3054	struct iommu_group *group;	2793	struct iommu_group *group;
3055	struct iommu_sva *handle = ERR_PTR(-EINVAL);	2794	struct iommu_sva *handle = ERR_PTR(-EINVAL);
3056	const struct iommu_ops *ops = dev->bus->iommu_ops;	2795	const struct iommu_ops *ops = dev_iommu_ops(dev);
3057		2796
3058	if (!ops \|\| !ops->sva_bind)	2797	if (!ops->sva_bind)
3059	return ERR_PTR(-ENODEV);	2798	return ERR_PTR(-ENODEV);
3060		2799
3061	group = iommu_group_get(dev);	2800	group = iommu_group_get(dev);
@@ -3096,9 +2835,9 @@ void iommu_sva_unbind_device(struct iommu_sva *handle) · 34 unmodified lines
3096	{	2835	{
3097	struct iommu_group *group;	2836	struct iommu_group *group;
3098	struct device *dev = handle->dev;	2837	struct device *dev = handle->dev;
3099	const struct iommu_ops *ops = dev->bus->iommu_ops;	2838	const struct iommu_ops *ops = dev_iommu_ops(dev);
3100		2839
3101	if (!ops \|\| !ops->sva_unbind)	2840	if (!ops->sva_unbind)
3102	return;	2841	return;
3103		2842
3104	group = iommu_group_get(dev);	2843	group = iommu_group_get(dev);
@@ -3115,9 +2854,9 @@ EXPORT_SYMBOL_GPL(iommu_sva_unbind_device); · 10 unmodified lines
3115		2854
3116	u32 iommu_sva_get_pasid(struct iommu_sva *handle)	2855	u32 iommu_sva_get_pasid(struct iommu_sva *handle)
3117	{	2856	{
3118	const struct iommu_ops *ops = handle->dev->bus->iommu_ops;	2857	const struct iommu_ops *ops = dev_iommu_ops(handle->dev);
3119		2858
3120	if (!ops \|\| !ops->sva_get_pasid)	2859	if (!ops->sva_get_pasid)
3121	return IOMMU_PASID_INVALID;	2860	return IOMMU_PASID_INVALID;
3122		2861
3123	return ops->sva_get_pasid(handle);	2862	return ops->sva_get_pasid(handle);

drivers/iommu/iova.c+62−16

@@ -15,13 +15,14 @@ · 14 unmodified lines
15	/* The anchor node sits above the top of the usable address space */	15	/* The anchor node sits above the top of the usable address space */
16	#define IOVA_ANCHOR ~0UL	16	#define IOVA_ANCHOR ~0UL
17		17
		18	#define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */
		19
18	static bool iova_rcache_insert(struct iova_domain *iovad,	20	static bool iova_rcache_insert(struct iova_domain *iovad,
19	unsigned long pfn,	21	unsigned long pfn,
20	unsigned long size);	22	unsigned long size);
21	static unsigned long iova_rcache_get(struct iova_domain *iovad,	23	static unsigned long iova_rcache_get(struct iova_domain *iovad,
22	unsigned long size,	24	unsigned long size,
23	unsigned long limit_pfn);	25	unsigned long limit_pfn);
24	static void init_iova_rcaches(struct iova_domain *iovad);
25	static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);	26	static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
26	static void free_iova_rcaches(struct iova_domain *iovad);	27	static void free_iova_rcaches(struct iova_domain *iovad);
27		28
@@ -64,8 +65,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, · 36 unmodified lines
64	iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;	65	iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
65	rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);	66	rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
66	rb_insert_color(&iovad->anchor.node, &iovad->rbroot);	67	rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
67	cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, &iovad->cpuhp_dead);
68	init_iova_rcaches(iovad);
69	}	68	}
70	EXPORT_SYMBOL_GPL(init_iova_domain);	69	EXPORT_SYMBOL_GPL(init_iova_domain);
71		70
@@ -95,10 +94,11 @@ __cached_rbnode_delete_update(struct iova_domain iovad, struct iova free) · 23 unmodified lines
95	cached_iova = to_iova(iovad->cached32_node);	94	cached_iova = to_iova(iovad->cached32_node);
96	if (free == cached_iova \|\|	95	if (free == cached_iova \|\|
97	(free->pfn_hi < iovad->dma_32bit_pfn &&	96	(free->pfn_hi < iovad->dma_32bit_pfn &&
98	free->pfn_lo >= cached_iova->pfn_lo)) {	97	free->pfn_lo >= cached_iova->pfn_lo))
99	iovad->cached32_node = rb_next(&free->node);	98	iovad->cached32_node = rb_next(&free->node);
		99
		100	if (free->pfn_lo < iovad->dma_32bit_pfn)
100	iovad->max32_alloc_size = iovad->dma_32bit_pfn;	101	iovad->max32_alloc_size = iovad->dma_32bit_pfn;
101	}
102		102
103	cached_iova = to_iova(iovad->cached_node);	103	cached_iova = to_iova(iovad->cached_node);
104	if (free->pfn_lo >= cached_iova->pfn_lo)	104	if (free->pfn_lo >= cached_iova->pfn_lo)
@@ -488,6 +488,13 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) · 383 unmodified lines
488	}	488	}
489	EXPORT_SYMBOL_GPL(free_iova_fast);	489	EXPORT_SYMBOL_GPL(free_iova_fast);
490		490
		491	static void iova_domain_free_rcaches(struct iova_domain *iovad)
		492	{
		493	cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
		494	&iovad->cpuhp_dead);
		495	free_iova_rcaches(iovad);
		496	}
		497
491	/**	498	/**
492	* put_iova_domain - destroys the iova domain	499	* put_iova_domain - destroys the iova domain
493	* @iovad: - iova domain in question.	500	* @iovad: - iova domain in question.
@@ -497,9 +504,9 @@ void put_iova_domain(struct iova_domain *iovad) · 3 unmodified lines
497	{	504	{
498	struct iova iova, tmp;	505	struct iova iova, tmp;
499		506
500	cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,	507	if (iovad->rcaches)
501	&iovad->cpuhp_dead);	508	iova_domain_free_rcaches(iovad);
502	free_iova_rcaches(iovad);	509
503	rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)	510	rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
504	free_iova_mem(iova);	511	free_iova_mem(iova);
505	}	512	}
@@ -608,6 +615,7 @@ EXPORT_SYMBOL_GPL(reserve_iova); · 102 unmodified lines
608	*/	615	*/
609		616
610	#define IOVA_MAG_SIZE 128	617	#define IOVA_MAG_SIZE 128
		618	#define MAX_GLOBAL_MAGS 32 /* magazines per bin */
611		619
612	struct iova_magazine {	620	struct iova_magazine {
613	unsigned long size;	621	unsigned long size;
@@ -620,6 +628,13 @@ struct iova_cpu_rcache { · 6 unmodified lines
620	struct iova_magazine *prev;	628	struct iova_magazine *prev;
621	};	629	};
622		630
		631	struct iova_rcache {
		632	spinlock_t lock;
		633	unsigned long depot_size;
		634	struct iova_magazine *depot[MAX_GLOBAL_MAGS];
		635	struct iova_cpu_rcache __percpu *cpu_rcaches;
		636	};
		637
623	static struct iova_magazine *iova_magazine_alloc(gfp_t flags)	638	static struct iova_magazine *iova_magazine_alloc(gfp_t flags)
624	{	639	{
625	return kzalloc(sizeof(struct iova_magazine), flags);	640	return kzalloc(sizeof(struct iova_magazine), flags);
@@ -693,28 +708,54 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn) · 67 unmodified lines
693	mag->pfns[mag->size++] = pfn;	708	mag->pfns[mag->size++] = pfn;
694	}	709	}
695		710
696	static void init_iova_rcaches(struct iova_domain *iovad)	711	int iova_domain_init_rcaches(struct iova_domain *iovad)
697	{	712	{
698	struct iova_cpu_rcache *cpu_rcache;
699	struct iova_rcache *rcache;
700	unsigned int cpu;	713	unsigned int cpu;
701	int i;	714	int i, ret;
		715
		716	iovad->rcaches = kcalloc(IOVA_RANGE_CACHE_MAX_SIZE,
		717	sizeof(struct iova_rcache),
		718	GFP_KERNEL);
		719	if (!iovad->rcaches)
		720	return -ENOMEM;
702		721
703	for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {	722	for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
		723	struct iova_cpu_rcache *cpu_rcache;
		724	struct iova_rcache *rcache;
		725
704	rcache = &iovad->rcaches[i];	726	rcache = &iovad->rcaches[i];
705	spin_lock_init(&rcache->lock);	727	spin_lock_init(&rcache->lock);
706	rcache->depot_size = 0;	728	rcache->depot_size = 0;
707	rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());	729	rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache),
708	if (WARN_ON(!rcache->cpu_rcaches))	730	cache_line_size());
709	continue;	731	if (!rcache->cpu_rcaches) {
		732	ret = -ENOMEM;
		733	goto out_err;
		734	}
710	for_each_possible_cpu(cpu) {	735	for_each_possible_cpu(cpu) {
711	cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);	736	cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
		737
712	spin_lock_init(&cpu_rcache->lock);	738	spin_lock_init(&cpu_rcache->lock);
713	cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);	739	cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);
714	cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);	740	cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
		741	if (!cpu_rcache->loaded \|\| !cpu_rcache->prev) {
		742	ret = -ENOMEM;
		743	goto out_err;
		744	}
715	}	745	}
716	}	746	}
		747
		748	ret = cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
		749	&iovad->cpuhp_dead);
		750	if (ret)
		751	goto out_err;
		752	return 0;
		753
		754	out_err:
		755	free_iova_rcaches(iovad);
		756	return ret;
717	}	757	}
		758	EXPORT_SYMBOL_GPL(iova_domain_init_rcaches);
718		759
719	/*	760	/*
720	* Try inserting IOVA range starting with 'iova_pfn' into 'rcache', and	761	* Try inserting IOVA range starting with 'iova_pfn' into 'rcache', and
@@ -831,7 +872,7 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, · 110 unmodified lines
831	{	872	{
832	unsigned int log_size = order_base_2(size);	873	unsigned int log_size = order_base_2(size);
833		874
834	if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)	875	if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE \|\| !iovad->rcaches)
835	return 0;	876	return 0;
836		877
837	return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);	878	return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);
@@ -849,6 +890,8 @@ static void free_iova_rcaches(struct iova_domain *iovad) · 11 unmodified lines
849		890
850	for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {	891	for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
851	rcache = &iovad->rcaches[i];	892	rcache = &iovad->rcaches[i];
		893	if (!rcache->cpu_rcaches)
		894	break;
852	for_each_possible_cpu(cpu) {	895	for_each_possible_cpu(cpu) {
853	cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);	896	cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
854	iova_magazine_free(cpu_rcache->loaded);	897	iova_magazine_free(cpu_rcache->loaded);
@@ -858,6 +901,9 @@ static void free_iova_rcaches(struct iova_domain *iovad) · 3 unmodified lines
858	for (j = 0; j < rcache->depot_size; ++j)	901	for (j = 0; j < rcache->depot_size; ++j)
859	iova_magazine_free(rcache->depot[j]);	902	iova_magazine_free(rcache->depot[j]);
860	}	903	}
		904
		905	kfree(iovad->rcaches);
		906	iovad->rcaches = NULL;
861	}	907	}
862		908
863	/*	909	/*

drivers/iommu/ipmmu-vmsa.c+20−12

 static const struct soc_device_attribute soc_needs_opt_in[] = {
 	{ .family = "R-Car Gen3", },
+	{ .family = "R-Car Gen4", },
 	{ .family = "RZ/G2", },
 	{ /* sentinel */ }
 };
 	unsigned int i;
 	/*
-	 * R-Car Gen3 and RZ/G2 use the allow list to opt-in devices.
+	 * R-Car Gen3/4 and RZ/G2 use the allow list to opt-in devices.
 	 * For Other SoCs, this returns true anyway.
 	 */
 	if (!soc_device_match(soc_needs_opt_in))
 static const struct iommu_ops ipmmu_ops = {
 	.domain_alloc = ipmmu_domain_alloc,
-	.domain_free = ipmmu_domain_free,
-	.attach_dev = ipmmu_attach_device,
-	.detach_dev = ipmmu_detach_device,
-	.map = ipmmu_map,
-	.unmap = ipmmu_unmap,
-	.flush_iotlb_all = ipmmu_flush_iotlb_all,
-	.iotlb_sync = ipmmu_iotlb_sync,
-	.iova_to_phys = ipmmu_iova_to_phys,
 	.probe_device = ipmmu_probe_device,
 	.release_device = ipmmu_release_device,
 	.probe_finalize = ipmmu_probe_finalize,
 			? generic_device_group : ipmmu_find_group,
 	.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
 	.of_xlate = ipmmu_of_xlate,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= ipmmu_attach_device,
+		.detach_dev	= ipmmu_detach_device,
+		.map		= ipmmu_map,
+		.unmap		= ipmmu_unmap,
+		.flush_iotlb_all = ipmmu_flush_iotlb_all,
+		.iotlb_sync	= ipmmu_iotlb_sync,
+		.iova_to_phys	= ipmmu_iova_to_phys,
+		.free		= ipmmu_domain_free,
+	}
 };
 /* -----------------------------------------------------------------------------
 	.utlb_offset_base = 0,
 };
-static const struct ipmmu_features ipmmu_features_r8a779a0 = {
+static const struct ipmmu_features ipmmu_features_rcar_gen4 = {
 	.use_ns_alias_offset = false,
 	.has_cache_leaf_nodes = true,
 	.number_of_contexts = 16,
 		.data = &ipmmu_features_rcar_gen3,
 	}, {
 		.compatible = "renesas,ipmmu-r8a779a0",
-		.data = &ipmmu_features_r8a779a0,
+		.data = &ipmmu_features_rcar_gen4,
+	}, {
+		.compatible = "renesas,rcar-gen4-ipmmu",
+		.data = &ipmmu_features_rcar_gen4,
 	}, {
 		/* Terminator */
 	},
 	bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);
 	mmu->features = of_device_get_match_data(&pdev->dev);
 	memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);
-	dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
+	if (ret)
+		return ret;
 	/* Map I/O memory and request IRQ. */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);

drivers/iommu/msm_iommu.c+27−47

@@ -558,11 +558,6 @@ fail: · 557 unmodified lines
558	return ret;	558	return ret;
559	}	559	}
560		560
561	static bool msm_iommu_capable(enum iommu_cap cap)
562	{
563	return false;
564	}
565
566	static void print_ctx_regs(void __iomem *base, int ctx)	561	static void print_ctx_regs(void __iomem *base, int ctx)
567	{	562	{
568	unsigned int fsr = GET_FSR(base, ctx);	563	unsigned int fsr = GET_FSR(base, ctx);
@@ -672,27 +667,28 @@ fail: · 103 unmodified lines
672	}	667	}
673		668
674	static struct iommu_ops msm_iommu_ops = {	669	static struct iommu_ops msm_iommu_ops = {
675	.capable = msm_iommu_capable,
676	.domain_alloc = msm_iommu_domain_alloc,	670	.domain_alloc = msm_iommu_domain_alloc,
677	.domain_free = msm_iommu_domain_free,
678	.attach_dev = msm_iommu_attach_dev,
679	.detach_dev = msm_iommu_detach_dev,
680	.map = msm_iommu_map,
681	.unmap = msm_iommu_unmap,
682	/*
683	* Nothing is needed here, the barrier to guarantee
684	* completion of the tlb sync operation is implicitly
685	* taken care when the iommu client does a writel before
686	* kick starting the other master.
687	*/
688	.iotlb_sync = NULL,
689	.iotlb_sync_map = msm_iommu_sync_map,
690	.iova_to_phys = msm_iommu_iova_to_phys,
691	.probe_device = msm_iommu_probe_device,	671	.probe_device = msm_iommu_probe_device,
692	.release_device = msm_iommu_release_device,	672	.release_device = msm_iommu_release_device,
693	.device_group = generic_device_group,	673	.device_group = generic_device_group,
694	.pgsize_bitmap = MSM_IOMMU_PGSIZES,	674	.pgsize_bitmap = MSM_IOMMU_PGSIZES,
695	.of_xlate = qcom_iommu_of_xlate,	675	.of_xlate = qcom_iommu_of_xlate,
		676	.default_domain_ops = &(const struct iommu_domain_ops) {
		677	.attach_dev = msm_iommu_attach_dev,
		678	.detach_dev = msm_iommu_detach_dev,
		679	.map = msm_iommu_map,
		680	.unmap = msm_iommu_unmap,
		681	/*
		682	* Nothing is needed here, the barrier to guarantee
		683	* completion of the tlb sync operation is implicitly
		684	* taken care when the iommu client does a writel before
		685	* kick starting the other master.
		686	*/
		687	.iotlb_sync = NULL,
		688	.iotlb_sync_map = msm_iommu_sync_map,
		689	.iova_to_phys = msm_iommu_iova_to_phys,
		690	.free = msm_iommu_domain_free,
		691	}
696	};	692	};
697		693
698	static int msm_iommu_probe(struct platform_device *pdev)	694	static int msm_iommu_probe(struct platform_device *pdev)
@@ -710,36 +706,32 @@ static int msm_iommu_probe(struct platform_device *pdev) · 11 unmodified lines
710	INIT_LIST_HEAD(&iommu->ctx_list);	706	INIT_LIST_HEAD(&iommu->ctx_list);
711		707
712	iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk");	708	iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk");
713	if (IS_ERR(iommu->pclk)) {	709	if (IS_ERR(iommu->pclk))
714	dev_err(iommu->dev, "could not get smmu_pclk\n");	710	return dev_err_probe(iommu->dev, PTR_ERR(iommu->pclk),
715	return PTR_ERR(iommu->pclk);	711	"could not get smmu_pclk\n");
716	}
717		712
718	ret = clk_prepare(iommu->pclk);	713	ret = clk_prepare(iommu->pclk);
719	if (ret) {	714	if (ret)
720	dev_err(iommu->dev, "could not prepare smmu_pclk\n");	715	return dev_err_probe(iommu->dev, ret,
721	return ret;	716	"could not prepare smmu_pclk\n");
722	}
723		717
724	iommu->clk = devm_clk_get(iommu->dev, "iommu_clk");	718	iommu->clk = devm_clk_get(iommu->dev, "iommu_clk");
725	if (IS_ERR(iommu->clk)) {	719	if (IS_ERR(iommu->clk)) {
726	dev_err(iommu->dev, "could not get iommu_clk\n");
727	clk_unprepare(iommu->pclk);	720	clk_unprepare(iommu->pclk);
728	return PTR_ERR(iommu->clk);	721	return dev_err_probe(iommu->dev, PTR_ERR(iommu->clk),
		722	"could not get iommu_clk\n");
729	}	723	}
730		724
731	ret = clk_prepare(iommu->clk);	725	ret = clk_prepare(iommu->clk);
732	if (ret) {	726	if (ret) {
733	dev_err(iommu->dev, "could not prepare iommu_clk\n");
734	clk_unprepare(iommu->pclk);	727	clk_unprepare(iommu->pclk);
735	return ret;	728	return dev_err_probe(iommu->dev, ret, "could not prepare iommu_clk\n");
736	}	729	}
737		730
738	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);	731	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
739	iommu->base = devm_ioremap_resource(iommu->dev, r);	732	iommu->base = devm_ioremap_resource(iommu->dev, r);
740	if (IS_ERR(iommu->base)) {	733	if (IS_ERR(iommu->base)) {
741	dev_err(iommu->dev, "could not get iommu base\n");	734	ret = dev_err_probe(iommu->dev, PTR_ERR(iommu->base), "could not get iommu base\n");
742	ret = PTR_ERR(iommu->base);
743	goto fail;	735	goto fail;
744	}	736	}
745	ioaddr = r->start;	737	ioaddr = r->start;
@@ -831,16 +823,4 @@ static struct platform_driver msm_iommu_driver = { · 85 unmodified lines
831	.probe = msm_iommu_probe,	823	.probe = msm_iommu_probe,
832	.remove = msm_iommu_remove,	824	.remove = msm_iommu_remove,
833	};	825	};
834		826	builtin_platform_driver(msm_iommu_driver);
835	static int __init msm_iommu_driver_init(void)
836	{
837	int ret;
838
839	ret = platform_driver_register(&msm_iommu_driver);
840	if (ret != 0)
841	pr_err("Failed to register IOMMU driver\n");
842
843	return ret;
844	}
845	subsys_initcall(msm_iommu_driver_init);
846

drivers/iommu/mtk_iommu.c+33−29

 static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data)
 {
-	for_each_m4u(data) {
+	unsigned long flags;
-		if (pm_runtime_get_if_in_use(data->dev) <= 0)
-			continue;
-		writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
-			       data->base + data->plat_data->inv_sel_reg);
-		writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
-		wmb(); /* Make sure the tlb flush all done */
-		pm_runtime_put(data->dev);
+	spin_lock_irqsave(&data->tlb_lock, flags);
-	}
+	writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
+		       data->base + data->plat_data->inv_sel_reg);
+	writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
+	wmb(); /* Make sure the tlb flush all done */
+	spin_unlock_irqrestore(&data->tlb_lock, flags);
 }
 static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size,
 					   size_t granule,
 					   struct mtk_iommu_data *data)
 {
-	bool has_pm = !!data->dev->pm_domain;
 	unsigned long flags;
 	int ret;
 	u32 tmp;
 	for_each_m4u(data) {
-		if (has_pm) {
+		if (pm_runtime_get_if_in_use(data->dev) <= 0)
-			if (pm_runtime_get_if_in_use(data->dev) <= 0)
+			continue;
-				continue;
-		}
 		spin_lock_irqsave(&data->tlb_lock, flags);
 		writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
 		/* tlb sync */
 		ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
 						tmp, tmp != 0, 10, 1000);
+		/* Clear the CPE status */
+		writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
+		spin_unlock_irqrestore(&data->tlb_lock, flags);
 		if (ret) {
 			dev_warn(data->dev,
 				 "Partial TLB flush timed out, falling back to full flush\n");
 			mtk_iommu_tlb_flush_all(data);
 		}
-		/* Clear the CPE status */
-		writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
-		spin_unlock_irqrestore(&data->tlb_lock, flags);
-		if (has_pm)
+		pm_runtime_put(data->dev);
-			pm_runtime_put(data->dev);
 	}
 }
 static const struct iommu_ops mtk_iommu_ops = {
 	.domain_alloc	= mtk_iommu_domain_alloc,
-	.domain_free	= mtk_iommu_domain_free,
-	.attach_dev	= mtk_iommu_attach_device,
-	.detach_dev	= mtk_iommu_detach_device,
-	.map		= mtk_iommu_map,
-	.unmap		= mtk_iommu_unmap,
-	.flush_iotlb_all = mtk_iommu_flush_iotlb_all,
-	.iotlb_sync	= mtk_iommu_iotlb_sync,
-	.iotlb_sync_map	= mtk_iommu_sync_map,
-	.iova_to_phys	= mtk_iommu_iova_to_phys,
 	.probe_device	= mtk_iommu_probe_device,
 	.release_device	= mtk_iommu_release_device,
 	.device_group	= mtk_iommu_device_group,
 	.put_resv_regions = generic_iommu_put_resv_regions,
 	.pgsize_bitmap	= SZ_4K | SZ_64K | SZ_1M | SZ_16M,
 	.owner		= THIS_MODULE,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= mtk_iommu_attach_device,
+		.detach_dev	= mtk_iommu_detach_device,
+		.map		= mtk_iommu_map,
+		.unmap		= mtk_iommu_unmap,
+		.flush_iotlb_all = mtk_iommu_flush_iotlb_all,
+		.iotlb_sync	= mtk_iommu_iotlb_sync,
+		.iotlb_sync_map	= mtk_iommu_sync_map,
+		.iova_to_phys	= mtk_iommu_iova_to_phys,
+		.free		= mtk_iommu_domain_free,
+	}
 };
 static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
 	writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
 	writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);
 	writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR);
+	/*
+	 * Users may allocate dma buffer before they call pm_runtime_get,
+	 * in which case it will lack the necessary tlb flush.
+	 * Thus, make sure to update the tlb after each PM resume.
+	 */
+	mtk_iommu_tlb_flush_all(data);
 	return 0;
 }

drivers/iommu/mtk_iommu_v1.c+8−6

 static const struct iommu_ops mtk_iommu_ops = {
 	.domain_alloc	= mtk_iommu_domain_alloc,
-	.domain_free	= mtk_iommu_domain_free,
-	.attach_dev	= mtk_iommu_attach_device,
-	.detach_dev	= mtk_iommu_detach_device,
-	.map		= mtk_iommu_map,
-	.unmap		= mtk_iommu_unmap,
-	.iova_to_phys	= mtk_iommu_iova_to_phys,
 	.probe_device	= mtk_iommu_probe_device,
 	.probe_finalize = mtk_iommu_probe_finalize,
 	.release_device	= mtk_iommu_release_device,
 	.device_group	= generic_device_group,
 	.pgsize_bitmap	= ~0UL << MT2701_IOMMU_PAGE_SHIFT,
 	.owner          = THIS_MODULE,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= mtk_iommu_attach_device,
+		.detach_dev	= mtk_iommu_detach_device,
+		.map		= mtk_iommu_map,
+		.unmap		= mtk_iommu_unmap,
+		.iova_to_phys	= mtk_iommu_iova_to_phys,
+		.free		= mtk_iommu_domain_free,
+	}
 };
 static const struct of_device_id mtk_iommu_of_ids[] = {

drivers/iommu/omap-iommu.c+8−6

 static const struct iommu_ops omap_iommu_ops = {
 	.domain_alloc	= omap_iommu_domain_alloc,
-	.domain_free	= omap_iommu_domain_free,
-	.attach_dev	= omap_iommu_attach_dev,
-	.detach_dev	= omap_iommu_detach_dev,
-	.map		= omap_iommu_map,
-	.unmap		= omap_iommu_unmap,
-	.iova_to_phys	= omap_iommu_iova_to_phys,
 	.probe_device	= omap_iommu_probe_device,
 	.release_device	= omap_iommu_release_device,
 	.device_group	= omap_iommu_device_group,
 	.pgsize_bitmap	= OMAP_IOMMU_PGSIZES,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= omap_iommu_attach_dev,
+		.detach_dev	= omap_iommu_detach_dev,
+		.map		= omap_iommu_map,
+		.unmap		= omap_iommu_unmap,
+		.iova_to_phys	= omap_iommu_iova_to_phys,
+		.free		= omap_iommu_domain_free,
+	}
 };
 static int __init omap_iommu_init(void)

drivers/iommu/rockchip-iommu.c+9−12

@@ -1187,17 +1187,19 @@ static int rk_iommu_of_xlate(struct device *dev, · 1186 unmodified lines
1187		1187
1188	static const struct iommu_ops rk_iommu_ops = {	1188	static const struct iommu_ops rk_iommu_ops = {
1189	.domain_alloc = rk_iommu_domain_alloc,	1189	.domain_alloc = rk_iommu_domain_alloc,
1190	.domain_free = rk_iommu_domain_free,
1191	.attach_dev = rk_iommu_attach_device,
1192	.detach_dev = rk_iommu_detach_device,
1193	.map = rk_iommu_map,
1194	.unmap = rk_iommu_unmap,
1195	.probe_device = rk_iommu_probe_device,	1190	.probe_device = rk_iommu_probe_device,
1196	.release_device = rk_iommu_release_device,	1191	.release_device = rk_iommu_release_device,
1197	.iova_to_phys = rk_iommu_iova_to_phys,
1198	.device_group = rk_iommu_device_group,	1192	.device_group = rk_iommu_device_group,
1199	.pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,	1193	.pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,
1200	.of_xlate = rk_iommu_of_xlate,	1194	.of_xlate = rk_iommu_of_xlate,
		1195	.default_domain_ops = &(const struct iommu_domain_ops) {
		1196	.attach_dev = rk_iommu_attach_device,
		1197	.detach_dev = rk_iommu_detach_device,
		1198	.map = rk_iommu_map,
		1199	.unmap = rk_iommu_unmap,
		1200	.iova_to_phys = rk_iommu_iova_to_phys,
		1201	.free = rk_iommu_domain_free,
		1202	}
1201	};	1203	};
1202		1204
1203	static int rk_iommu_probe(struct platform_device *pdev)	1205	static int rk_iommu_probe(struct platform_device *pdev)
@@ -1407,9 +1409,4 @@ static struct platform_driver rk_iommu_driver = { · 203 unmodified lines
1407	.suppress_bind_attrs = true,	1409	.suppress_bind_attrs = true,
1408	},	1410	},
1409	};	1411	};
1410		1412	builtin_platform_driver(rk_iommu_driver);
1411	static int __init rk_iommu_init(void)
1412	{
1413	return platform_driver_register(&rk_iommu_driver);
1414	}
1415	subsys_initcall(rk_iommu_init);

drivers/iommu/s390-iommu.c+8−6

 static const struct iommu_ops s390_iommu_ops = {
 	.capable = s390_iommu_capable,
 	.domain_alloc = s390_domain_alloc,
-	.domain_free = s390_domain_free,
-	.attach_dev = s390_iommu_attach_device,
-	.detach_dev = s390_iommu_detach_device,
-	.map = s390_iommu_map,
-	.unmap = s390_iommu_unmap,
-	.iova_to_phys = s390_iommu_iova_to_phys,
 	.probe_device = s390_iommu_probe_device,
 	.release_device = s390_iommu_release_device,
 	.device_group = generic_device_group,
 	.pgsize_bitmap = S390_IOMMU_PGSIZES,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= s390_iommu_attach_device,
+		.detach_dev	= s390_iommu_detach_device,
+		.map		= s390_iommu_map,
+		.unmap		= s390_iommu_unmap,
+		.iova_to_phys	= s390_iommu_iova_to_phys,
+		.free		= s390_domain_free,
+	}
 };
 static int __init s390_iommu_init(void)

drivers/iommu/sprd-iommu.c+10−8

 static const struct iommu_ops sprd_iommu_ops = {
 	.domain_alloc	= sprd_iommu_domain_alloc,
-	.domain_free	= sprd_iommu_domain_free,
-	.attach_dev	= sprd_iommu_attach_device,
-	.detach_dev	= sprd_iommu_detach_device,
-	.map		= sprd_iommu_map,
-	.unmap		= sprd_iommu_unmap,
-	.iotlb_sync_map	= sprd_iommu_sync_map,
-	.iotlb_sync	= sprd_iommu_sync,
-	.iova_to_phys	= sprd_iommu_iova_to_phys,
 	.probe_device	= sprd_iommu_probe_device,
 	.release_device	= sprd_iommu_release_device,
 	.device_group	= sprd_iommu_device_group,
 	.of_xlate	= sprd_iommu_of_xlate,
 	.pgsize_bitmap	= ~0UL << SPRD_IOMMU_PAGE_SHIFT,
 	.owner		= THIS_MODULE,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= sprd_iommu_attach_device,
+		.detach_dev	= sprd_iommu_detach_device,
+		.map		= sprd_iommu_map,
+		.unmap		= sprd_iommu_unmap,
+		.iotlb_sync_map	= sprd_iommu_sync_map,
+		.iotlb_sync	= sprd_iommu_sync,
+		.iova_to_phys	= sprd_iommu_iova_to_phys,
+		.free		= sprd_iommu_domain_free,
+	}
 };
 static const struct of_device_id sprd_iommu_of_match[] = {

drivers/iommu/sun50i-iommu.c+10−8

 static const struct iommu_ops sun50i_iommu_ops = {
 	.pgsize_bitmap	= SZ_4K,
-	.attach_dev	= sun50i_iommu_attach_device,
-	.detach_dev	= sun50i_iommu_detach_device,
 	.device_group	= sun50i_iommu_device_group,
 	.domain_alloc	= sun50i_iommu_domain_alloc,
-	.domain_free	= sun50i_iommu_domain_free,
-	.flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
-	.iotlb_sync	= sun50i_iommu_iotlb_sync,
-	.iova_to_phys	= sun50i_iommu_iova_to_phys,
-	.map		= sun50i_iommu_map,
 	.of_xlate	= sun50i_iommu_of_xlate,
 	.probe_device	= sun50i_iommu_probe_device,
 	.release_device	= sun50i_iommu_release_device,
-	.unmap		= sun50i_iommu_unmap,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= sun50i_iommu_attach_device,
+		.detach_dev	= sun50i_iommu_detach_device,
+		.flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
+		.iotlb_sync	= sun50i_iommu_iotlb_sync,
+		.iova_to_phys	= sun50i_iommu_iova_to_phys,
+		.map		= sun50i_iommu_map,
+		.unmap		= sun50i_iommu_unmap,
+		.free		= sun50i_iommu_domain_free,
+	}
 };
 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,

drivers/iommu/tegra-gart.c+10−14

 	return pte & GART_PAGE_MASK;
 }
-static bool gart_iommu_capable(enum iommu_cap cap)
-{
-	return false;
-}
 static struct iommu_device *gart_iommu_probe_device(struct device *dev)
 {
 	if (!dev_iommu_fwspec_get(dev))
 }
 static const struct iommu_ops gart_iommu_ops = {
-	.capable	= gart_iommu_capable,
 	.domain_alloc	= gart_iommu_domain_alloc,
-	.domain_free	= gart_iommu_domain_free,
-	.attach_dev	= gart_iommu_attach_dev,
-	.detach_dev	= gart_iommu_detach_dev,
 	.probe_device	= gart_iommu_probe_device,
 	.release_device	= gart_iommu_release_device,
 	.device_group	= generic_device_group,
-	.map		= gart_iommu_map,
-	.unmap		= gart_iommu_unmap,
-	.iova_to_phys	= gart_iommu_iova_to_phys,
 	.pgsize_bitmap	= GART_IOMMU_PGSIZES,
 	.of_xlate	= gart_iommu_of_xlate,
-	.iotlb_sync_map	= gart_iommu_sync_map,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
-	.iotlb_sync	= gart_iommu_sync,
+		.attach_dev	= gart_iommu_attach_dev,
+		.detach_dev	= gart_iommu_detach_dev,
+		.map		= gart_iommu_map,
+		.unmap		= gart_iommu_unmap,
+		.iova_to_phys	= gart_iommu_iova_to_phys,
+		.iotlb_sync_map	= gart_iommu_sync_map,
+		.iotlb_sync	= gart_iommu_sync,
+		.free		= gart_iommu_domain_free,
+	}
 };
 int tegra_gart_suspend(struct gart_device *gart)

drivers/iommu/tegra-smmu.c+8−12

 	clear_bit(id, smmu->asids);
 }
-static bool tegra_smmu_capable(enum iommu_cap cap)
-{
-	return false;
-}
 static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
 {
 	struct tegra_smmu_as *as;
 }
 static const struct iommu_ops tegra_smmu_ops = {
-	.capable = tegra_smmu_capable,
 	.domain_alloc = tegra_smmu_domain_alloc,
-	.domain_free = tegra_smmu_domain_free,
-	.attach_dev = tegra_smmu_attach_dev,
-	.detach_dev = tegra_smmu_detach_dev,
 	.probe_device = tegra_smmu_probe_device,
 	.release_device = tegra_smmu_release_device,
 	.device_group = tegra_smmu_device_group,
-	.map = tegra_smmu_map,
-	.unmap = tegra_smmu_unmap,
-	.iova_to_phys = tegra_smmu_iova_to_phys,
 	.of_xlate = tegra_smmu_of_xlate,
 	.pgsize_bitmap = SZ_4K,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev	= tegra_smmu_attach_dev,
+		.detach_dev	= tegra_smmu_detach_dev,
+		.map		= tegra_smmu_map,
+		.unmap		= tegra_smmu_unmap,
+		.iova_to_phys	= tegra_smmu_iova_to_phys,
+		.free		= tegra_smmu_domain_free,
+	}
 };
 static void tegra_smmu_ahb_enable(void)

drivers/iommu/virtio-iommu.c+8−6

 static struct iommu_ops viommu_ops = {
 	.domain_alloc		= viommu_domain_alloc,
-	.domain_free		= viommu_domain_free,
-	.attach_dev		= viommu_attach_dev,
-	.map			= viommu_map,
-	.unmap			= viommu_unmap,
-	.iova_to_phys		= viommu_iova_to_phys,
-	.iotlb_sync		= viommu_iotlb_sync,
 	.probe_device		= viommu_probe_device,
 	.probe_finalize		= viommu_probe_finalize,
 	.release_device		= viommu_release_device,
 	.put_resv_regions	= generic_iommu_put_resv_regions,
 	.of_xlate		= viommu_of_xlate,
 	.owner			= THIS_MODULE,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev		= viommu_attach_dev,
+		.map			= viommu_map,
+		.unmap			= viommu_unmap,
+		.iova_to_phys		= viommu_iova_to_phys,
+		.iotlb_sync		= viommu_iotlb_sync,
+		.free			= viommu_domain_free,
+	}
 };
 static int viommu_init_vqs(struct viommu_dev *viommu)

drivers/perf/arm_smmuv3_pmu.c+3−1

 static irqreturn_t smmu_pmu_handle_irq(int irq_num, void *data)
 {
 	struct smmu_pmu *smmu_pmu = data;
+	DECLARE_BITMAP(ovs, BITS_PER_TYPE(u64));
 	u64 ovsr;
 	unsigned int idx;
 	writeq(ovsr, smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0);
-	for_each_set_bit(idx, (unsigned long *)&ovsr, smmu_pmu->num_counters) {
+	bitmap_from_u64(ovs, ovsr);
+	for_each_set_bit(idx, ovs, smmu_pmu->num_counters) {
 		struct perf_event *event = smmu_pmu->events[idx];
 		struct hw_perf_event *hwc;

drivers/vdpa/vdpa_user/iova_domain.c+11−0

 	struct file *file;
 	struct vduse_bounce_map *map;
 	unsigned long pfn, bounce_pfns;
+	int ret;
 	bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
 	if (iova_limit <= bounce_size)
 	spin_lock_init(&domain->iotlb_lock);
 	init_iova_domain(&domain->stream_iovad,
 			PAGE_SIZE, IOVA_START_PFN);
+	ret = iova_domain_init_rcaches(&domain->stream_iovad);
+	if (ret)
+		goto err_iovad_stream;
 	init_iova_domain(&domain->consistent_iovad,
 			PAGE_SIZE, bounce_pfns);
+	ret = iova_domain_init_rcaches(&domain->consistent_iovad);
+	if (ret)
+		goto err_iovad_consistent;
 	return domain;
+err_iovad_consistent:
+	put_iova_domain(&domain->stream_iovad);
+err_iovad_stream:
+	fput(file);
 err_file:
 	vfree(domain->bounce_maps);
 err_map:

include/linux/intel-iommu.h+2−31

  */
 #define DOMAIN_FLAG_USE_FIRST_LEVEL		BIT(1)
-/*
- * Domain represents a virtual machine which demands iommu nested
- * translation mode support.
- */
-#define DOMAIN_FLAG_NESTING_MODE		BIT(2)
 struct dmar_domain {
 	int	nid;			/* node id */
 	u8 iommu_snooping: 1;		/* indicate snooping control feature */
 	struct list_head devices;	/* all devices' list */
-	struct list_head subdevices;	/* all subdevices' list */
 	struct iova_domain iovad;	/* iova's that belong to this domain */
 	struct dma_pte	*pgd;		/* virtual address */
 					   2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 	u64		max_addr;	/* maximum mapped address */
-	u32		default_pasid;	/*
-					 * The default pasid used for non-SVM
-					 * traffic on mediated devices.
-					 */
 	struct iommu_domain domain;	/* generic domain data structure for
 					   iommu core */
 };
 #ifdef CONFIG_INTEL_IOMMU
 	unsigned long 	*domain_ids; /* bitmap of domains */
-	struct dmar_domain ***domains; /* ptr to domains */
 	spinlock_t	lock; /* protect context, domain ids */
 	struct root_entry *root_entry; /* virtual address */
 	void *perf_statistic;
 };
-/* Per subdevice private data */
-struct subdev_domain_info {
-	struct list_head link_phys;	/* link to phys device siblings */
-	struct list_head link_domain;	/* link to domain siblings */
-	struct device *pdev;		/* physical device derived from */
-	struct dmar_domain *domain;	/* aux-domain */
-	int users;			/* user count */
-};
 /* PCI domain-device relationship */
 struct device_domain_info {
 	struct list_head link;	/* link to domain siblings */
 	struct list_head global; /* link to global list */
 	struct list_head table;	/* link to pasid table */
-	struct list_head subdevices; /* subdevices sibling */
 	u32 segment;		/* PCI segment number */
 	u8 bus;			/* PCI bus number */
 	u8 devfn;		/* PCI devfn number */
 	u8 pri_enabled:1;
 	u8 ats_supported:1;
 	u8 ats_enabled:1;
-	u8 auxd_enabled:1;	/* Multiple domains per device */
 	u8 ats_qdep;
 	struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 	struct intel_iommu *iommu; /* IOMMU used by this device */
 }
 extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev);
-extern int dmar_find_matched_atsr_unit(struct pci_dev *dev);
 extern int dmar_enable_qi(struct intel_iommu *iommu);
 extern void dmar_disable_qi(struct intel_iommu *iommu);
 				     void *data), void *data);
 void iommu_flush_write_buffer(struct intel_iommu *iommu);
 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev);
-struct dmar_domain *find_domain(struct device *dev);
-struct device_domain_info *get_domain_info(struct device *dev);
 struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);
 #ifdef CONFIG_INTEL_IOMMU_SVM
 extern void intel_svm_check(struct intel_iommu *iommu);
 extern int intel_svm_enable_prq(struct intel_iommu *iommu);
 extern int intel_svm_finish_prq(struct intel_iommu *iommu);
-int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,
-			  struct iommu_gpasid_bind_data *data);
-int intel_svm_unbind_gpasid(struct device *dev, u32 pasid);
 struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm,
 				 void *drvdata);
 void intel_svm_unbind(struct iommu_sva *handle);
 	unsigned int flags;
 	u32 pasid;
-	int gpasid; /* In case that guest PASID is different from host PASID */
 	struct list_head devs;
 };
 #else
 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
 					 u8 devfn, int alloc);
+extern const struct iommu_ops intel_iommu_ops;
 #ifdef CONFIG_INTEL_IOMMU
 extern int iommu_calculate_agaw(struct intel_iommu *iommu);
 extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu);

include/linux/intel-svm.h+0−12

@@ -25,17 +25,5 @@ · 24 unmodified lines
25	* do such IOTLB flushes automatically.	25	* do such IOTLB flushes automatically.
26	*/	26	*/
27	#define SVM_FLAG_SUPERVISOR_MODE BIT(0)	27	#define SVM_FLAG_SUPERVISOR_MODE BIT(0)
28	/*
29	* The SVM_FLAG_GUEST_MODE flag is used when a PASID bind is for guest
30	* processes. Compared to the host bind, the primary differences are:
31	* 1. mm life cycle management
32	* 2. fault reporting
33	*/
34	#define SVM_FLAG_GUEST_MODE BIT(1)
35	/*
36	* The SVM_FLAG_GUEST_PASID flag is used when a guest has its own PASID space,
37	* which requires guest and host PASID translation at both directions.
38	*/
39	#define SVM_FLAG_GUEST_PASID BIT(2)
40		28
41	#endif /* __INTEL_SVM_H__ */	29	#endif /* __INTEL_SVM_H__ */

include/linux/iommu.h+65−116

 struct bus_type;
 struct device;
 struct iommu_domain;
+struct iommu_domain_ops;
 struct notifier_block;
 struct iommu_sva;
 struct iommu_fault_event;
 struct iommu_domain {
 	unsigned type;
-	const struct iommu_ops *ops;
+	const struct iommu_domain_ops *ops;
 	unsigned long pgsize_bitmap;	/* Bitmap of page sizes in use */
 	iommu_fault_handler_t handler;
 	void *handler_token;
 /**
  * enum iommu_dev_features - Per device IOMMU features
- * @IOMMU_DEV_FEAT_AUX: Auxiliary domain feature
  * @IOMMU_DEV_FEAT_SVA: Shared Virtual Addresses
  * @IOMMU_DEV_FEAT_IOPF: I/O Page Faults such as PRI or Stall. Generally
  *			 enabling %IOMMU_DEV_FEAT_SVA requires
  * iommu_dev_has_feature(), and enable it using iommu_dev_enable_feature().
  */
 enum iommu_dev_features {
-	IOMMU_DEV_FEAT_AUX,
 	IOMMU_DEV_FEAT_SVA,
 	IOMMU_DEV_FEAT_IOPF,
 };
  * struct iommu_ops - iommu ops and capabilities
  * @capable: check capability
  * @domain_alloc: allocate iommu domain
- * @domain_free: free iommu domain
- * @attach_dev: attach device to an iommu domain
- * @detach_dev: detach device from an iommu domain
- * @map: map a physically contiguous memory region to an iommu domain
- * @map_pages: map a physically contiguous set of pages of the same size to
- *             an iommu domain.
- * @unmap: unmap a physically contiguous memory region from an iommu domain
- * @unmap_pages: unmap a number of pages of the same size from an iommu domain
- * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
- * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
- * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
- *            queue
- * @iova_to_phys: translate iova to physical address
  * @probe_device: Add device to iommu driver handling
  * @release_device: Remove device from iommu driver handling
  * @probe_finalize: Do final setup work after the device is added to an IOMMU
  *                  group and attached to the groups domain
  * @device_group: find iommu group for a particular device
- * @enable_nesting: Enable nesting
- * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)
  * @get_resv_regions: Request list of reserved regions for a device
  * @put_resv_regions: Free list of reserved regions for a device
- * @apply_resv_region: Temporary helper call-back for iova reserved ranges
  * @of_xlate: add OF master IDs to iommu grouping
  * @is_attach_deferred: Check if domain attach should be deferred from iommu
  *                      driver init to device driver init (default no)
  * @dev_has/enable/disable_feat: per device entries to check/enable/disable
  *                               iommu specific features.
  * @dev_feat_enabled: check enabled feature
- * @aux_attach/detach_dev: aux-domain specific attach/detach entries.
- * @aux_get_pasid: get the pasid given an aux-domain
  * @sva_bind: Bind process address space to device
  * @sva_unbind: Unbind process address space from device
  * @sva_get_pasid: Get PASID associated to a SVA handle
  * @page_response: handle page request response
- * @cache_invalidate: invalidate translation caches
- * @sva_bind_gpasid: bind guest pasid and mm
- * @sva_unbind_gpasid: unbind guest pasid and mm
  * @def_domain_type: device default domain type, return value:
  *		- IOMMU_DOMAIN_IDENTITY: must use an identity domain
  *		- IOMMU_DOMAIN_DMA: must use a dma domain
  *		- 0: use the default setting
+ * @default_domain_ops: the default ops for domains
  * @pgsize_bitmap: bitmap of all possible supported page sizes
  * @owner: Driver module providing these ops
  */
 	/* Domain allocation and freeing by the iommu driver */
 	struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type);
-	void (*domain_free)(struct iommu_domain *);
-	int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
-	void (*detach_dev)(struct iommu_domain *domain, struct device *dev);
-	int (*map)(struct iommu_domain *domain, unsigned long iova,
-		   phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
-	int (*map_pages)(struct iommu_domain *domain, unsigned long iova,
-			 phys_addr_t paddr, size_t pgsize, size_t pgcount,
-			 int prot, gfp_t gfp, size_t *mapped);
-	size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
-		     size_t size, struct iommu_iotlb_gather *iotlb_gather);
-	size_t (*unmap_pages)(struct iommu_domain *domain, unsigned long iova,
-			      size_t pgsize, size_t pgcount,
-			      struct iommu_iotlb_gather *iotlb_gather);
-	void (*flush_iotlb_all)(struct iommu_domain *domain);
-	void (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova,
-			       size_t size);
-	void (*iotlb_sync)(struct iommu_domain *domain,
-			   struct iommu_iotlb_gather *iotlb_gather);
-	phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova);
 	struct iommu_device *(*probe_device)(struct device *dev);
 	void (*release_device)(struct device *dev);
 	void (*probe_finalize)(struct device *dev);
 	struct iommu_group *(*device_group)(struct device *dev);
-	int (*enable_nesting)(struct iommu_domain *domain);
-	int (*set_pgtable_quirks)(struct iommu_domain *domain,
-				  unsigned long quirks);
 	/* Request/Free a list of reserved regions for a device */
 	void (*get_resv_regions)(struct device *dev, struct list_head *list);
 	void (*put_resv_regions)(struct device *dev, struct list_head *list);
-	void (*apply_resv_region)(struct device *dev,
-				  struct iommu_domain *domain,
-				  struct iommu_resv_region *region);
 	int (*of_xlate)(struct device *dev, struct of_phandle_args *args);
-	bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev);
+	bool (*is_attach_deferred)(struct device *dev);
 	/* Per device IOMMU features */
 	bool (*dev_has_feat)(struct device *dev, enum iommu_dev_features f);
 	int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f);
 	int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f);
-	/* Aux-domain specific attach/detach entries */
-	int (*aux_attach_dev)(struct iommu_domain *domain, struct device *dev);
-	void (*aux_detach_dev)(struct iommu_domain *domain, struct device *dev);
-	int (*aux_get_pasid)(struct iommu_domain *domain, struct device *dev);
 	struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm,
 				      void *drvdata);
 	void (*sva_unbind)(struct iommu_sva *handle);
 	int (*page_response)(struct device *dev,
 			     struct iommu_fault_event *evt,
 			     struct iommu_page_response *msg);
-	int (*cache_invalidate)(struct iommu_domain *domain, struct device *dev,
-				struct iommu_cache_invalidate_info *inv_info);
-	int (*sva_bind_gpasid)(struct iommu_domain *domain,
-			struct device *dev, struct iommu_gpasid_bind_data *data);
-	int (*sva_unbind_gpasid)(struct device *dev, u32 pasid);
 	int (*def_domain_type)(struct device *dev);
+	const struct iommu_domain_ops *default_domain_ops;
 	unsigned long pgsize_bitmap;
 	struct module *owner;
 };
+/**
+ * struct iommu_domain_ops - domain specific operations
+ * @attach_dev: attach an iommu domain to a device
+ * @detach_dev: detach an iommu domain from a device
+ * @map: map a physically contiguous memory region to an iommu domain
+ * @map_pages: map a physically contiguous set of pages of the same size to
+ *             an iommu domain.
+ * @unmap: unmap a physically contiguous memory region from an iommu domain
+ * @unmap_pages: unmap a number of pages of the same size from an iommu domain
+ * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
+ * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
+ * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
+ *            queue
+ * @iova_to_phys: translate iova to physical address
+ * @enable_nesting: Enable nesting
+ * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)
+ * @free: Release the domain after use.
+ */
+struct iommu_domain_ops {
+	int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
+	void (*detach_dev)(struct iommu_domain *domain, struct device *dev);
+	int (*map)(struct iommu_domain *domain, unsigned long iova,
+		   phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
+	int (*map_pages)(struct iommu_domain *domain, unsigned long iova,
+			 phys_addr_t paddr, size_t pgsize, size_t pgcount,
+			 int prot, gfp_t gfp, size_t *mapped);
+	size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
+			size_t size, struct iommu_iotlb_gather *iotlb_gather);
+	size_t (*unmap_pages)(struct iommu_domain *domain, unsigned long iova,
+			      size_t pgsize, size_t pgcount,
+			      struct iommu_iotlb_gather *iotlb_gather);
+	void (*flush_iotlb_all)(struct iommu_domain *domain);
+	void (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova,
+			       size_t size);
+	void (*iotlb_sync)(struct iommu_domain *domain,
+			   struct iommu_iotlb_gather *iotlb_gather);
+	phys_addr_t (*iova_to_phys)(struct iommu_domain *domain,
+				    dma_addr_t iova);
+	int (*enable_nesting)(struct iommu_domain *domain);
+	int (*set_pgtable_quirks)(struct iommu_domain *domain,
+				  unsigned long quirks);
+	void (*free)(struct iommu_domain *domain);
+};
 /**
  * struct iommu_device - IOMMU core representation of one IOMMU hardware
  *			 instance
 	};
 }
+static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
+{
+	/*
+	 * Assume that valid ops must be installed if iommu_probe_device()
+	 * has succeeded. The device ops are essentially for internal use
+	 * within the IOMMU subsystem itself, so we should be able to trust
+	 * ourselves not to misuse the helper.
+	 */
+	return dev->iommu->iommu_dev->ops;
+}
 #define IOMMU_GROUP_NOTIFY_ADD_DEVICE		1 /* Device added */
 #define IOMMU_GROUP_NOTIFY_DEL_DEVICE		2 /* Pre Device removed */
 #define IOMMU_GROUP_NOTIFY_BIND_DRIVER		3 /* Pre Driver bind */
 			       struct device *dev);
 extern void iommu_detach_device(struct iommu_domain *domain,
 				struct device *dev);
-extern int iommu_uapi_cache_invalidate(struct iommu_domain *domain,
-				       struct device *dev,
-				       void __user *uinfo);
-extern int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain,
-				      struct device *dev, void __user *udata);
-extern int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain,
-					struct device *dev, void __user *udata);
 extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
 				   struct device *dev, ioasid_t pasid);
 extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
 bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f);
-int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev);
-void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev);
-int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev);
 struct iommu_sva *iommu_sva_bind_device(struct device *dev,
 					struct mm_struct *mm,
 	return -ENODEV;
 }
-static inline int
-iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev)
-{
-	return -ENODEV;
-}
-static inline void
-iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev)
-{
-}
-static inline int
-iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
-{
-	return -ENODEV;
-}
 static inline struct iommu_sva *
 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
 {
 	return IOMMU_PASID_INVALID;
 }
-static inline int
-iommu_uapi_cache_invalidate(struct iommu_domain *domain,
-			    struct device *dev,
-			    struct iommu_cache_invalidate_info *inv_info)
-{
-	return -ENODEV;
-}
-static inline int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain,
-					     struct device *dev, void __user *udata)
-{
-	return -ENODEV;
-}
-static inline int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain,
-					       struct device *dev, void __user *udata)
-{
-	return -ENODEV;
-}
-static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
-					  struct device *dev,
-					  ioasid_t pasid)
-{
-	return -ENODEV;
-}
 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
 {
 	return NULL;

include/linux/iova.h+3−12

 	unsigned long	pfn_lo; /* Lowest allocated pfn */
 };
-struct iova_magazine;
-struct iova_cpu_rcache;
-#define IOVA_RANGE_CACHE_MAX_SIZE 6	/* log of max cached IOVA range size (in pages) */
+struct iova_rcache;
-#define MAX_GLOBAL_MAGS 32	/* magazines per bin */
-struct iova_rcache {
-	spinlock_t lock;
-	unsigned long depot_size;
-	struct iova_magazine *depot[MAX_GLOBAL_MAGS];
-	struct iova_cpu_rcache __percpu *cpu_rcaches;
-};
 /* holds all the iova translations for a domain */
 struct iova_domain {
 	unsigned long	max32_alloc_size; /* Size of last failed allocation */
 	struct iova	anchor;		/* rbtree lookup anchor */
-	struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE];	/* IOVA range caches */
+	struct iova_rcache	*rcaches;
 	struct hlist_node	cpuhp_dead;
 };
 	unsigned long pfn_hi);
 void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
 	unsigned long start_pfn);
+int iova_domain_init_rcaches(struct iova_domain *iovad);
 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
 void put_iova_domain(struct iova_domain *iovad);
 #else

include/uapi/linux/iommu.h+0−181

@@ -158,185 +158,4 @@ struct iommu_page_response { · 157 unmodified lines
158	__u32 code;	158	__u32 code;
159	};	159	};
160		160
161	/* defines the granularity of the invalidation */
162	enum iommu_inv_granularity {
163	IOMMU_INV_GRANU_DOMAIN, /* domain-selective invalidation */
164	IOMMU_INV_GRANU_PASID, /* PASID-selective invalidation */
165	IOMMU_INV_GRANU_ADDR, /* page-selective invalidation */
166	IOMMU_INV_GRANU_NR, /* number of invalidation granularities */
167	};
168
169	/**
170	* struct iommu_inv_addr_info - Address Selective Invalidation Structure
171	*
172	* @flags: indicates the granularity of the address-selective invalidation
173	* - If the PASID bit is set, the @pasid field is populated and the invalidation
174	* relates to cache entries tagged with this PASID and matching the address
175	* range.
176	* - If ARCHID bit is set, @archid is populated and the invalidation relates
177	* to cache entries tagged with this architecture specific ID and matching
178	* the address range.
179	* - Both PASID and ARCHID can be set as they may tag different caches.
180	* - If neither PASID or ARCHID is set, global addr invalidation applies.
181	* - The LEAF flag indicates whether only the leaf PTE caching needs to be
182	* invalidated and other paging structure caches can be preserved.
183	* @pasid: process address space ID
184	* @archid: architecture-specific ID
185	* @addr: first stage/level input address
186	* @granule_size: page/block size of the mapping in bytes
187	* @nb_granules: number of contiguous granules to be invalidated
188	*/
189	struct iommu_inv_addr_info {
190	#define IOMMU_INV_ADDR_FLAGS_PASID (1 << 0)
191	#define IOMMU_INV_ADDR_FLAGS_ARCHID (1 << 1)
192	#define IOMMU_INV_ADDR_FLAGS_LEAF (1 << 2)
193	__u32 flags;
194	__u32 archid;
195	__u64 pasid;
196	__u64 addr;
197	__u64 granule_size;
198	__u64 nb_granules;
199	};
200
201	/**
202	* struct iommu_inv_pasid_info - PASID Selective Invalidation Structure
203	*
204	* @flags: indicates the granularity of the PASID-selective invalidation
205	* - If the PASID bit is set, the @pasid field is populated and the invalidation
206	* relates to cache entries tagged with this PASID and matching the address
207	* range.
208	* - If the ARCHID bit is set, the @archid is populated and the invalidation
209	* relates to cache entries tagged with this architecture specific ID and
210	* matching the address range.
211	* - Both PASID and ARCHID can be set as they may tag different caches.
212	* - At least one of PASID or ARCHID must be set.
213	* @pasid: process address space ID
214	* @archid: architecture-specific ID
215	*/
216	struct iommu_inv_pasid_info {
217	#define IOMMU_INV_PASID_FLAGS_PASID (1 << 0)
218	#define IOMMU_INV_PASID_FLAGS_ARCHID (1 << 1)
219	__u32 flags;
220	__u32 archid;
221	__u64 pasid;
222	};
223
224	/**
225	* struct iommu_cache_invalidate_info - First level/stage invalidation
226	* information
227	* @argsz: User filled size of this data
228	* @version: API version of this structure
229	* @cache: bitfield that allows to select which caches to invalidate
230	* @granularity: defines the lowest granularity used for the invalidation:
231	* domain > PASID > addr
232	* @padding: reserved for future use (should be zero)
233	* @pasid_info: invalidation data when @granularity is %IOMMU_INV_GRANU_PASID
234	* @addr_info: invalidation data when @granularity is %IOMMU_INV_GRANU_ADDR
235	*
236	* Not all the combinations of cache/granularity are valid:
237	*
238	* +--------------+---------------+---------------+---------------+
239	* \| type / \| DEV_IOTLB \| IOTLB \| PASID \|
240	* \| granularity \| \| \| cache \|
241	* +==============+===============+===============+===============+
242	* \| DOMAIN \| N/A \| Y \| Y \|
243	* +--------------+---------------+---------------+---------------+
244	* \| PASID \| Y \| Y \| Y \|
245	* +--------------+---------------+---------------+---------------+
246	* \| ADDR \| Y \| Y \| N/A \|
247	* +--------------+---------------+---------------+---------------+
248	*
249	* Invalidations by %IOMMU_INV_GRANU_DOMAIN don't take any argument other than
250	* @version and @cache.
251	*
252	* If multiple cache types are invalidated simultaneously, they all
253	* must support the used granularity.
254	*/
255	struct iommu_cache_invalidate_info {
256	__u32 argsz;
257	#define IOMMU_CACHE_INVALIDATE_INFO_VERSION_1 1
258	__u32 version;
259	/* IOMMU paging structure cache */
260	#define IOMMU_CACHE_INV_TYPE_IOTLB (1 << 0) /* IOMMU IOTLB */
261	#define IOMMU_CACHE_INV_TYPE_DEV_IOTLB (1 << 1) /* Device IOTLB */
262	#define IOMMU_CACHE_INV_TYPE_PASID (1 << 2) /* PASID cache */
263	#define IOMMU_CACHE_INV_TYPE_NR (3)
264	__u8 cache;
265	__u8 granularity;
266	__u8 padding[6];
267	union {
268	struct iommu_inv_pasid_info pasid_info;
269	struct iommu_inv_addr_info addr_info;
270	} granu;
271	};
272
273	/**
274	* struct iommu_gpasid_bind_data_vtd - Intel VT-d specific data on device and guest
275	* SVA binding.
276	*
277	* @flags: VT-d PASID table entry attributes
278	* @pat: Page attribute table data to compute effective memory type
279	* @emt: Extended memory type
280	*
281	* Only guest vIOMMU selectable and effective options are passed down to
282	* the host IOMMU.
283	*/
284	struct iommu_gpasid_bind_data_vtd {
285	#define IOMMU_SVA_VTD_GPASID_SRE (1 << 0) /* supervisor request */
286	#define IOMMU_SVA_VTD_GPASID_EAFE (1 << 1) /* extended access enable */
287	#define IOMMU_SVA_VTD_GPASID_PCD (1 << 2) /* page-level cache disable */
288	#define IOMMU_SVA_VTD_GPASID_PWT (1 << 3) /* page-level write through */
289	#define IOMMU_SVA_VTD_GPASID_EMTE (1 << 4) /* extended mem type enable */
290	#define IOMMU_SVA_VTD_GPASID_CD (1 << 5) /* PASID-level cache disable */
291	#define IOMMU_SVA_VTD_GPASID_WPE (1 << 6) /* Write protect enable */
292	#define IOMMU_SVA_VTD_GPASID_LAST (1 << 7)
293	__u64 flags;
294	__u32 pat;
295	__u32 emt;
296	};
297
298	#define IOMMU_SVA_VTD_GPASID_MTS_MASK (IOMMU_SVA_VTD_GPASID_CD \| \
299	IOMMU_SVA_VTD_GPASID_EMTE \| \
300	IOMMU_SVA_VTD_GPASID_PCD \| \
301	IOMMU_SVA_VTD_GPASID_PWT)
302
303	/**
304	* struct iommu_gpasid_bind_data - Information about device and guest PASID binding
305	* @argsz: User filled size of this data
306	* @version: Version of this data structure
307	* @format: PASID table entry format
308	* @flags: Additional information on guest bind request
309	* @gpgd: Guest page directory base of the guest mm to bind
310	* @hpasid: Process address space ID used for the guest mm in host IOMMU
311	* @gpasid: Process address space ID used for the guest mm in guest IOMMU
312	* @addr_width: Guest virtual address width
313	* @padding: Reserved for future use (should be zero)
314	* @vtd: Intel VT-d specific data
315	*
316	* Guest to host PASID mapping can be an identity or non-identity, where guest
317	* has its own PASID space. For non-identify mapping, guest to host PASID lookup
318	* is needed when VM programs guest PASID into an assigned device. VMM may
319	* trap such PASID programming then request host IOMMU driver to convert guest
320	* PASID to host PASID based on this bind data.
321	*/
322	struct iommu_gpasid_bind_data {
323	__u32 argsz;
324	#define IOMMU_GPASID_BIND_VERSION_1 1
325	__u32 version;
326	#define IOMMU_PASID_FORMAT_INTEL_VTD 1
327	#define IOMMU_PASID_FORMAT_LAST 2
328	__u32 format;
329	__u32 addr_width;
330	#define IOMMU_SVA_GPASID_VAL (1 << 0) /* guest PASID valid */
331	__u64 flags;
332	__u64 gpgd;
333	__u64 hpasid;
334	__u64 gpasid;
335	__u8 padding[8];
336	/* Vendor specific data */
337	union {
338	struct iommu_gpasid_bind_data_vtd vtd;
339	} vendor;
340	};
341
342	#endif /* _UAPI_IOMMU_H */	161	#endif /* _UAPI_IOMMU_H */

Merge tag 'iommu-updates-v5.18' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code