Merge tag 'iommu-updates-v5.18' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Pull iommu updates from Joerg Roedel: - IOMMU Core changes: - Removal of aux domain related code as it is basically dead and will be replaced by iommu-fd framework - Split of iommu_ops to carry domain-specific call-backs separatly - Cleanup to remove useless ops->capable implementations - Improve 32-bit free space estimate in iova allocator - Intel VT-d updates: - Various cleanups of the driver - Support for ATS of SoC-integrated devices listed in ACPI/SATC table - ARM SMMU updates: - Fix SMMUv3 soft lockup during continuous stream of events - Fix error path for Qualcomm SMMU probe() - Rework SMMU IRQ setup to prepare the ground for PMU support - Minor cleanups and refactoring - AMD IOMMU driver: - Some minor cleanups and error-handling fixes - Rockchip IOMMU driver: - Use standard driver registration - MSM IOMMU driver: - Minor cleanup and change to standard driver registration - Mediatek IOMMU driver: - Fixes for IOTLB flushing logic * tag 'iommu-updates-v5.18' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (47 commits) iommu/amd: Improve amd_iommu_v2_exit() iommu/amd: Remove unused struct fault.devid iommu/amd: Clean up function declarations iommu/amd: Call memunmap in error path iommu/arm-smmu: Account for PMU interrupts iommu/vt-d: Enable ATS for the devices in SATC table iommu/vt-d: Remove unused function intel_svm_capable() iommu/vt-d: Add missing "__init" for rmrr_sanity_check() iommu/vt-d: Move intel_iommu_ops to header file iommu/vt-d: Fix indentation of goto labels iommu/vt-d: Remove unnecessary prototypes iommu/vt-d: Remove unnecessary includes iommu/vt-d: Remove DEFER_DEVICE_DOMAIN_INFO iommu/vt-d: Remove domain and devinfo mempool iommu/vt-d: Remove iova_cache_get/put() iommu/vt-d: Remove finding domain in dmar_insert_one_dev_info() iommu/vt-d: Remove intel_iommu::domains iommu/mediatek: Always tlb_flush_all when each PM resume iommu/mediatek: Add tlb_lock in tlb_flush_all iommu/mediatek: Remove the power status checking in tlb flush all ...

Linus Torvalds committed 4 years ago

Commit 34af78c2 parents 6f2689a e17c6de41 files · +662 −2251Browse files

Unified · Split

41 files

Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml +4 −0
drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c +1 −1
drivers/iommu/amd/amd_iommu.h +1 −3
drivers/iommu/amd/init.c +14 −4
drivers/iommu/amd/iommu.c +12 −11
drivers/iommu/amd/iommu_v2.c +18 −19
drivers/iommu/apple-dart.c +11 −9
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +18 −27
drivers/iommu/arm/arm-smmu/arm-smmu.c +55 −58
drivers/iommu/arm/arm-smmu/arm-smmu.h +2 −3
drivers/iommu/arm/arm-smmu/qcom_iommu.c +17 −11
drivers/iommu/dma-iommu.c +4 −0
drivers/iommu/exynos-iommu.c +8 −6
drivers/iommu/fsl_pamu_domain.c +6 −4
drivers/iommu/intel/debugfs.c +3 −3
drivers/iommu/intel/dmar.c +2 −3
drivers/iommu/intel/iommu.c +133 −866
drivers/iommu/intel/pasid.c +6 −167
drivers/iommu/intel/pasid.h +0 −4
drivers/iommu/intel/svm.c +3 −217
drivers/iommu/iommu.c +39 −300
drivers/iommu/iova.c +62 −16
drivers/iommu/ipmmu-vmsa.c +20 −12
drivers/iommu/msm_iommu.c +27 −47
drivers/iommu/mtk_iommu.c +33 −29
drivers/iommu/mtk_iommu_v1.c +8 −6
drivers/iommu/omap-iommu.c +8 −6
drivers/iommu/rockchip-iommu.c +9 −12
drivers/iommu/s390-iommu.c +8 −6
drivers/iommu/sprd-iommu.c +10 −8
drivers/iommu/sun50i-iommu.c +10 −8
drivers/iommu/tegra-gart.c +10 −14
drivers/iommu/tegra-smmu.c +8 −12
drivers/iommu/virtio-iommu.c +8 −6
drivers/perf/arm_smmuv3_pmu.c +3 −1
drivers/vdpa/vdpa_user/iova_domain.c +11 −0
include/linux/intel-iommu.h +2 −31
include/linux/intel-svm.h +0 −12
include/linux/iommu.h +65 −116
include/linux/iova.h +3 −12
include/uapi/linux/iommu.h +0 −181

Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml+4−0

@@ -44,6 +44,10 @@ properties: · 43 unmodified lines
44	44	- renesas,ipmmu-r8a77990 # R-Car E3
45	45	- renesas,ipmmu-r8a77995 # R-Car D3
46	46	- renesas,ipmmu-r8a779a0 # R-Car V3U
	47	+ - items:
	48	+ - enum:
	49	+ - renesas,ipmmu-r8a779f0 # R-Car S4-8
	50	+ - const: renesas,rcar-gen4-ipmmu-vmsa # R-Car Gen4
47	51
48	52	reg:
49	53	maxItems: 1

drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c+1−1

@@ -133,7 +133,7 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev) · 132 unmodified lines
133	133	* or equal to the system's PAGE_SIZE, with a preference if
134	134	* both are equal.
135	135	*/
136		− pgsize_bitmap = tdev->iommu.domain->ops->pgsize_bitmap;
	136	+ pgsize_bitmap = tdev->iommu.domain->pgsize_bitmap;
137	137	if (pgsize_bitmap & PAGE_SIZE) {
138	138	tdev->iommu.pgshift = PAGE_SHIFT;
139	139	} else {

drivers/iommu/amd/amd_iommu.h+1−3

@@ -15,7 +15,6 @@ extern irqreturn_t amd_iommu_int_thread(int irq, void *data); · 14 unmodified lines
15	15	extern irqreturn_t amd_iommu_int_handler(int irq, void *data);
16	16	extern void amd_iommu_apply_erratum_63(u16 devid);
17	17	extern void amd_iommu_restart_event_logging(struct amd_iommu *iommu);
18		−extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu);
19	18	extern int amd_iommu_init_devices(void);
20	19	extern void amd_iommu_uninit_devices(void);
21	20	extern void amd_iommu_init_notifier(void);
@@ -117,8 +116,7 @@ void amd_iommu_domain_clr_pt_root(struct protection_domain *domain) · 95 unmodified lines
117	116
118	117
119	118	extern bool translation_pre_enabled(struct amd_iommu *iommu);
120		−extern bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
121		− struct device *dev);
	119	+extern bool amd_iommu_is_attach_deferred(struct device *dev);
122	120	extern int __init add_special_device(u8 type, u8 id, u16 *devid,
123	121	bool cmd_line);
124	122

drivers/iommu/amd/init.c+14−4

@@ -671,7 +671,7 @@ void amd_iommu_restart_event_logging(struct amd_iommu *iommu) · 670 unmodified lines
671	671	* This function resets the command buffer if the IOMMU stopped fetching
672	672	* commands from it.
673	673	*/
674		−void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
	674	+static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
675	675	{
676	676	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
677	677
@@ -990,6 +990,7 @@ static bool copy_device_table(void) · 312 unmodified lines
990	990	get_order(dev_table_size));
991	991	if (old_dev_tbl_cpy == NULL) {
992	992	pr_err("Failed to allocate memory for copying old device table!\n");
	993	+ memunmap(old_devtb);
993	994	return false;
994	995	}
995	996
@@ -1020,6 +1021,7 @@ static bool copy_device_table(void) · 24 unmodified lines
1020	1021	if ((int_ctl != DTE_IRQ_REMAP_INTCTL) \|\|
1021	1022	(int_tab_len != DTE_INTTABLEN)) {
1022	1023	pr_err("Wrong old irq remapping flag: %#x\n", devid);
	1024	+ memunmap(old_devtb);
1023	1025	return false;
1024	1026	}
1025	1027
@@ -1953,9 +1955,11 @@ static int __init amd_iommu_init_pci(void) · 927 unmodified lines
1953	1955
1954	1956	for_each_iommu(iommu) {
1955	1957	ret = iommu_init_pci(iommu);
1956		− if (ret)
1957		− break;
1958		−
	1958	+ if (ret) {
	1959	+ pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n",
	1960	+ iommu->index, ret);
	1961	+ goto out;
	1962	+ }
1959	1963	/* Need to setup range after PCI init */
1960	1964	iommu_set_cwwb_range(iommu);
1961	1965	}
@@ -1971,6 +1975,11 @@ static int __init amd_iommu_init_pci(void) · 9 unmodified lines
1971	1975	* active.
1972	1976	*/
1973	1977	ret = amd_iommu_init_api();
	1978	+ if (ret) {
	1979	+ pr_err("IOMMU: Failed to initialize IOMMU-API interface (error=%d)!\n",
	1980	+ ret);
	1981	+ goto out;
	1982	+ }
1974	1983
1975	1984	init_device_table_dma();
1976	1985
@@ -1980,6 +1989,7 @@ static int __init amd_iommu_init_pci(void) · 3 unmodified lines
1980	1989	if (!ret)
1981	1990	print_iommu_info();
1982	1991
	1992	+out:
1983	1993	return ret;
1984	1994	}
1985	1995

drivers/iommu/amd/iommu.c+12−11

@@ -2221,8 +2221,7 @@ static void amd_iommu_get_resv_regions(struct device *dev, · 2220 unmodified lines
2221	2221	list_add_tail(&region->list, head);
2222	2222	}
2223	2223
2224		−bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
2225		− struct device *dev)
	2224	+bool amd_iommu_is_attach_deferred(struct device *dev)
2226	2225	{
2227	2226	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2228	2227
@@ -2275,13 +2274,6 @@ static int amd_iommu_def_domain_type(struct device *dev) · 46 unmodified lines
2275	2274	const struct iommu_ops amd_iommu_ops = {
2276	2275	.capable = amd_iommu_capable,
2277	2276	.domain_alloc = amd_iommu_domain_alloc,
2278		− .domain_free = amd_iommu_domain_free,
2279		− .attach_dev = amd_iommu_attach_device,
2280		− .detach_dev = amd_iommu_detach_device,
2281		− .map = amd_iommu_map,
2282		− .iotlb_sync_map = amd_iommu_iotlb_sync_map,
2283		− .unmap = amd_iommu_unmap,
2284		− .iova_to_phys = amd_iommu_iova_to_phys,
2285	2277	.probe_device = amd_iommu_probe_device,
2286	2278	.release_device = amd_iommu_release_device,
2287	2279	.probe_finalize = amd_iommu_probe_finalize,
@@ -2290,9 +2282,18 @@ const struct iommu_ops amd_iommu_ops = { · 2 unmodified lines
2290	2282	.put_resv_regions = generic_iommu_put_resv_regions,
2291	2283	.is_attach_deferred = amd_iommu_is_attach_deferred,
2292	2284	.pgsize_bitmap = AMD_IOMMU_PGSIZES,
2293		− .flush_iotlb_all = amd_iommu_flush_iotlb_all,
2294		− .iotlb_sync = amd_iommu_iotlb_sync,
2295	2285	.def_domain_type = amd_iommu_def_domain_type,
	2286	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	2287	+ .attach_dev = amd_iommu_attach_device,
	2288	+ .detach_dev = amd_iommu_detach_device,
	2289	+ .map = amd_iommu_map,
	2290	+ .unmap = amd_iommu_unmap,
	2291	+ .iotlb_sync_map = amd_iommu_iotlb_sync_map,
	2292	+ .iova_to_phys = amd_iommu_iova_to_phys,
	2293	+ .flush_iotlb_all = amd_iommu_flush_iotlb_all,
	2294	+ .iotlb_sync = amd_iommu_iotlb_sync,
	2295	+ .free = amd_iommu_domain_free,
	2296	+ }
2296	2297	};
2297	2298
2298	2299	/*****************************************************************************

drivers/iommu/amd/iommu_v2.c+18−19

@@ -24,7 +24,6 @@ · 23 unmodified lines
24	24	MODULE_LICENSE("GPL v2");
25	25	MODULE_AUTHOR("Joerg Roedel <jroedel@suse.de>");
26	26
27		−#define MAX_DEVICES 0x10000
28	27	#define PRI_QUEUE_SIZE 512
29	28
30	29	struct pri_queue {
@@ -71,7 +70,6 @@ struct fault { · 40 unmodified lines
71	70	struct pasid_state *state;
72	71	struct mm_struct *mm;
73	72	u64 address;
74		− u16 devid;
75	73	u32 pasid;
76	74	u16 tag;
77	75	u16 finish;
@@ -125,6 +123,15 @@ static void free_device_state(struct device_state *dev_state) · 47 unmodified lines
125	123	{
126	124	struct iommu_group *group;
127	125
	126	+ /* Get rid of any remaining pasid states */
	127	+ free_pasid_states(dev_state);
	128	+
	129	+ /*
	130	+ * Wait until the last reference is dropped before freeing
	131	+ * the device state.
	132	+ */
	133	+ wait_event(dev_state->wq, !atomic_read(&dev_state->count));
	134	+
128	135	/*
129	136	* First detach device from domain - No more PRI requests will arrive
130	137	* from that device after it is unbound from the IOMMUv2 domain.
@@ -537,7 +544,7 @@ static int ppr_notifier(struct notifier_block nb, unsigned long e, void data) · 406 unmodified lines
537	544	ret = NOTIFY_DONE;
538	545
539	546	/* In kdump kernel pci dev is not initialized yet -> send INVALID */
540		− if (amd_iommu_is_attach_deferred(NULL, &pdev->dev)) {
	547	+ if (amd_iommu_is_attach_deferred(&pdev->dev)) {
541	548	amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
542	549	PPR_INVALID, tag);
543	550	goto out;
@@ -850,15 +857,7 @@ void amd_iommu_free_device(struct pci_dev *pdev) · 306 unmodified lines
850	857
851	858	spin_unlock_irqrestore(&state_lock, flags);
852	859
853		− /* Get rid of any remaining pasid states */
854		− free_pasid_states(dev_state);
855		−
856	860	put_device_state(dev_state);
857		− /*
858		− * Wait until the last reference is dropped before freeing
859		− * the device state.
860		− */
861		− wait_event(dev_state->wq, !atomic_read(&dev_state->count));
862	861	free_device_state(dev_state);
863	862	}
864	863	EXPORT_SYMBOL(amd_iommu_free_device);
@@ -955,8 +954,8 @@ out: · 90 unmodified lines
955	954
956	955	static void __exit amd_iommu_v2_exit(void)
957	956	{
958		− struct device_state *dev_state;
959		− int i;
	957	+ struct device_state dev_state, next;
	958	+ unsigned long flags;
960	959
961	960	if (!amd_iommu_v2_supported())
962	961	return;
@@ -969,18 +968,18 @@ static void __exit amd_iommu_v2_exit(void) · 6 unmodified lines
969	968	* The loop below might call flush_workqueue(), so call
970	969	* destroy_workqueue() after it
971	970	*/
972		− for (i = 0; i < MAX_DEVICES; ++i) {
973		− dev_state = get_device_state(i);
974		−
975		− if (dev_state == NULL)
976		− continue;
	971	+ spin_lock_irqsave(&state_lock, flags);
977	972
	973	+ list_for_each_entry_safe(dev_state, next, &state_list, list) {
978	974	WARN_ON_ONCE(1);
979	975
980	976	put_device_state(dev_state);
981		− amd_iommu_free_device(dev_state->pdev);
	977	+ list_del(&dev_state->list);
	978	+ free_device_state(dev_state);
982	979	}
983	980
	981	+ spin_unlock_irqrestore(&state_lock, flags);
	982	+
984	983	destroy_workqueue(iommu_wq);
985	984	}
986	985

drivers/iommu/apple-dart.c+11−9

@@ -765,15 +765,6 @@ static void apple_dart_get_resv_regions(struct device *dev, · 764 unmodified lines
765	765
766	766	static const struct iommu_ops apple_dart_iommu_ops = {
767	767	.domain_alloc = apple_dart_domain_alloc,
768		− .domain_free = apple_dart_domain_free,
769		− .attach_dev = apple_dart_attach_dev,
770		− .detach_dev = apple_dart_detach_dev,
771		− .map_pages = apple_dart_map_pages,
772		− .unmap_pages = apple_dart_unmap_pages,
773		− .flush_iotlb_all = apple_dart_flush_iotlb_all,
774		− .iotlb_sync = apple_dart_iotlb_sync,
775		− .iotlb_sync_map = apple_dart_iotlb_sync_map,
776		− .iova_to_phys = apple_dart_iova_to_phys,
777	768	.probe_device = apple_dart_probe_device,
778	769	.release_device = apple_dart_release_device,
779	770	.device_group = apple_dart_device_group,
@@ -782,6 +773,17 @@ static const struct iommu_ops apple_dart_iommu_ops = { · 2 unmodified lines
782	773	.get_resv_regions = apple_dart_get_resv_regions,
783	774	.put_resv_regions = generic_iommu_put_resv_regions,
784	775	.pgsize_bitmap = -1UL, /* Restricted during dart probe */
	776	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	777	+ .attach_dev = apple_dart_attach_dev,
	778	+ .detach_dev = apple_dart_detach_dev,
	779	+ .map_pages = apple_dart_map_pages,
	780	+ .unmap_pages = apple_dart_unmap_pages,
	781	+ .flush_iotlb_all = apple_dart_flush_iotlb_all,
	782	+ .iotlb_sync = apple_dart_iotlb_sync,
	783	+ .iotlb_sync_map = apple_dart_iotlb_sync_map,
	784	+ .iova_to_phys = apple_dart_iova_to_phys,
	785	+ .free = apple_dart_domain_free,
	786	+ }
785	787	};
786	788
787	789	static irqreturn_t apple_dart_irq(int irq, void *dev)

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c+18−27

@@ -1558,6 +1558,7 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) · 1557 unmodified lines
1558	1558	dev_info(smmu->dev, "\t0x%016llx\n",
1559	1559	(unsigned long long)evt[i]);
1560	1560
	1561	+ cond_resched();
1561	1562	}
1562	1563
1563	1564	/*
@@ -2841,17 +2842,9 @@ static int arm_smmu_dev_disable_feature(struct device *dev, · 1277 unmodified lines
2841	2842	static struct iommu_ops arm_smmu_ops = {
2842	2843	.capable = arm_smmu_capable,
2843	2844	.domain_alloc = arm_smmu_domain_alloc,
2844		− .domain_free = arm_smmu_domain_free,
2845		− .attach_dev = arm_smmu_attach_dev,
2846		− .map_pages = arm_smmu_map_pages,
2847		− .unmap_pages = arm_smmu_unmap_pages,
2848		− .flush_iotlb_all = arm_smmu_flush_iotlb_all,
2849		− .iotlb_sync = arm_smmu_iotlb_sync,
2850		− .iova_to_phys = arm_smmu_iova_to_phys,
2851	2845	.probe_device = arm_smmu_probe_device,
2852	2846	.release_device = arm_smmu_release_device,
2853	2847	.device_group = arm_smmu_device_group,
2854		− .enable_nesting = arm_smmu_enable_nesting,
2855	2848	.of_xlate = arm_smmu_of_xlate,
2856	2849	.get_resv_regions = arm_smmu_get_resv_regions,
2857	2850	.put_resv_regions = generic_iommu_put_resv_regions,
@@ -2865,6 +2858,16 @@ static struct iommu_ops arm_smmu_ops = { · 7 unmodified lines
2865	2858	.page_response = arm_smmu_page_response,
2866	2859	.pgsize_bitmap = -1UL, /* Restricted during device attach */
2867	2860	.owner = THIS_MODULE,
	2861	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	2862	+ .attach_dev = arm_smmu_attach_dev,
	2863	+ .map_pages = arm_smmu_map_pages,
	2864	+ .unmap_pages = arm_smmu_unmap_pages,
	2865	+ .flush_iotlb_all = arm_smmu_flush_iotlb_all,
	2866	+ .iotlb_sync = arm_smmu_iotlb_sync,
	2867	+ .iova_to_phys = arm_smmu_iova_to_phys,
	2868	+ .enable_nesting = arm_smmu_enable_nesting,
	2869	+ .free = arm_smmu_domain_free,
	2870	+ }
2868	2871	};
2869	2872
2870	2873	/* Probing and initialisation functions */
@@ -2911,32 +2914,20 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, · 40 unmodified lines
2911	2914	return 0;
2912	2915	}
2913	2916
2914		−static void arm_smmu_cmdq_free_bitmap(void *data)
2915		−{
2916		− unsigned long *bitmap = data;
2917		− bitmap_free(bitmap);
2918		−}
2919		−
2920	2917	static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
2921	2918	{
2922		− int ret = 0;
2923	2919	struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
2924	2920	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
2925		− atomic_long_t *bitmap;
2926	2921
2927	2922	atomic_set(&cmdq->owner_prod, 0);
2928	2923	atomic_set(&cmdq->lock, 0);
2929	2924
2930		− bitmap = (atomic_long_t *)bitmap_zalloc(nents, GFP_KERNEL);
2931		− if (!bitmap) {
2932		− dev_err(smmu->dev, "failed to allocate cmdq bitmap\n");
2933		− ret = -ENOMEM;
2934		− } else {
2935		− cmdq->valid_map = bitmap;
2936		− devm_add_action(smmu->dev, arm_smmu_cmdq_free_bitmap, bitmap);
2937		− }
	2925	+ cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
	2926	+ GFP_KERNEL);
	2927	+ if (!cmdq->valid_map)
	2928	+ return -ENOMEM;
2938	2929
2939		− return ret;
	2930	+ return 0;
2940	2931	}
2941	2932
2942	2933	static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
@@ -2981,10 +2972,10 @@ static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) · 38 unmodified lines
2981	2972	{
2982	2973	unsigned int i;
2983	2974	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2984		− size_t size = sizeof(cfg->l1_desc) cfg->num_l1_ents;
2985	2975	void *strtab = smmu->strtab_cfg.strtab;
2986	2976
2987		− cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
	2977	+ cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents,
	2978	+ sizeof(*cfg->l1_desc), GFP_KERNEL);
2988	2979	if (!cfg->l1_desc)
2989	2980	return -ENOMEM;
2990	2981

drivers/iommu/arm/arm-smmu/arm-smmu.c+55−58

@@ -807,7 +807,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, · 806 unmodified lines
807	807	* Request context fault interrupt. Do this last to avoid the
808	808	* handler seeing a half-initialised domain state.
809	809	*/
810		− irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
	810	+ irq = smmu->irqs[cfg->irptndx];
811	811
812	812	if (smmu->impl && smmu->impl->context_fault)
813	813	context_fault = smmu->impl->context_fault;
@@ -858,7 +858,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) · 44 unmodified lines
858	858	arm_smmu_write_context_bank(smmu, cfg->cbndx);
859	859
860	860	if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
861		− irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
	861	+ irq = smmu->irqs[cfg->irptndx];
862	862	devm_free_irq(smmu->dev, irq, domain);
863	863	}
864	864
@@ -1583,25 +1583,27 @@ static int arm_smmu_def_domain_type(struct device *dev) · 718 unmodified lines
1583	1583	static struct iommu_ops arm_smmu_ops = {
1584	1584	.capable = arm_smmu_capable,
1585	1585	.domain_alloc = arm_smmu_domain_alloc,
1586		− .domain_free = arm_smmu_domain_free,
1587		− .attach_dev = arm_smmu_attach_dev,
1588		− .map_pages = arm_smmu_map_pages,
1589		− .unmap_pages = arm_smmu_unmap_pages,
1590		− .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1591		− .iotlb_sync = arm_smmu_iotlb_sync,
1592		− .iova_to_phys = arm_smmu_iova_to_phys,
1593	1586	.probe_device = arm_smmu_probe_device,
1594	1587	.release_device = arm_smmu_release_device,
1595	1588	.probe_finalize = arm_smmu_probe_finalize,
1596	1589	.device_group = arm_smmu_device_group,
1597		− .enable_nesting = arm_smmu_enable_nesting,
1598		− .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1599	1590	.of_xlate = arm_smmu_of_xlate,
1600	1591	.get_resv_regions = arm_smmu_get_resv_regions,
1601	1592	.put_resv_regions = generic_iommu_put_resv_regions,
1602	1593	.def_domain_type = arm_smmu_def_domain_type,
1603	1594	.pgsize_bitmap = -1UL, /* Restricted during device attach */
1604	1595	.owner = THIS_MODULE,
	1596	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	1597	+ .attach_dev = arm_smmu_attach_dev,
	1598	+ .map_pages = arm_smmu_map_pages,
	1599	+ .unmap_pages = arm_smmu_unmap_pages,
	1600	+ .flush_iotlb_all = arm_smmu_flush_iotlb_all,
	1601	+ .iotlb_sync = arm_smmu_iotlb_sync,
	1602	+ .iova_to_phys = arm_smmu_iova_to_phys,
	1603	+ .enable_nesting = arm_smmu_enable_nesting,
	1604	+ .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
	1605	+ .free = arm_smmu_domain_free,
	1606	+ }
1605	1607	};
1606	1608
1607	1609	static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
@@ -1951,8 +1953,8 @@ static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) · 343 unmodified lines
1951	1953	return ret;
1952	1954	}
1953	1955
1954		−static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1955		− struct arm_smmu_device *smmu)
	1956	+static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
	1957	+ u32 global_irqs, u32 pmu_irqs)
1956	1958	{
1957	1959	struct device *dev = smmu->dev;
1958	1960	struct acpi_iort_node *node =
@@ -1968,7 +1970,8 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev, · 9 unmodified lines
1968	1970	return ret;
1969	1971
1970	1972	/* Ignore the configuration access interrupt */
1971		− smmu->num_global_irqs = 1;
	1973	+ *global_irqs = 1;
	1974	+ *pmu_irqs = 0;
1972	1975
1973	1976	if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1974	1977	smmu->features \|= ARM_SMMU_FEAT_COHERENT_WALK;
@@ -1976,25 +1979,24 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev, · 1 unmodified line
1976	1979	return 0;
1977	1980	}
1978	1981	#else
1979		−static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1980		− struct arm_smmu_device *smmu)
	1982	+static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
	1983	+ u32 global_irqs, u32 pmu_irqs)
1981	1984	{
1982	1985	return -ENODEV;
1983	1986	}
1984	1987	#endif
1985	1988
1986		−static int arm_smmu_device_dt_probe(struct platform_device *pdev,
1987		− struct arm_smmu_device *smmu)
	1989	+static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
	1990	+ u32 global_irqs, u32 pmu_irqs)
1988	1991	{
1989	1992	const struct arm_smmu_match_data *data;
1990		− struct device *dev = &pdev->dev;
	1993	+ struct device *dev = smmu->dev;
1991	1994	bool legacy_binding;
1992	1995
1993		− if (of_property_read_u32(dev->of_node, "#global-interrupts",
1994		− &smmu->num_global_irqs)) {
1995		− dev_err(dev, "missing #global-interrupts property\n");
1996		− return -ENODEV;
1997		− }
	1996	+ if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
	1997	+ return dev_err_probe(dev, -ENODEV,
	1998	+ "missing #global-interrupts property\n");
	1999	+ *pmu_irqs = 0;
1998	2000
1999	2001	data = of_device_get_match_data(dev);
2000	2002	smmu->version = data->version;
@@ -2073,6 +2075,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 72 unmodified lines
2073	2075	struct arm_smmu_device *smmu;
2074	2076	struct device *dev = &pdev->dev;
2075	2077	int num_irqs, i, err;
	2078	+ u32 global_irqs, pmu_irqs;
2076	2079	irqreturn_t (global_fault)(int irq, void dev);
2077	2080
2078	2081	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
@@ -2083,10 +2086,9 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 4 unmodified lines
2083	2086	smmu->dev = dev;
2084	2087
2085	2088	if (dev->of_node)
2086		− err = arm_smmu_device_dt_probe(pdev, smmu);
	2089	+ err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2087	2090	else
2088		− err = arm_smmu_device_acpi_probe(pdev, smmu);
2089		−
	2091	+ err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2090	2092	if (err)
2091	2093	return err;
2092	2094
@@ -2105,31 +2107,25 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 12 unmodified lines
2105	2107	if (IS_ERR(smmu))
2106	2108	return PTR_ERR(smmu);
2107	2109
2108		− num_irqs = 0;
2109		− while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2110		− num_irqs++;
2111		− if (num_irqs > smmu->num_global_irqs)
2112		− smmu->num_context_irqs++;
2113		− }
	2110	+ num_irqs = platform_irq_count(pdev);
2114	2111
2115		− if (!smmu->num_context_irqs) {
2116		− dev_err(dev, "found %d interrupts but expected at least %d\n",
2117		− num_irqs, smmu->num_global_irqs + 1);
2118		− return -ENODEV;
2119		− }
	2112	+ smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
	2113	+ if (smmu->num_context_irqs <= 0)
	2114	+ return dev_err_probe(dev, -ENODEV,
	2115	+ "found %d interrupts but expected at least %d\n",
	2116	+ num_irqs, global_irqs + pmu_irqs + 1);
2120	2117
2121		− smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2122		− GFP_KERNEL);
2123		− if (!smmu->irqs) {
2124		− dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2125		− return -ENOMEM;
2126		− }
	2118	+ smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
	2119	+ sizeof(*smmu->irqs), GFP_KERNEL);
	2120	+ if (!smmu->irqs)
	2121	+ return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
	2122	+ smmu->num_context_irqs);
2127	2123
2128		− for (i = 0; i < num_irqs; ++i) {
2129		− int irq = platform_get_irq(pdev, i);
	2124	+ for (i = 0; i < smmu->num_context_irqs; i++) {
	2125	+ int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2130	2126
2131	2127	if (irq < 0)
2132		− return -ENODEV;
	2128	+ return irq;
2133	2129	smmu->irqs[i] = irq;
2134	2130	}
2135	2131
@@ -2165,17 +2161,18 @@ static int arm_smmu_device_probe(struct platform_device *pdev) · 29 unmodified lines
2165	2161	else
2166	2162	global_fault = arm_smmu_global_fault;
2167	2163
2168		− for (i = 0; i < smmu->num_global_irqs; ++i) {
2169		− err = devm_request_irq(smmu->dev, smmu->irqs[i],
2170		− global_fault,
2171		− IRQF_SHARED,
2172		− "arm-smmu global fault",
2173		− smmu);
2174		− if (err) {
2175		− dev_err(dev, "failed to request global IRQ %d (%u)\n",
2176		− i, smmu->irqs[i]);
2177		− return err;
2178		− }
	2164	+ for (i = 0; i < global_irqs; i++) {
	2165	+ int irq = platform_get_irq(pdev, i);
	2166	+
	2167	+ if (irq < 0)
	2168	+ return irq;
	2169	+
	2170	+ err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
	2171	+ "arm-smmu global fault", smmu);
	2172	+ if (err)
	2173	+ return dev_err_probe(dev, err,
	2174	+ "failed to request global IRQ %d (%u)\n",
	2175	+ i, irq);
2179	2176	}
2180	2177
2181	2178	err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,

drivers/iommu/arm/arm-smmu/arm-smmu.h+2−3

@@ -318,11 +318,10 @@ struct arm_smmu_device { · 317 unmodified lines
318	318	unsigned long pa_size;
319	319	unsigned long pgsize_bitmap;
320	320
321		− u32 num_global_irqs;
322		− u32 num_context_irqs;
	321	+ int num_context_irqs;
	322	+ int num_clks;
323	323	unsigned int *irqs;
324	324	struct clk_bulk_data *clks;
325		− int num_clks;
326	325
327	326	spinlock_t global_sync_lock;
328	327

drivers/iommu/arm/arm-smmu/qcom_iommu.c+17−11

@@ -590,19 +590,21 @@ static int qcom_iommu_of_xlate(struct device dev, struct of_phandle_args args) · 589 unmodified lines
590	590	static const struct iommu_ops qcom_iommu_ops = {
591	591	.capable = qcom_iommu_capable,
592	592	.domain_alloc = qcom_iommu_domain_alloc,
593		− .domain_free = qcom_iommu_domain_free,
594		− .attach_dev = qcom_iommu_attach_dev,
595		− .detach_dev = qcom_iommu_detach_dev,
596		− .map = qcom_iommu_map,
597		− .unmap = qcom_iommu_unmap,
598		− .flush_iotlb_all = qcom_iommu_flush_iotlb_all,
599		− .iotlb_sync = qcom_iommu_iotlb_sync,
600		− .iova_to_phys = qcom_iommu_iova_to_phys,
601	593	.probe_device = qcom_iommu_probe_device,
602	594	.release_device = qcom_iommu_release_device,
603	595	.device_group = generic_device_group,
604	596	.of_xlate = qcom_iommu_of_xlate,
605	597	.pgsize_bitmap = SZ_4K \| SZ_64K \| SZ_1M \| SZ_16M,
	598	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	599	+ .attach_dev = qcom_iommu_attach_dev,
	600	+ .detach_dev = qcom_iommu_detach_dev,
	601	+ .map = qcom_iommu_map,
	602	+ .unmap = qcom_iommu_unmap,
	603	+ .flush_iotlb_all = qcom_iommu_flush_iotlb_all,
	604	+ .iotlb_sync = qcom_iommu_iotlb_sync,
	605	+ .iova_to_phys = qcom_iommu_iova_to_phys,
	606	+ .free = qcom_iommu_domain_free,
	607	+ }
606	608	};
607	609
608	610	static int qcom_iommu_sec_ptbl_init(struct device *dev)
@@ -827,20 +829,20 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) · 218 unmodified lines
827	829	ret = devm_of_platform_populate(dev);
828	830	if (ret) {
829	831	dev_err(dev, "Failed to populate iommu contexts\n");
830		− return ret;
	832	+ goto err_pm_disable;
831	833	}
832	834
833	835	ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
834	836	dev_name(dev));
835	837	if (ret) {
836	838	dev_err(dev, "Failed to register iommu in sysfs\n");
837		− return ret;
	839	+ goto err_pm_disable;
838	840	}
839	841
840	842	ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);
841	843	if (ret) {
842	844	dev_err(dev, "Failed to register iommu\n");
843		− return ret;
	845	+ goto err_pm_disable;
844	846	}
845	847
846	848	bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
@@ -852,6 +854,10 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) · 5 unmodified lines
852	854	}
853	855
854	856	return 0;
	857	+
	858	+err_pm_disable:
	859	+ pm_runtime_disable(dev);
	860	+ return ret;
855	861	}
856	862
857	863	static int qcom_iommu_device_remove(struct platform_device *pdev)

drivers/iommu/dma-iommu.c+4−0

@@ -525,6 +525,7 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, · 524 unmodified lines
525	525	struct iommu_dma_cookie *cookie = domain->iova_cookie;
526	526	unsigned long order, base_pfn;
527	527	struct iova_domain *iovad;
	528	+ int ret;
528	529
529	530	if (!cookie \|\| cookie->type != IOMMU_DMA_IOVA_COOKIE)
530	531	return -EINVAL;
@@ -559,6 +560,9 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, · 28 unmodified lines
559	560	}
560	561
561	562	init_iova_domain(iovad, 1UL << order, base_pfn);
	563	+ ret = iova_domain_init_rcaches(iovad);
	564	+ if (ret)
	565	+ return ret;
562	566
563	567	/* If the FQ fails we can simply fall back to strict mode */
564	568	if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain))

drivers/iommu/exynos-iommu.c+8−6

@@ -1309,17 +1309,19 @@ static int exynos_iommu_of_xlate(struct device *dev, · 1308 unmodified lines
1309	1309
1310	1310	static const struct iommu_ops exynos_iommu_ops = {
1311	1311	.domain_alloc = exynos_iommu_domain_alloc,
1312		− .domain_free = exynos_iommu_domain_free,
1313		− .attach_dev = exynos_iommu_attach_device,
1314		− .detach_dev = exynos_iommu_detach_device,
1315		− .map = exynos_iommu_map,
1316		− .unmap = exynos_iommu_unmap,
1317		− .iova_to_phys = exynos_iommu_iova_to_phys,
1318	1312	.device_group = generic_device_group,
1319	1313	.probe_device = exynos_iommu_probe_device,
1320	1314	.release_device = exynos_iommu_release_device,
1321	1315	.pgsize_bitmap = SECT_SIZE \| LPAGE_SIZE \| SPAGE_SIZE,
1322	1316	.of_xlate = exynos_iommu_of_xlate,
	1317	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	1318	+ .attach_dev = exynos_iommu_attach_device,
	1319	+ .detach_dev = exynos_iommu_detach_device,
	1320	+ .map = exynos_iommu_map,
	1321	+ .unmap = exynos_iommu_unmap,
	1322	+ .iova_to_phys = exynos_iommu_iova_to_phys,
	1323	+ .free = exynos_iommu_domain_free,
	1324	+ }
1323	1325	};
1324	1326
1325	1327	static int __init exynos_iommu_init(void)

drivers/iommu/fsl_pamu_domain.c+6−4

@@ -453,13 +453,15 @@ static void fsl_pamu_release_device(struct device *dev) · 452 unmodified lines
453	453	static const struct iommu_ops fsl_pamu_ops = {
454	454	.capable = fsl_pamu_capable,
455	455	.domain_alloc = fsl_pamu_domain_alloc,
456		− .domain_free = fsl_pamu_domain_free,
457		− .attach_dev = fsl_pamu_attach_device,
458		− .detach_dev = fsl_pamu_detach_device,
459		− .iova_to_phys = fsl_pamu_iova_to_phys,
460	456	.probe_device = fsl_pamu_probe_device,
461	457	.release_device = fsl_pamu_release_device,
462	458	.device_group = fsl_pamu_device_group,
	459	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	460	+ .attach_dev = fsl_pamu_attach_device,
	461	+ .detach_dev = fsl_pamu_detach_device,
	462	+ .iova_to_phys = fsl_pamu_iova_to_phys,
	463	+ .free = fsl_pamu_domain_free,
	464	+ }
463	465	};
464	466
465	467	int __init pamu_domain_init(void)

drivers/iommu/intel/debugfs.c+3−3

@@ -344,15 +344,15 @@ static void pgtable_walk_level(struct seq_file m, struct dma_pte pde, · 343 unmodified lines
344	344
345	345	static int show_device_domain_translation(struct device dev, void data)
346	346	{
347		− struct dmar_domain *domain = find_domain(dev);
	347	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
	348	+ struct dmar_domain *domain = info->domain;
348	349	struct seq_file *m = data;
349	350	u64 path[6] = { 0 };
350	351
351	352	if (!domain)
352	353	return 0;
353	354
354		− seq_printf(m, "Device %s with pasid %d @0x%llx\n",
355		− dev_name(dev), domain->default_pasid,
	355	+ seq_printf(m, "Device %s @0x%llx\n", dev_name(dev),
356	356	(u64)virt_to_phys(domain->pgd));
357	357	seq_puts(m, "IOVA_PFN\t\tPML5E\t\t\tPML4E\t\t\tPDPE\t\t\tPDE\t\t\tPTE\n");
358	358

drivers/iommu/intel/dmar.c+2−3

@@ -66,8 +66,6 @@ static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)]; · 65 unmodified lines
66	66	static int alloc_iommu(struct dmar_drhd_unit *drhd);
67	67	static void free_iommu(struct intel_iommu *iommu);
68	68
69		−extern const struct iommu_ops intel_iommu_ops;
70		−
71	69	static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
72	70	{
73	71	/*
@@ -789,7 +787,8 @@ static int __init dmar_acpi_dev_scope_init(void) · 715 unmodified lines
789	787	andd->device_name);
790	788	continue;
791	789	}
792		− if (acpi_bus_get_device(h, &adev)) {
	790	+ adev = acpi_fetch_acpi_dev(h);
	791	+ if (!adev) {
793	792	pr_err("Failed to get device for ACPI object %s\n",
794	793	andd->device_name);
795	794	continue;

drivers/iommu/intel/iommu.c+133−866

@@ -13,38 +13,18 @@ · 12 unmodified lines
13	13	#define pr_fmt(fmt) "DMAR: " fmt
14	14	#define dev_fmt(fmt) pr_fmt(fmt)
15	15
16		−#include <linux/init.h>
17		−#include <linux/bitmap.h>
18		−#include <linux/debugfs.h>
19		−#include <linux/export.h>
20		−#include <linux/slab.h>
21		−#include <linux/irq.h>
22		−#include <linux/interrupt.h>
23		−#include <linux/spinlock.h>
24		−#include <linux/pci.h>
25		−#include <linux/dmar.h>
26		−#include <linux/dma-map-ops.h>
27		−#include <linux/mempool.h>
28		−#include <linux/memory.h>
29		−#include <linux/cpu.h>
30		−#include <linux/timer.h>
31		−#include <linux/io.h>
32		−#include <linux/iova.h>
33		−#include <linux/iommu.h>
	16	+#include <linux/crash_dump.h>
	17	+#include <linux/dma-direct.h>
34	18	#include <linux/dma-iommu.h>
	19	+#include <linux/dmi.h>
35	20	#include <linux/intel-iommu.h>
36	21	#include <linux/intel-svm.h>
	22	+#include <linux/memory.h>
	23	+#include <linux/pci.h>
	24	+#include <linux/pci-ats.h>
	25	+#include <linux/spinlock.h>
37	26	#include <linux/syscore_ops.h>
38	27	#include <linux/tboot.h>
39		−#include <linux/dmi.h>
40		−#include <linux/pci-ats.h>
41		−#include <linux/memblock.h>
42		−#include <linux/dma-direct.h>
43		−#include <linux/crash_dump.h>
44		−#include <linux/numa.h>
45		−#include <asm/irq_remapping.h>
46		−#include <asm/cacheflush.h>
47		−#include <asm/iommu.h>
48	28
49	29	#include "../irq_remapping.h"
50	30	#include "../iommu-sva-lib.h"
@@ -316,14 +296,9 @@ static LIST_HEAD(dmar_satc_units); · 265 unmodified lines
316	296	/* bitmap for indexing intel_iommus */
317	297	static int g_num_of_iommus;
318	298
319		−static void domain_exit(struct dmar_domain *domain);
320	299	static void domain_remove_dev_info(struct dmar_domain *domain);
321	300	static void dmar_remove_one_dev_info(struct device *dev);
322	301	static void __dmar_remove_one_dev_info(struct device_domain_info *info);
323		−static int intel_iommu_attach_device(struct iommu_domain *domain,
324		− struct device *dev);
325		−static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
326		− dma_addr_t iova);
327	302
328	303	int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
329	304	int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
@@ -342,21 +317,6 @@ static int iommu_skip_te_disable; · 12 unmodified lines
342	317	int intel_iommu_gfx_mapped;
343	318	EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
344	319
345		−#define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
346		−struct device_domain_info get_domain_info(struct device dev)
347		−{
348		− struct device_domain_info *info;
349		−
350		− if (!dev)
351		− return NULL;
352		−
353		− info = dev_iommu_priv_get(dev);
354		− if (unlikely(info == DEFER_DEVICE_DOMAIN_INFO))
355		− return NULL;
356		−
357		− return info;
358		−}
359		−
360	320	DEFINE_SPINLOCK(device_domain_lock);
361	321	static LIST_HEAD(device_domain_list);
362	322
@@ -452,39 +412,6 @@ static int __init intel_iommu_setup(char *str) · 89 unmodified lines
452	412	}
453	413	__setup("intel_iommu=", intel_iommu_setup);
454	414
455		−static struct kmem_cache *iommu_domain_cache;
456		−static struct kmem_cache *iommu_devinfo_cache;
457		−
458		−static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
459		−{
460		− struct dmar_domain **domains;
461		− int idx = did >> 8;
462		−
463		− domains = iommu->domains[idx];
464		− if (!domains)
465		− return NULL;
466		−
467		− return domains[did & 0xff];
468		−}
469		−
470		−static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
471		− struct dmar_domain *domain)
472		−{
473		− struct dmar_domain **domains;
474		− int idx = did >> 8;
475		−
476		− if (!iommu->domains[idx]) {
477		− size_t size = 256 * sizeof(struct dmar_domain *);
478		− iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
479		− }
480		−
481		− domains = iommu->domains[idx];
482		− if (WARN_ON(!domains))
483		− return;
484		− else
485		− domains[did & 0xff] = domain;
486		−}
487		−
488	415	void *alloc_pgtable_page(int node)
489	416	{
490	417	struct page *page;
@@ -501,26 +428,6 @@ void free_pgtable_page(void *vaddr) · 10 unmodified lines
501	428	free_page((unsigned long)vaddr);
502	429	}
503	430
504		−static inline void *alloc_domain_mem(void)
505		−{
506		− return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
507		−}
508		−
509		−static void free_domain_mem(void *vaddr)
510		−{
511		− kmem_cache_free(iommu_domain_cache, vaddr);
512		−}
513		−
514		−static inline void * alloc_devinfo_mem(void)
515		−{
516		− return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
517		−}
518		−
519		−static inline void free_devinfo_mem(void *vaddr)
520		−{
521		− kmem_cache_free(iommu_devinfo_cache, vaddr);
522		−}
523		−
524	431	static inline int domain_type_is_si(struct dmar_domain *domain)
525	432	{
526	433	return domain->domain.type == IOMMU_DOMAIN_IDENTITY;
@@ -794,11 +701,6 @@ struct context_entry iommu_context_addr(struct intel_iommu iommu, u8 bus, · 267 unmodified lines
794	701	return &context[devfn];
795	702	}
796	703
797		−static bool attach_deferred(struct device *dev)
798		−{
799		− return dev_iommu_priv_get(dev) == DEFER_DEVICE_DOMAIN_INFO;
800		−}
801		−
802	704	/**
803	705	* is_downstream_to_pci_bridge - test if a device belongs to the PCI
804	706	* sub-hierarchy of a candidate PCI-PCI bridge
@@ -925,7 +827,7 @@ struct intel_iommu device_to_iommu(struct device dev, u8 bus, u8 devfn) · 120 unmodified lines
925	827	}
926	828
927	829	if (pdev && drhd->include_all) {
928		− got_pdev:
	830	+got_pdev:
929	831	if (bus && devfn) {
930	832	*bus = pdev->bus->number;
931	833	*devfn = pdev->devfn;
@@ -934,7 +836,7 @@ struct intel_iommu device_to_iommu(struct device dev, u8 bus, u8 devfn) · 2 unmodified lines
934	836	}
935	837	}
936	838	iommu = NULL;
937		− out:
	839	+out:
938	840	if (iommu_is_dummy(iommu, dev))
939	841	iommu = NULL;
940	842
@@ -1573,18 +1475,6 @@ static void domain_update_iotlb(struct dmar_domain *domain) · 632 unmodified lines
1573	1475	break;
1574	1476	}
1575	1477
1576		− if (!has_iotlb_device) {
1577		− struct subdev_domain_info *sinfo;
1578		−
1579		− list_for_each_entry(sinfo, &domain->subdevices, link_domain) {
1580		− info = get_domain_info(sinfo->pdev);
1581		− if (info && info->ats_enabled) {
1582		− has_iotlb_device = true;
1583		− break;
1584		− }
1585		− }
1586		− }
1587		−
1588	1478	domain->has_iotlb_device = has_iotlb_device;
1589	1479	}
1590	1480
@@ -1682,7 +1572,6 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, · 91 unmodified lines
1682	1572	{
1683	1573	unsigned long flags;
1684	1574	struct device_domain_info *info;
1685		− struct subdev_domain_info *sinfo;
1686	1575
1687	1576	if (!domain->has_iotlb_device)
1688	1577	return;
@@ -1691,27 +1580,9 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, · 2 unmodified lines
1691	1580	list_for_each_entry(info, &domain->devices, link)
1692	1581	__iommu_flush_dev_iotlb(info, addr, mask);
1693	1582
1694		− list_for_each_entry(sinfo, &domain->subdevices, link_domain) {
1695		− info = get_domain_info(sinfo->pdev);
1696		− __iommu_flush_dev_iotlb(info, addr, mask);
1697		− }
1698	1583	spin_unlock_irqrestore(&device_domain_lock, flags);
1699	1584	}
1700	1585
1701		−static void domain_flush_piotlb(struct intel_iommu *iommu,
1702		− struct dmar_domain *domain,
1703		− u64 addr, unsigned long npages, bool ih)
1704		−{
1705		− u16 did = domain->iommu_did[iommu->seq_id];
1706		−
1707		− if (domain->default_pasid)
1708		− qi_flush_piotlb(iommu, did, domain->default_pasid,
1709		− addr, npages, ih);
1710		−
1711		− if (!list_empty(&domain->devices))
1712		− qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, npages, ih);
1713		−}
1714		−
1715	1586	static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1716	1587	struct dmar_domain *domain,
1717	1588	unsigned long pfn, unsigned int pages,
@@ -1727,7 +1598,7 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, · 9 unmodified lines
1727	1598	ih = 1 << 6;
1728	1599
1729	1600	if (domain_use_first_level(domain)) {
1730		− domain_flush_piotlb(iommu, domain, addr, pages, ih);
	1601	+ qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, pages, ih);
1731	1602	} else {
1732	1603	/*
1733	1604	* Fallback to domain selective flush if no PSI support or
@@ -1776,14 +1647,13 @@ static void intel_flush_iotlb_all(struct iommu_domain *domain) · 42 unmodified lines
1776	1647	u16 did = dmar_domain->iommu_did[iommu->seq_id];
1777	1648
1778	1649	if (domain_use_first_level(dmar_domain))
1779		− domain_flush_piotlb(iommu, dmar_domain, 0, -1, 0);
	1650	+ qi_flush_piotlb(iommu, did, PASID_RID2PASID, 0, -1, 0);
1780	1651	else
1781	1652	iommu->flush.flush_iotlb(iommu, did, 0, 0,
1782	1653	DMA_TLB_DSI_FLUSH);
1783	1654
1784	1655	if (!cap_caching_mode(iommu->cap))
1785		− iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1786		− 0, MAX_AGAW_PFN_WIDTH);
	1656	+ iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH);
1787	1657	}
1788	1658	}
1789	1659
@@ -1846,7 +1716,6 @@ static void iommu_disable_translation(struct intel_iommu *iommu) · 56 unmodified lines
1846	1716	static int iommu_init_domains(struct intel_iommu *iommu)
1847	1717	{
1848	1718	u32 ndomains;
1849		− size_t size;
1850	1719
1851	1720	ndomains = cap_ndoms(iommu->cap);
1852	1721	pr_debug("%s: Number of Domains supported <%d>\n",
@@ -1858,24 +1727,6 @@ static int iommu_init_domains(struct intel_iommu *iommu) · 5 unmodified lines
1858	1727	if (!iommu->domain_ids)
1859	1728	return -ENOMEM;
1860	1729
1861		− size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1862		− iommu->domains = kzalloc(size, GFP_KERNEL);
1863		−
1864		− if (iommu->domains) {
1865		− size = 256 * sizeof(struct dmar_domain *);
1866		− iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1867		− }
1868		−
1869		− if (!iommu->domains \|\| !iommu->domains[0]) {
1870		− pr_err("%s: Allocating domain array failed\n",
1871		− iommu->name);
1872		− bitmap_free(iommu->domain_ids);
1873		− kfree(iommu->domains);
1874		− iommu->domain_ids = NULL;
1875		− iommu->domains = NULL;
1876		− return -ENOMEM;
1877		− }
1878		−
1879	1730	/*
1880	1731	* If Caching mode is set, then invalid translations are tagged
1881	1732	* with domain-id 0, hence we need to pre-allocate it. We also
@@ -1902,7 +1753,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) · 20 unmodified lines
1902	1753	struct device_domain_info info, tmp;
1903	1754	unsigned long flags;
1904	1755
1905		− if (!iommu->domains \|\| !iommu->domain_ids)
	1756	+ if (!iommu->domain_ids)
1906	1757	return;
1907	1758
1908	1759	spin_lock_irqsave(&device_domain_lock, flags);
@@ -1923,15 +1774,8 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) · 14 unmodified lines
1923	1774
1924	1775	static void free_dmar_iommu(struct intel_iommu *iommu)
1925	1776	{
1926		− if ((iommu->domains) && (iommu->domain_ids)) {
1927		− int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1928		− int i;
1929		−
1930		− for (i = 0; i < elems; i++)
1931		− kfree(iommu->domains[i]);
1932		− kfree(iommu->domains);
	1777	+ if (iommu->domain_ids) {
1933	1778	bitmap_free(iommu->domain_ids);
1934		− iommu->domains = NULL;
1935	1779	iommu->domain_ids = NULL;
1936	1780	}
1937	1781
@@ -1973,17 +1817,15 @@ static struct dmar_domain *alloc_domain(unsigned int type) · 35 unmodified lines
1973	1817	{
1974	1818	struct dmar_domain *domain;
1975	1819
1976		− domain = alloc_domain_mem();
	1820	+ domain = kzalloc(sizeof(*domain), GFP_KERNEL);
1977	1821	if (!domain)
1978	1822	return NULL;
1979	1823
1980		− memset(domain, 0, sizeof(*domain));
1981	1824	domain->nid = NUMA_NO_NODE;
1982	1825	if (first_level_by_default(type))
1983	1826	domain->flags \|= DOMAIN_FLAG_USE_FIRST_LEVEL;
1984	1827	domain->has_iotlb_device = false;
1985	1828	INIT_LIST_HEAD(&domain->devices);
1986		− INIT_LIST_HEAD(&domain->subdevices);
1987	1829
1988	1830	return domain;
1989	1831	}
@@ -2010,11 +1852,8 @@ static int domain_attach_iommu(struct dmar_domain *domain, · 20 unmodified lines
2010	1852	}
2011	1853
2012	1854	set_bit(num, iommu->domain_ids);
2013		− set_iommu_domain(iommu, num, domain);
2014		−
2015	1855	domain->iommu_did[iommu->seq_id] = num;
2016	1856	domain->nid = iommu->node;
2017		−
2018	1857	domain_update_iommu_cap(domain);
2019	1858	}
2020	1859
@@ -2033,8 +1872,6 @@ static void domain_detach_iommu(struct dmar_domain *domain, · 12 unmodified lines
2033	1872	if (domain->iommu_refcnt[iommu->seq_id] == 0) {
2034	1873	num = domain->iommu_did[iommu->seq_id];
2035	1874	clear_bit(num, iommu->domain_ids);
2036		− set_iommu_domain(iommu, num, NULL);
2037		−
2038	1875	domain_update_iommu_cap(domain);
2039	1876	domain->iommu_did[iommu->seq_id] = 0;
2040	1877	}
@@ -2067,7 +1904,7 @@ static void domain_exit(struct dmar_domain *domain) · 26 unmodified lines
2067	1904	put_pages_list(&freelist);
2068	1905	}
2069	1906
2070		− free_domain_mem(domain);
	1907	+ kfree(domain);
2071	1908	}
2072	1909
2073	1910	/*
@@ -2550,15 +2387,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 · 476 unmodified lines
2550	2387	__iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);
2551	2388	}
2552	2389
2553		−static inline void unlink_domain_info(struct device_domain_info *info)
2554		−{
2555		− assert_spin_locked(&device_domain_lock);
2556		− list_del(&info->link);
2557		− list_del(&info->global);
2558		− if (info->dev)
2559		− dev_iommu_priv_set(info->dev, NULL);
2560		−}
2561		−
2562	2390	static void domain_remove_dev_info(struct dmar_domain *domain)
2563	2391	{
2564	2392	struct device_domain_info info, tmp;
@@ -2570,24 +2398,6 @@ static void domain_remove_dev_info(struct dmar_domain *domain) · 5 unmodified lines
2570	2398	spin_unlock_irqrestore(&device_domain_lock, flags);
2571	2399	}
2572	2400
2573		−struct dmar_domain find_domain(struct device dev)
2574		−{
2575		− struct device_domain_info *info;
2576		−
2577		− if (unlikely(!dev \|\| !dev->iommu))
2578		− return NULL;
2579		−
2580		− if (unlikely(attach_deferred(dev)))
2581		− return NULL;
2582		−
2583		− /* No lock here, assumes no domain exit in normal case */
2584		− info = get_domain_info(dev);
2585		− if (likely(info))
2586		− return info->domain;
2587		−
2588		− return NULL;
2589		−}
2590		−
2591	2401	static inline struct device_domain_info *
2592	2402	dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2593	2403	{
@@ -2648,93 +2458,20 @@ static struct dmar_domain dmar_insert_one_dev_info(struct intel_iommu iommu, · 54 unmodified lines
2648	2458	struct device *dev,
2649	2459	struct dmar_domain *domain)
2650	2460	{
2651		− struct dmar_domain *found = NULL;
2652		− struct device_domain_info *info;
	2461	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
2653	2462	unsigned long flags;
2654	2463	int ret;
2655	2464
2656		− info = alloc_devinfo_mem();
2657		− if (!info)
2658		− return NULL;
2659		−
2660		− if (!dev_is_real_dma_subdevice(dev)) {
2661		− info->bus = bus;
2662		− info->devfn = devfn;
2663		− info->segment = iommu->segment;
2664		− } else {
2665		− struct pci_dev *pdev = to_pci_dev(dev);
2666		−
2667		− info->bus = pdev->bus->number;
2668		− info->devfn = pdev->devfn;
2669		− info->segment = pci_domain_nr(pdev->bus);
2670		− }
2671		−
2672		− info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2673		− info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2674		− info->ats_qdep = 0;
2675		− info->dev = dev;
2676		− info->domain = domain;
2677		− info->iommu = iommu;
2678		− info->pasid_table = NULL;
2679		− info->auxd_enabled = 0;
2680		− INIT_LIST_HEAD(&info->subdevices);
2681		−
2682		− if (dev && dev_is_pci(dev)) {
2683		− struct pci_dev *pdev = to_pci_dev(info->dev);
2684		−
2685		− if (ecap_dev_iotlb_support(iommu->ecap) &&
2686		− pci_ats_supported(pdev) &&
2687		− dmar_find_matched_atsr_unit(pdev))
2688		− info->ats_supported = 1;
2689		−
2690		− if (sm_supported(iommu)) {
2691		− if (pasid_supported(iommu)) {
2692		− int features = pci_pasid_features(pdev);
2693		− if (features >= 0)
2694		− info->pasid_supported = features \| 1;
2695		− }
2696		−
2697		− if (info->ats_supported && ecap_prs(iommu->ecap) &&
2698		− pci_pri_supported(pdev))
2699		− info->pri_supported = 1;
2700		− }
2701		− }
2702		−
2703	2465	spin_lock_irqsave(&device_domain_lock, flags);
2704		− if (dev)
2705		− found = find_domain(dev);
2706		−
2707		− if (!found) {
2708		− struct device_domain_info *info2;
2709		− info2 = dmar_search_domain_by_dev_info(info->segment, info->bus,
2710		− info->devfn);
2711		− if (info2) {
2712		− found = info2->domain;
2713		− info2->dev = dev;
2714		− }
2715		− }
2716		−
2717		− if (found) {
2718		− spin_unlock_irqrestore(&device_domain_lock, flags);
2719		− free_devinfo_mem(info);
2720		− /* Caller must free the original domain */
2721		− return found;
2722		− }
2723		−
	2466	+ info->domain = domain;
2724	2467	spin_lock(&iommu->lock);
2725	2468	ret = domain_attach_iommu(domain, iommu);
2726	2469	spin_unlock(&iommu->lock);
2727		−
2728	2470	if (ret) {
2729	2471	spin_unlock_irqrestore(&device_domain_lock, flags);
2730		− free_devinfo_mem(info);
2731	2472	return NULL;
2732	2473	}
2733		−
2734	2474	list_add(&info->link, &domain->devices);
2735		− list_add(&info->global, &device_domain_list);
2736		− if (dev)
2737		− dev_iommu_priv_set(dev, info);
2738	2475	spin_unlock_irqrestore(&device_domain_lock, flags);
2739	2476
2740	2477	/* PASID table is mandatory for a PCI device in scalable mode. */
@@ -3460,70 +3197,6 @@ error: · 719 unmodified lines
3460	3197	return ret;
3461	3198	}
3462	3199
3463		−static inline int iommu_domain_cache_init(void)
3464		−{
3465		− int ret = 0;
3466		−
3467		− iommu_domain_cache = kmem_cache_create("iommu_domain",
3468		− sizeof(struct dmar_domain),
3469		− 0,
3470		− SLAB_HWCACHE_ALIGN,
3471		−
3472		− NULL);
3473		− if (!iommu_domain_cache) {
3474		− pr_err("Couldn't create iommu_domain cache\n");
3475		− ret = -ENOMEM;
3476		− }
3477		−
3478		− return ret;
3479		−}
3480		−
3481		−static inline int iommu_devinfo_cache_init(void)
3482		−{
3483		− int ret = 0;
3484		−
3485		− iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3486		− sizeof(struct device_domain_info),
3487		− 0,
3488		− SLAB_HWCACHE_ALIGN,
3489		− NULL);
3490		− if (!iommu_devinfo_cache) {
3491		− pr_err("Couldn't create devinfo cache\n");
3492		− ret = -ENOMEM;
3493		− }
3494		−
3495		− return ret;
3496		−}
3497		−
3498		−static int __init iommu_init_mempool(void)
3499		−{
3500		− int ret;
3501		− ret = iova_cache_get();
3502		− if (ret)
3503		− return ret;
3504		−
3505		− ret = iommu_domain_cache_init();
3506		− if (ret)
3507		− goto domain_error;
3508		−
3509		− ret = iommu_devinfo_cache_init();
3510		− if (!ret)
3511		− return ret;
3512		−
3513		− kmem_cache_destroy(iommu_domain_cache);
3514		−domain_error:
3515		− iova_cache_put();
3516		−
3517		− return -ENOMEM;
3518		−}
3519		−
3520		−static void __init iommu_exit_mempool(void)
3521		−{
3522		− kmem_cache_destroy(iommu_devinfo_cache);
3523		− kmem_cache_destroy(iommu_domain_cache);
3524		− iova_cache_put();
3525		−}
3526		−
3527	3200	static void __init init_no_remapping_devices(void)
3528	3201	{
3529	3202	struct dmar_drhd_unit *drhd;
@@ -3691,7 +3364,7 @@ static void __init init_iommu_pm_ops(void) · 161 unmodified lines
3691	3364	static inline void init_iommu_pm_ops(void) {}
3692	3365	#endif /* CONFIG_PM */
3693	3366
3694		−static int rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
	3367	+static int __init rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
3695	3368	{
3696	3369	if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) \|\|
3697	3370	!IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) \|\|
@@ -4020,7 +3693,31 @@ static void intel_iommu_free_dmars(void) · 322 unmodified lines
4020	3693	}
4021	3694	}
4022	3695
4023		−int dmar_find_matched_atsr_unit(struct pci_dev *dev)
	3696	+static struct dmar_satc_unit dmar_find_matched_satc_unit(struct pci_dev dev)
	3697	+{
	3698	+ struct dmar_satc_unit *satcu;
	3699	+ struct acpi_dmar_satc *satc;
	3700	+ struct device *tmp;
	3701	+ int i;
	3702	+
	3703	+ dev = pci_physfn(dev);
	3704	+ rcu_read_lock();
	3705	+
	3706	+ list_for_each_entry_rcu(satcu, &dmar_satc_units, list) {
	3707	+ satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
	3708	+ if (satc->segment != pci_domain_nr(dev->bus))
	3709	+ continue;
	3710	+ for_each_dev_scope(satcu->devices, satcu->devices_cnt, i, tmp)
	3711	+ if (to_pci_dev(tmp) == dev)
	3712	+ goto out;
	3713	+ }
	3714	+ satcu = NULL;
	3715	+out:
	3716	+ rcu_read_unlock();
	3717	+ return satcu;
	3718	+}
	3719	+
	3720	+static int dmar_ats_supported(struct pci_dev dev, struct intel_iommu iommu)
4024	3721	{
4025	3722	int i, ret = 1;
4026	3723	struct pci_bus *bus;
@@ -4028,8 +3725,20 @@ int dmar_find_matched_atsr_unit(struct pci_dev *dev) · 1 unmodified line
4028	3725	struct device *tmp;
4029	3726	struct acpi_dmar_atsr *atsr;
4030	3727	struct dmar_atsr_unit *atsru;
	3728	+ struct dmar_satc_unit *satcu;
4031	3729
4032	3730	dev = pci_physfn(dev);
	3731	+ satcu = dmar_find_matched_satc_unit(dev);
	3732	+ if (satcu)
	3733	+ /*
	3734	+ * This device supports ATS as it is in SATC table.
	3735	+ * When IOMMU is in legacy mode, enabling ATS is done
	3736	+ * automatically by HW for the device that requires
	3737	+ * ATS, hence OS should not enable this device ATS
	3738	+ * to avoid duplicated TLB invalidation.
	3739	+ */
	3740	+ return !(satcu->atc_required && !sm_supported(iommu));
	3741	+
4033	3742	for (bus = dev->bus; bus; bus = bus->parent) {
4034	3743	bridge = bus->self;
4035	3744	/* If it's an integrated device, allow ATS */
@@ -4375,12 +4084,6 @@ int __init intel_iommu_init(void) · 339 unmodified lines
4375	4084	force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) \|\|
4376	4085	platform_optin_force_iommu();
4377	4086
4378		− if (iommu_init_mempool()) {
4379		− if (force_on)
4380		− panic("tboot: Failed to initialize iommu memory\n");
4381		− return -ENOMEM;
4382		− }
4383		−
4384	4087	down_write(&dmar_global_lock);
4385	4088	if (dmar_table_init()) {
4386	4089	if (force_on)
@@ -4501,7 +4204,6 @@ int __init intel_iommu_init(void) · 114 unmodified lines
4501	4204	out_free_dmar:
4502	4205	intel_iommu_free_dmars();
4503	4206	up_write(&dmar_global_lock);
4504		− iommu_exit_mempool();
4505	4207	return ret;
4506	4208	}
4507	4209
@@ -4552,13 +4254,11 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info) · 44 unmodified lines
4552	4254	intel_pasid_free_table(info->dev);
4553	4255	}
4554	4256
4555		− unlink_domain_info(info);
	4257	+ list_del(&info->link);
4556	4258
4557	4259	spin_lock_irqsave(&iommu->lock, flags);
4558	4260	domain_detach_iommu(domain, iommu);
4559	4261	spin_unlock_irqrestore(&iommu->lock, flags);
4560		−
4561		− free_devinfo_mem(info);
4562	4262	}
4563	4263
4564	4264	static void dmar_remove_one_dev_info(struct device *dev)
@@ -4567,7 +4267,7 @@ static void dmar_remove_one_dev_info(struct device *dev) · 2 unmodified lines
4567	4267	unsigned long flags;
4568	4268
4569	4269	spin_lock_irqsave(&device_domain_lock, flags);
4570		− info = get_domain_info(dev);
	4270	+ info = dev_iommu_priv_get(dev);
4571	4271	if (info)
4572	4272	__dmar_remove_one_dev_info(info);
4573	4273	spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -4637,183 +4337,6 @@ static void intel_iommu_domain_free(struct iommu_domain *domain) · 63 unmodified lines
4637	4337	domain_exit(to_dmar_domain(domain));
4638	4338	}
4639	4339
4640		−/*
4641		− * Check whether a @domain could be attached to the @dev through the
4642		− * aux-domain attach/detach APIs.
4643		− */
4644		−static inline bool
4645		−is_aux_domain(struct device dev, struct iommu_domain domain)
4646		−{
4647		− struct device_domain_info *info = get_domain_info(dev);
4648		−
4649		− return info && info->auxd_enabled &&
4650		− domain->type == IOMMU_DOMAIN_UNMANAGED;
4651		−}
4652		−
4653		−static inline struct subdev_domain_info *
4654		−lookup_subdev_info(struct dmar_domain domain, struct device dev)
4655		−{
4656		− struct subdev_domain_info *sinfo;
4657		−
4658		− if (!list_empty(&domain->subdevices)) {
4659		− list_for_each_entry(sinfo, &domain->subdevices, link_domain) {
4660		− if (sinfo->pdev == dev)
4661		− return sinfo;
4662		− }
4663		− }
4664		−
4665		− return NULL;
4666		−}
4667		−
4668		−static int auxiliary_link_device(struct dmar_domain *domain,
4669		− struct device *dev)
4670		−{
4671		− struct device_domain_info *info = get_domain_info(dev);
4672		− struct subdev_domain_info *sinfo = lookup_subdev_info(domain, dev);
4673		−
4674		− assert_spin_locked(&device_domain_lock);
4675		− if (WARN_ON(!info))
4676		− return -EINVAL;
4677		−
4678		− if (!sinfo) {
4679		− sinfo = kzalloc(sizeof(*sinfo), GFP_ATOMIC);
4680		− if (!sinfo)
4681		− return -ENOMEM;
4682		− sinfo->domain = domain;
4683		− sinfo->pdev = dev;
4684		− list_add(&sinfo->link_phys, &info->subdevices);
4685		− list_add(&sinfo->link_domain, &domain->subdevices);
4686		− }
4687		−
4688		− return ++sinfo->users;
4689		−}
4690		−
4691		−static int auxiliary_unlink_device(struct dmar_domain *domain,
4692		− struct device *dev)
4693		−{
4694		− struct device_domain_info *info = get_domain_info(dev);
4695		− struct subdev_domain_info *sinfo = lookup_subdev_info(domain, dev);
4696		− int ret;
4697		−
4698		− assert_spin_locked(&device_domain_lock);
4699		− if (WARN_ON(!info \|\| !sinfo \|\| sinfo->users <= 0))
4700		− return -EINVAL;
4701		−
4702		− ret = --sinfo->users;
4703		− if (!ret) {
4704		− list_del(&sinfo->link_phys);
4705		− list_del(&sinfo->link_domain);
4706		− kfree(sinfo);
4707		− }
4708		−
4709		− return ret;
4710		−}
4711		−
4712		−static int aux_domain_add_dev(struct dmar_domain *domain,
4713		− struct device *dev)
4714		−{
4715		− int ret;
4716		− unsigned long flags;
4717		− struct intel_iommu *iommu;
4718		−
4719		− iommu = device_to_iommu(dev, NULL, NULL);
4720		− if (!iommu)
4721		− return -ENODEV;
4722		−
4723		− if (domain->default_pasid <= 0) {
4724		− u32 pasid;
4725		−
4726		− /* No private data needed for the default pasid */
4727		− pasid = ioasid_alloc(NULL, PASID_MIN,
4728		− pci_max_pasids(to_pci_dev(dev)) - 1,
4729		− NULL);
4730		− if (pasid == INVALID_IOASID) {
4731		− pr_err("Can't allocate default pasid\n");
4732		− return -ENODEV;
4733		− }
4734		− domain->default_pasid = pasid;
4735		− }
4736		−
4737		− spin_lock_irqsave(&device_domain_lock, flags);
4738		− ret = auxiliary_link_device(domain, dev);
4739		− if (ret <= 0)
4740		− goto link_failed;
4741		−
4742		− /*
4743		− * Subdevices from the same physical device can be attached to the
4744		− * same domain. For such cases, only the first subdevice attachment
4745		− * needs to go through the full steps in this function. So if ret >
4746		− * 1, just goto out.
4747		− */
4748		− if (ret > 1)
4749		− goto out;
4750		−
4751		− /*
4752		− * iommu->lock must be held to attach domain to iommu and setup the
4753		− * pasid entry for second level translation.
4754		− */
4755		− spin_lock(&iommu->lock);
4756		− ret = domain_attach_iommu(domain, iommu);
4757		− if (ret)
4758		− goto attach_failed;
4759		−
4760		− /* Setup the PASID entry for mediated devices: */
4761		− if (domain_use_first_level(domain))
4762		− ret = domain_setup_first_level(iommu, domain, dev,
4763		− domain->default_pasid);
4764		− else
4765		− ret = intel_pasid_setup_second_level(iommu, domain, dev,
4766		− domain->default_pasid);
4767		− if (ret)
4768		− goto table_failed;
4769		−
4770		− spin_unlock(&iommu->lock);
4771		−out:
4772		− spin_unlock_irqrestore(&device_domain_lock, flags);
4773		−
4774		− return 0;
4775		−
4776		−table_failed:
4777		− domain_detach_iommu(domain, iommu);
4778		−attach_failed:
4779		− spin_unlock(&iommu->lock);
4780		− auxiliary_unlink_device(domain, dev);
4781		−link_failed:
4782		− spin_unlock_irqrestore(&device_domain_lock, flags);
4783		− if (list_empty(&domain->subdevices) && domain->default_pasid > 0)
4784		− ioasid_free(domain->default_pasid);
4785		−
4786		− return ret;
4787		−}
4788		−
4789		−static void aux_domain_remove_dev(struct dmar_domain *domain,
4790		− struct device *dev)
4791		−{
4792		− struct device_domain_info *info;
4793		− struct intel_iommu *iommu;
4794		− unsigned long flags;
4795		−
4796		− if (!is_aux_domain(dev, &domain->domain))
4797		− return;
4798		−
4799		− spin_lock_irqsave(&device_domain_lock, flags);
4800		− info = get_domain_info(dev);
4801		− iommu = info->iommu;
4802		−
4803		− if (!auxiliary_unlink_device(domain, dev)) {
4804		− spin_lock(&iommu->lock);
4805		− intel_pasid_tear_down_entry(iommu, dev,
4806		− domain->default_pasid, false);
4807		− domain_detach_iommu(domain, iommu);
4808		− spin_unlock(&iommu->lock);
4809		− }
4810		−
4811		− spin_unlock_irqrestore(&device_domain_lock, flags);
4812		−
4813		− if (list_empty(&domain->subdevices) && domain->default_pasid > 0)
4814		− ioasid_free(domain->default_pasid);
4815		−}
4816		−
4817	4340	static int prepare_domain_attach_device(struct iommu_domain *domain,
4818	4341	struct device *dev)
4819	4342	{
@@ -4825,13 +4348,6 @@ static int prepare_domain_attach_device(struct iommu_domain *domain, · 5 unmodified lines
4825	4348	if (!iommu)
4826	4349	return -ENODEV;
4827	4350
4828		− if ((dmar_domain->flags & DOMAIN_FLAG_NESTING_MODE) &&
4829		− !ecap_nest(iommu->ecap)) {
4830		− dev_err(dev, "%s: iommu not support nested translation\n",
4831		− iommu->name);
4832		− return -EINVAL;
4833		− }
4834		−
4835	4351	/* check if this iommu agaw is sufficient for max mapped address */
4836	4352	addr_width = agaw_to_width(iommu->agaw);
4837	4353	if (addr_width > cap_mgaw(iommu->cap))
@@ -4873,15 +4389,11 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, · 35 unmodified lines
4873	4389	return -EPERM;
4874	4390	}
4875	4391
4876		− if (is_aux_domain(dev, domain))
4877		− return -EPERM;
4878		−
4879	4392	/* normally dev is not mapped */
4880	4393	if (unlikely(domain_context_mapped(dev))) {
4881		− struct dmar_domain *old_domain;
	4394	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
4882	4395
4883		− old_domain = find_domain(dev);
4884		− if (old_domain)
	4396	+ if (info->domain)
4885	4397	dmar_remove_one_dev_info(dev);
4886	4398	}
4887	4399
@@ -4892,212 +4404,12 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, · 4 unmodified lines
4892	4404	return domain_add_dev_info(to_dmar_domain(domain), dev);
4893	4405	}
4894	4406
4895		−static int intel_iommu_aux_attach_device(struct iommu_domain *domain,
4896		− struct device *dev)
4897		−{
4898		− int ret;
4899		−
4900		− if (!is_aux_domain(dev, domain))
4901		− return -EPERM;
4902		−
4903		− ret = prepare_domain_attach_device(domain, dev);
4904		− if (ret)
4905		− return ret;
4906		−
4907		− return aux_domain_add_dev(to_dmar_domain(domain), dev);
4908		−}
4909		−
4910	4407	static void intel_iommu_detach_device(struct iommu_domain *domain,
4911	4408	struct device *dev)
4912	4409	{
4913	4410	dmar_remove_one_dev_info(dev);
4914	4411	}
4915	4412
4916		−static void intel_iommu_aux_detach_device(struct iommu_domain *domain,
4917		− struct device *dev)
4918		−{
4919		− aux_domain_remove_dev(to_dmar_domain(domain), dev);
4920		−}
4921		−
4922		−#ifdef CONFIG_INTEL_IOMMU_SVM
4923		−/*
4924		− * 2D array for converting and sanitizing IOMMU generic TLB granularity to
4925		− * VT-d granularity. Invalidation is typically included in the unmap operation
4926		− * as a result of DMA or VFIO unmap. However, for assigned devices guest
4927		− * owns the first level page tables. Invalidations of translation caches in the
4928		− * guest are trapped and passed down to the host.
4929		− *
4930		− * vIOMMU in the guest will only expose first level page tables, therefore
4931		− * we do not support IOTLB granularity for request without PASID (second level).
4932		− *
4933		− * For example, to find the VT-d granularity encoding for IOTLB
4934		− * type and page selective granularity within PASID:
4935		− * X: indexed by iommu cache type
4936		− * Y: indexed by enum iommu_inv_granularity
4937		− * [IOMMU_CACHE_INV_TYPE_IOTLB][IOMMU_INV_GRANU_ADDR]
4938		− */
4939		−
4940		−static const int
4941		−inv_type_granu_table[IOMMU_CACHE_INV_TYPE_NR][IOMMU_INV_GRANU_NR] = {
4942		− /*
4943		− * PASID based IOTLB invalidation: PASID selective (per PASID),
4944		− * page selective (address granularity)
4945		− */
4946		− {-EINVAL, QI_GRAN_NONG_PASID, QI_GRAN_PSI_PASID},
4947		− /* PASID based dev TLBs */
4948		− {-EINVAL, -EINVAL, QI_DEV_IOTLB_GRAN_PASID_SEL},
4949		− /* PASID cache */
4950		− {-EINVAL, -EINVAL, -EINVAL}
4951		−};
4952		−
4953		−static inline int to_vtd_granularity(int type, int granu)
4954		−{
4955		− return inv_type_granu_table[type][granu];
4956		−}
4957		−
4958		−static inline u64 to_vtd_size(u64 granu_size, u64 nr_granules)
4959		−{
4960		− u64 nr_pages = (granu_size * nr_granules) >> VTD_PAGE_SHIFT;
4961		−
4962		− /* VT-d size is encoded as 2^size of 4K pages, 0 for 4k, 9 for 2MB, etc.
4963		− * IOMMU cache invalidate API passes granu_size in bytes, and number of
4964		− * granu size in contiguous memory.
4965		− */
4966		− return order_base_2(nr_pages);
4967		−}
4968		−
4969		−static int
4970		−intel_iommu_sva_invalidate(struct iommu_domain domain, struct device dev,
4971		− struct iommu_cache_invalidate_info *inv_info)
4972		−{
4973		− struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4974		− struct device_domain_info *info;
4975		− struct intel_iommu *iommu;
4976		− unsigned long flags;
4977		− int cache_type;
4978		− u8 bus, devfn;
4979		− u16 did, sid;
4980		− int ret = 0;
4981		− u64 size = 0;
4982		−
4983		− if (!inv_info \|\| !dmar_domain)
4984		− return -EINVAL;
4985		−
4986		− if (!dev \|\| !dev_is_pci(dev))
4987		− return -ENODEV;
4988		−
4989		− iommu = device_to_iommu(dev, &bus, &devfn);
4990		− if (!iommu)
4991		− return -ENODEV;
4992		−
4993		− if (!(dmar_domain->flags & DOMAIN_FLAG_NESTING_MODE))
4994		− return -EINVAL;
4995		−
4996		− spin_lock_irqsave(&device_domain_lock, flags);
4997		− spin_lock(&iommu->lock);
4998		− info = get_domain_info(dev);
4999		− if (!info) {
5000		− ret = -EINVAL;
5001		− goto out_unlock;
5002		− }
5003		− did = dmar_domain->iommu_did[iommu->seq_id];
5004		− sid = PCI_DEVID(bus, devfn);
5005		−
5006		− /* Size is only valid in address selective invalidation */
5007		− if (inv_info->granularity == IOMMU_INV_GRANU_ADDR)
5008		− size = to_vtd_size(inv_info->granu.addr_info.granule_size,
5009		− inv_info->granu.addr_info.nb_granules);
5010		−
5011		− for_each_set_bit(cache_type,
5012		− (unsigned long *)&inv_info->cache,
5013		− IOMMU_CACHE_INV_TYPE_NR) {
5014		− int granu = 0;
5015		− u64 pasid = 0;
5016		− u64 addr = 0;
5017		−
5018		− granu = to_vtd_granularity(cache_type, inv_info->granularity);
5019		− if (granu == -EINVAL) {
5020		− pr_err_ratelimited("Invalid cache type and granu combination %d/%d\n",
5021		− cache_type, inv_info->granularity);
5022		− break;
5023		− }
5024		−
5025		− /*
5026		− * PASID is stored in different locations based on the
5027		− * granularity.
5028		− */
5029		− if (inv_info->granularity == IOMMU_INV_GRANU_PASID &&
5030		− (inv_info->granu.pasid_info.flags & IOMMU_INV_PASID_FLAGS_PASID))
5031		− pasid = inv_info->granu.pasid_info.pasid;
5032		− else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR &&
5033		− (inv_info->granu.addr_info.flags & IOMMU_INV_ADDR_FLAGS_PASID))
5034		− pasid = inv_info->granu.addr_info.pasid;
5035		−
5036		− switch (BIT(cache_type)) {
5037		− case IOMMU_CACHE_INV_TYPE_IOTLB:
5038		− /* HW will ignore LSB bits based on address mask */
5039		− if (inv_info->granularity == IOMMU_INV_GRANU_ADDR &&
5040		− size &&
5041		− (inv_info->granu.addr_info.addr & ((BIT(VTD_PAGE_SHIFT + size)) - 1))) {
5042		− pr_err_ratelimited("User address not aligned, 0x%llx, size order %llu\n",
5043		− inv_info->granu.addr_info.addr, size);
5044		− }
5045		−
5046		− /*
5047		− * If granu is PASID-selective, address is ignored.
5048		− * We use npages = -1 to indicate that.
5049		− */
5050		− qi_flush_piotlb(iommu, did, pasid,
5051		− mm_to_dma_pfn(inv_info->granu.addr_info.addr),
5052		− (granu == QI_GRAN_NONG_PASID) ? -1 : 1 << size,
5053		− inv_info->granu.addr_info.flags & IOMMU_INV_ADDR_FLAGS_LEAF);
5054		−
5055		− if (!info->ats_enabled)
5056		− break;
5057		− /*
5058		− * Always flush device IOTLB if ATS is enabled. vIOMMU
5059		− * in the guest may assume IOTLB flush is inclusive,
5060		− * which is more efficient.
5061		− */
5062		− fallthrough;
5063		− case IOMMU_CACHE_INV_TYPE_DEV_IOTLB:
5064		− /*
5065		− * PASID based device TLB invalidation does not support
5066		− * IOMMU_INV_GRANU_PASID granularity but only supports
5067		− * IOMMU_INV_GRANU_ADDR.
5068		− * The equivalent of that is we set the size to be the
5069		− * entire range of 64 bit. User only provides PASID info
5070		− * without address info. So we set addr to 0.
5071		− */
5072		− if (inv_info->granularity == IOMMU_INV_GRANU_PASID) {
5073		− size = 64 - VTD_PAGE_SHIFT;
5074		− addr = 0;
5075		− } else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR) {
5076		− addr = inv_info->granu.addr_info.addr;
5077		− }
5078		−
5079		− if (info->ats_enabled)
5080		− qi_flush_dev_iotlb_pasid(iommu, sid,
5081		− info->pfsid, pasid,
5082		− info->ats_qdep, addr,
5083		− size);
5084		− else
5085		− pr_warn_ratelimited("Passdown device IOTLB flush w/o ATS!\n");
5086		− break;
5087		− default:
5088		− dev_err_ratelimited(dev, "Unsupported IOMMU invalidation type %d\n",
5089		− cache_type);
5090		− ret = -EINVAL;
5091		− }
5092		− }
5093		−out_unlock:
5094		− spin_unlock(&iommu->lock);
5095		− spin_unlock_irqrestore(&device_domain_lock, flags);
5096		−
5097		− return ret;
5098		−}
5099		−#endif
5100		−
5101	4413	static int intel_iommu_map(struct iommu_domain *domain,
5102	4414	unsigned long iova, phys_addr_t hpa,
5103	4415	size_t size, int iommu_prot, gfp_t gfp)
@@ -5245,28 +4557,73 @@ static bool intel_iommu_capable(enum iommu_cap cap) · 141 unmodified lines
5245	4557
5246	4558	static struct iommu_device intel_iommu_probe_device(struct device dev)
5247	4559	{
	4560	+ struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
	4561	+ struct device_domain_info *info;
5248	4562	struct intel_iommu *iommu;
	4563	+ unsigned long flags;
	4564	+ u8 bus, devfn;
5249	4565
5250		− iommu = device_to_iommu(dev, NULL, NULL);
	4566	+ iommu = device_to_iommu(dev, &bus, &devfn);
5251	4567	if (!iommu)
5252	4568	return ERR_PTR(-ENODEV);
5253	4569
5254		− if (translation_pre_enabled(iommu))
5255		− dev_iommu_priv_set(dev, DEFER_DEVICE_DOMAIN_INFO);
	4570	+ info = kzalloc(sizeof(*info), GFP_KERNEL);
	4571	+ if (!info)
	4572	+ return ERR_PTR(-ENOMEM);
	4573	+
	4574	+ if (dev_is_real_dma_subdevice(dev)) {
	4575	+ info->bus = pdev->bus->number;
	4576	+ info->devfn = pdev->devfn;
	4577	+ info->segment = pci_domain_nr(pdev->bus);
	4578	+ } else {
	4579	+ info->bus = bus;
	4580	+ info->devfn = devfn;
	4581	+ info->segment = iommu->segment;
	4582	+ }
	4583	+
	4584	+ info->dev = dev;
	4585	+ info->iommu = iommu;
	4586	+ if (dev_is_pci(dev)) {
	4587	+ if (ecap_dev_iotlb_support(iommu->ecap) &&
	4588	+ pci_ats_supported(pdev) &&
	4589	+ dmar_ats_supported(pdev, iommu))
	4590	+ info->ats_supported = 1;
	4591	+
	4592	+ if (sm_supported(iommu)) {
	4593	+ if (pasid_supported(iommu)) {
	4594	+ int features = pci_pasid_features(pdev);
	4595	+
	4596	+ if (features >= 0)
	4597	+ info->pasid_supported = features \| 1;
	4598	+ }
	4599	+
	4600	+ if (info->ats_supported && ecap_prs(iommu->ecap) &&
	4601	+ pci_pri_supported(pdev))
	4602	+ info->pri_supported = 1;
	4603	+ }
	4604	+ }
	4605	+
	4606	+ spin_lock_irqsave(&device_domain_lock, flags);
	4607	+ list_add(&info->global, &device_domain_list);
	4608	+ dev_iommu_priv_set(dev, info);
	4609	+ spin_unlock_irqrestore(&device_domain_lock, flags);
5256	4610
5257	4611	return &iommu->iommu;
5258	4612	}
5259	4613
5260	4614	static void intel_iommu_release_device(struct device *dev)
5261	4615	{
5262		− struct intel_iommu *iommu;
5263		−
5264		− iommu = device_to_iommu(dev, NULL, NULL);
5265		− if (!iommu)
5266		− return;
	4616	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
	4617	+ unsigned long flags;
5267	4618
5268	4619	dmar_remove_one_dev_info(dev);
5269	4620
	4621	+ spin_lock_irqsave(&device_domain_lock, flags);
	4622	+ dev_iommu_priv_set(dev, NULL);
	4623	+ list_del(&info->global);
	4624	+ spin_unlock_irqrestore(&device_domain_lock, flags);
	4625	+
	4626	+ kfree(info);
5270	4627	set_dma_ops(dev, NULL);
5271	4628	}
5272	4629
@@ -5335,14 +4692,14 @@ static void intel_iommu_get_resv_regions(struct device *device, · 62 unmodified lines
5335	4692
5336	4693	int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev)
5337	4694	{
5338		− struct device_domain_info *info;
	4695	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
5339	4696	struct context_entry *context;
5340	4697	struct dmar_domain *domain;
5341	4698	unsigned long flags;
5342	4699	u64 ctx_lo;
5343	4700	int ret;
5344	4701
5345		− domain = find_domain(dev);
	4702	+ domain = info->domain;
5346	4703	if (!domain)
5347	4704	return -EINVAL;
5348	4705
@@ -5350,8 +4707,7 @@ int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev) · 1 unmodified line
5350	4707	spin_lock(&iommu->lock);
5351	4708
5352	4709	ret = -EINVAL;
5353		− info = get_domain_info(dev);
5354		− if (!info \|\| !info->pasid_supported)
	4710	+ if (!info->pasid_supported)
5355	4711	goto out;
5356	4712
5357	4713	context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
@@ -5391,49 +4747,9 @@ static struct iommu_group intel_iommu_device_group(struct device dev) · 33 unmodified lines
5391	4747	return generic_device_group(dev);
5392	4748	}
5393	4749
5394		−static int intel_iommu_enable_auxd(struct device *dev)
5395		−{
5396		− struct device_domain_info *info;
5397		− struct intel_iommu *iommu;
5398		− unsigned long flags;
5399		− int ret;
5400		−
5401		− iommu = device_to_iommu(dev, NULL, NULL);
5402		− if (!iommu \|\| dmar_disabled)
5403		− return -EINVAL;
5404		−
5405		− if (!sm_supported(iommu) \|\| !pasid_supported(iommu))
5406		− return -EINVAL;
5407		−
5408		− ret = intel_iommu_enable_pasid(iommu, dev);
5409		− if (ret)
5410		− return -ENODEV;
5411		−
5412		− spin_lock_irqsave(&device_domain_lock, flags);
5413		− info = get_domain_info(dev);
5414		− info->auxd_enabled = 1;
5415		− spin_unlock_irqrestore(&device_domain_lock, flags);
5416		−
5417		− return 0;
5418		−}
5419		−
5420		−static int intel_iommu_disable_auxd(struct device *dev)
5421		−{
5422		− struct device_domain_info *info;
5423		− unsigned long flags;
5424		−
5425		− spin_lock_irqsave(&device_domain_lock, flags);
5426		− info = get_domain_info(dev);
5427		− if (!WARN_ON(!info))
5428		− info->auxd_enabled = 0;
5429		− spin_unlock_irqrestore(&device_domain_lock, flags);
5430		−
5431		− return 0;
5432		−}
5433		−
5434	4750	static int intel_iommu_enable_sva(struct device *dev)
5435	4751	{
5436		− struct device_domain_info *info = get_domain_info(dev);
	4752	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
5437	4753	struct intel_iommu *iommu;
5438	4754	int ret;
5439	4755
@@ -5462,7 +4778,7 @@ static int intel_iommu_enable_sva(struct device *dev) · 22 unmodified lines
5462	4778
5463	4779	static int intel_iommu_disable_sva(struct device *dev)
5464	4780	{
5465		− struct device_domain_info *info = get_domain_info(dev);
	4781	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
5466	4782	struct intel_iommu *iommu = info->iommu;
5467	4783	int ret;
5468	4784
@@ -5475,7 +4791,7 @@ static int intel_iommu_disable_sva(struct device *dev) · 6 unmodified lines
5475	4791
5476	4792	static int intel_iommu_enable_iopf(struct device *dev)
5477	4793	{
5478		− struct device_domain_info *info = get_domain_info(dev);
	4794	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
5479	4795
5480	4796	if (info && info->pri_supported)
5481	4797	return 0;
@@ -5487,9 +4803,6 @@ static int · 5 unmodified lines
5487	4803	intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
5488	4804	{
5489	4805	switch (feat) {
5490		− case IOMMU_DEV_FEAT_AUX:
5491		− return intel_iommu_enable_auxd(dev);
5492		−
5493	4806	case IOMMU_DEV_FEAT_IOPF:
5494	4807	return intel_iommu_enable_iopf(dev);
5495	4808
@@ -5505,9 +4818,6 @@ static int · 9 unmodified lines
5505	4818	intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
5506	4819	{
5507	4820	switch (feat) {
5508		− case IOMMU_DEV_FEAT_AUX:
5509		− return intel_iommu_disable_auxd(dev);
5510		−
5511	4821	case IOMMU_DEV_FEAT_IOPF:
5512	4822	return 0;
5513	4823
@@ -5519,48 +4829,11 @@ intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) · 5 unmodified lines
5519	4829	}
5520	4830	}
5521	4831
5522		−static bool
5523		−intel_iommu_dev_feat_enabled(struct device *dev, enum iommu_dev_features feat)
5524		−{
5525		− struct device_domain_info *info = get_domain_info(dev);
5526		−
5527		− if (feat == IOMMU_DEV_FEAT_AUX)
5528		− return scalable_mode_support() && info && info->auxd_enabled;
5529		−
5530		− return false;
5531		−}
5532		−
5533		−static int
5534		−intel_iommu_aux_get_pasid(struct iommu_domain domain, struct device dev)
	4832	+static bool intel_iommu_is_attach_deferred(struct device *dev)
5535	4833	{
5536		− struct dmar_domain *dmar_domain = to_dmar_domain(domain);
	4834	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
5537	4835
5538		− return dmar_domain->default_pasid > 0 ?
5539		− dmar_domain->default_pasid : -EINVAL;
5540		−}
5541		−
5542		−static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain,
5543		− struct device *dev)
5544		−{
5545		− return attach_deferred(dev);
5546		−}
5547		−
5548		−static int
5549		−intel_iommu_enable_nesting(struct iommu_domain *domain)
5550		−{
5551		− struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5552		− unsigned long flags;
5553		− int ret = -ENODEV;
5554		−
5555		− spin_lock_irqsave(&device_domain_lock, flags);
5556		− if (list_empty(&dmar_domain->devices)) {
5557		− dmar_domain->flags \|= DOMAIN_FLAG_NESTING_MODE;
5558		− dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL;
5559		− ret = 0;
5560		− }
5561		− spin_unlock_irqrestore(&device_domain_lock, flags);
5562		−
5563		− return ret;
	4836	+ return translation_pre_enabled(info->iommu) && !info->domain;
5564	4837	}
5565	4838
5566	4839	/*
@@ -5598,40 +4871,34 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain, · 31 unmodified lines
5598	4871	const struct iommu_ops intel_iommu_ops = {
5599	4872	.capable = intel_iommu_capable,
5600	4873	.domain_alloc = intel_iommu_domain_alloc,
5601		− .domain_free = intel_iommu_domain_free,
5602		− .enable_nesting = intel_iommu_enable_nesting,
5603		− .attach_dev = intel_iommu_attach_device,
5604		− .detach_dev = intel_iommu_detach_device,
5605		− .aux_attach_dev = intel_iommu_aux_attach_device,
5606		− .aux_detach_dev = intel_iommu_aux_detach_device,
5607		− .aux_get_pasid = intel_iommu_aux_get_pasid,
5608		− .map_pages = intel_iommu_map_pages,
5609		− .unmap_pages = intel_iommu_unmap_pages,
5610		− .iotlb_sync_map = intel_iommu_iotlb_sync_map,
5611		− .flush_iotlb_all = intel_flush_iotlb_all,
5612		− .iotlb_sync = intel_iommu_tlb_sync,
5613		− .iova_to_phys = intel_iommu_iova_to_phys,
5614	4874	.probe_device = intel_iommu_probe_device,
5615	4875	.probe_finalize = intel_iommu_probe_finalize,
5616	4876	.release_device = intel_iommu_release_device,
5617	4877	.get_resv_regions = intel_iommu_get_resv_regions,
5618	4878	.put_resv_regions = generic_iommu_put_resv_regions,
5619	4879	.device_group = intel_iommu_device_group,
5620		− .dev_feat_enabled = intel_iommu_dev_feat_enabled,
5621	4880	.dev_enable_feat = intel_iommu_dev_enable_feat,
5622	4881	.dev_disable_feat = intel_iommu_dev_disable_feat,
5623	4882	.is_attach_deferred = intel_iommu_is_attach_deferred,
5624	4883	.def_domain_type = device_def_domain_type,
5625	4884	.pgsize_bitmap = SZ_4K,
5626	4885	#ifdef CONFIG_INTEL_IOMMU_SVM
5627		− .cache_invalidate = intel_iommu_sva_invalidate,
5628		− .sva_bind_gpasid = intel_svm_bind_gpasid,
5629		− .sva_unbind_gpasid = intel_svm_unbind_gpasid,
5630	4886	.sva_bind = intel_svm_bind,
5631	4887	.sva_unbind = intel_svm_unbind,
5632	4888	.sva_get_pasid = intel_svm_get_pasid,
5633	4889	.page_response = intel_svm_page_response,
5634	4890	#endif
	4891	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	4892	+ .attach_dev = intel_iommu_attach_device,
	4893	+ .detach_dev = intel_iommu_detach_device,
	4894	+ .map_pages = intel_iommu_map_pages,
	4895	+ .unmap_pages = intel_iommu_unmap_pages,
	4896	+ .iotlb_sync_map = intel_iommu_iotlb_sync_map,
	4897	+ .flush_iotlb_all = intel_flush_iotlb_all,
	4898	+ .iotlb_sync = intel_iommu_tlb_sync,
	4899	+ .iova_to_phys = intel_iommu_iova_to_phys,
	4900	+ .free = intel_iommu_domain_free,
	4901	+ }
5635	4902	};
5636	4903
5637	4904	static void quirk_iommu_igfx(struct pci_dev *dev)

drivers/iommu/intel/pasid.c+6−167

@@ -150,7 +150,7 @@ int intel_pasid_alloc_table(struct device *dev) · 149 unmodified lines
150	150	int size;
151	151
152	152	might_sleep();
153		− info = get_domain_info(dev);
	153	+ info = dev_iommu_priv_get(dev);
154	154	if (WARN_ON(!info \|\| !dev_is_pci(dev) \|\| info->pasid_table))
155	155	return -EINVAL;
156	156
@@ -197,7 +197,7 @@ void intel_pasid_free_table(struct device *dev) · 40 unmodified lines
197	197	struct pasid_entry *table;
198	198	int i, max_pde;
199	199
200		− info = get_domain_info(dev);
	200	+ info = dev_iommu_priv_get(dev);
201	201	if (!info \|\| !dev_is_pci(dev) \|\| !info->pasid_table)
202	202	return;
203	203
@@ -223,7 +223,7 @@ struct pasid_table intel_pasid_get_table(struct device dev) · 19 unmodified lines
223	223	{
224	224	struct device_domain_info *info;
225	225
226		− info = get_domain_info(dev);
	226	+ info = dev_iommu_priv_get(dev);
227	227	if (!info)
228	228	return NULL;
229	229
@@ -234,7 +234,7 @@ static int intel_pasid_get_dev_max_id(struct device *dev) · 4 unmodified lines
234	234	{
235	235	struct device_domain_info *info;
236	236
237		− info = get_domain_info(dev);
	237	+ info = dev_iommu_priv_get(dev);
238	238	if (!info \|\| !info->pasid_table)
239	239	return 0;
240	240
@@ -254,7 +254,7 @@ static struct pasid_entry intel_pasid_get_entry(struct device dev, u32 pasid) · 13 unmodified lines
254	254	return NULL;
255	255
256	256	dir = pasid_table->table;
257		− info = get_domain_info(dev);
	257	+ info = dev_iommu_priv_get(dev);
258	258	dir_index = pasid >> PASID_PDE_SHIFT;
259	259	index = pasid & PASID_PTE_MASK;
260	260
@@ -487,7 +487,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, · 226 unmodified lines
487	487	struct device_domain_info *info;
488	488	u16 sid, qdep, pfsid;
489	489
490		− info = get_domain_info(dev);
	490	+ info = dev_iommu_priv_get(dev);
491	491	if (!info \|\| !info->ats_enabled)
492	492	return;
493	493
@@ -762,164 +762,3 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, · 268 unmodified lines
762	762
763	763	return 0;
764	764	}
765		−
766		−static int
767		−intel_pasid_setup_bind_data(struct intel_iommu iommu, struct pasid_entry pte,
768		− struct iommu_gpasid_bind_data_vtd *pasid_data)
769		−{
770		− /*
771		− * Not all guest PASID table entry fields are passed down during bind,
772		− * here we only set up the ones that are dependent on guest settings.
773		− * Execution related bits such as NXE, SMEP are not supported.
774		− * Other fields, such as snoop related, are set based on host needs
775		− * regardless of guest settings.
776		− */
777		− if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_SRE) {
778		− if (!ecap_srs(iommu->ecap)) {
779		− pr_err_ratelimited("No supervisor request support on %s\n",
780		− iommu->name);
781		− return -EINVAL;
782		− }
783		− pasid_set_sre(pte);
784		− /* Enable write protect WP if guest requested */
785		− if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_WPE)
786		− pasid_set_wpe(pte);
787		− }
788		−
789		− if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_EAFE) {
790		− if (!ecap_eafs(iommu->ecap)) {
791		− pr_err_ratelimited("No extended access flag support on %s\n",
792		− iommu->name);
793		− return -EINVAL;
794		− }
795		− pasid_set_eafe(pte);
796		− }
797		−
798		− /*
799		− * Memory type is only applicable to devices inside processor coherent
800		− * domain. Will add MTS support once coherent devices are available.
801		− */
802		− if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_MTS_MASK) {
803		− pr_warn_ratelimited("No memory type support %s\n",
804		− iommu->name);
805		− return -EINVAL;
806		− }
807		−
808		− return 0;
809		−}
810		−
811		−/**
812		− * intel_pasid_setup_nested() - Set up PASID entry for nested translation.
813		− * This could be used for guest shared virtual address. In this case, the
814		− * first level page tables are used for GVA-GPA translation in the guest,
815		− * second level page tables are used for GPA-HPA translation.
816		− *
817		− * @iommu: IOMMU which the device belong to
818		− * @dev: Device to be set up for translation
819		− * @gpgd: FLPTPTR: First Level Page translation pointer in GPA
820		− * @pasid: PASID to be programmed in the device PASID table
821		− * @pasid_data: Additional PASID info from the guest bind request
822		− * @domain: Domain info for setting up second level page tables
823		− * @addr_width: Address width of the first level (guest)
824		− */
825		−int intel_pasid_setup_nested(struct intel_iommu iommu, struct device dev,
826		− pgd_t *gpgd, u32 pasid,
827		− struct iommu_gpasid_bind_data_vtd *pasid_data,
828		− struct dmar_domain *domain, int addr_width)
829		−{
830		− struct pasid_entry *pte;
831		− struct dma_pte *pgd;
832		− int ret = 0;
833		− u64 pgd_val;
834		− int agaw;
835		− u16 did;
836		−
837		− if (!ecap_nest(iommu->ecap)) {
838		− pr_err_ratelimited("IOMMU: %s: No nested translation support\n",
839		− iommu->name);
840		− return -EINVAL;
841		− }
842		−
843		− if (!(domain->flags & DOMAIN_FLAG_NESTING_MODE)) {
844		− pr_err_ratelimited("Domain is not in nesting mode, %x\n",
845		− domain->flags);
846		− return -EINVAL;
847		− }
848		−
849		− pte = intel_pasid_get_entry(dev, pasid);
850		− if (WARN_ON(!pte))
851		− return -EINVAL;
852		−
853		− /*
854		− * Caller must ensure PASID entry is not in use, i.e. not bind the
855		− * same PASID to the same device twice.
856		− */
857		− if (pasid_pte_is_present(pte))
858		− return -EBUSY;
859		−
860		− pasid_clear_entry(pte);
861		−
862		− /* Sanity checking performed by caller to make sure address
863		− * width matching in two dimensions:
864		− * 1. CPU vs. IOMMU
865		− * 2. Guest vs. Host.
866		− */
867		− switch (addr_width) {
868		−#ifdef CONFIG_X86
869		− case ADDR_WIDTH_5LEVEL:
870		− if (!cpu_feature_enabled(X86_FEATURE_LA57) \|\|
871		− !cap_5lp_support(iommu->cap)) {
872		− dev_err_ratelimited(dev,
873		− "5-level paging not supported\n");
874		− return -EINVAL;
875		− }
876		−
877		− pasid_set_flpm(pte, 1);
878		− break;
879		−#endif
880		− case ADDR_WIDTH_4LEVEL:
881		− pasid_set_flpm(pte, 0);
882		− break;
883		− default:
884		− dev_err_ratelimited(dev, "Invalid guest address width %d\n",
885		− addr_width);
886		− return -EINVAL;
887		− }
888		−
889		− /* First level PGD is in GPA, must be supported by the second level */
890		− if ((uintptr_t)gpgd > domain->max_addr) {
891		− dev_err_ratelimited(dev,
892		− "Guest PGD %lx not supported, max %llx\n",
893		− (uintptr_t)gpgd, domain->max_addr);
894		− return -EINVAL;
895		− }
896		− pasid_set_flptr(pte, (uintptr_t)gpgd);
897		−
898		− ret = intel_pasid_setup_bind_data(iommu, pte, pasid_data);
899		− if (ret)
900		− return ret;
901		−
902		− /* Setup the second level based on the given domain */
903		− pgd = domain->pgd;
904		−
905		− agaw = iommu_skip_agaw(domain, iommu, &pgd);
906		− if (agaw < 0) {
907		− dev_err_ratelimited(dev, "Invalid domain page table\n");
908		− return -EINVAL;
909		− }
910		− pgd_val = virt_to_phys(pgd);
911		− pasid_set_slptr(pte, pgd_val);
912		− pasid_set_fault_enable(pte);
913		−
914		− did = domain->iommu_did[iommu->seq_id];
915		− pasid_set_domain_id(pte, did);
916		−
917		− pasid_set_address_width(pte, agaw);
918		− pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
919		−
920		− pasid_set_translation_type(pte, PASID_ENTRY_PGTT_NESTED);
921		− pasid_set_present(pte);
922		− pasid_flush_caches(iommu, pte, pasid, did);
923		−
924		− return ret;
925		−}

drivers/iommu/intel/pasid.h+0−4

@@ -118,10 +118,6 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, · 117 unmodified lines
118	118	int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
119	119	struct dmar_domain *domain,
120	120	struct device *dev, u32 pasid);
121		−int intel_pasid_setup_nested(struct intel_iommu *iommu,
122		− struct device dev, pgd_t pgd, u32 pasid,
123		− struct iommu_gpasid_bind_data_vtd *pasid_data,
124		− struct dmar_domain *domain, int addr_width);
125	121	void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
126	122	struct device *dev, u32 pasid,
127	123	bool fault_ignore);

drivers/iommu/intel/svm.c+3−217

@@ -168,11 +168,6 @@ int intel_svm_finish_prq(struct intel_iommu *iommu) · 167 unmodified lines
168	168	return 0;
169	169	}
170	170
171		−static inline bool intel_svm_capable(struct intel_iommu *iommu)
172		−{
173		− return iommu->flags & VTD_FLAG_SVM_CAPABLE;
174		−}
175		−
176	171	void intel_svm_check(struct intel_iommu *iommu)
177	172	{
178	173	if (!pasid_supported(iommu))
@@ -200,7 +195,7 @@ static void __flush_svm_range_dev(struct intel_svm *svm, · 21 unmodified lines
200	195	unsigned long address,
201	196	unsigned long pages, int ih)
202	197	{
203		− struct device_domain_info *info = get_domain_info(sdev->dev);
	198	+ struct device_domain_info *info = dev_iommu_priv_get(sdev->dev);
204	199
205	200	if (WARN_ON(!pages))
206	201	return;
@@ -318,193 +313,6 @@ out: · 111 unmodified lines
318	313	return 0;
319	314	}
320	315
321		−int intel_svm_bind_gpasid(struct iommu_domain domain, struct device dev,
322		− struct iommu_gpasid_bind_data *data)
323		−{
324		− struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
325		− struct intel_svm_dev *sdev = NULL;
326		− struct dmar_domain *dmar_domain;
327		− struct device_domain_info *info;
328		− struct intel_svm *svm = NULL;
329		− unsigned long iflags;
330		− int ret = 0;
331		−
332		− if (WARN_ON(!iommu) \|\| !data)
333		− return -EINVAL;
334		−
335		− if (data->format != IOMMU_PASID_FORMAT_INTEL_VTD)
336		− return -EINVAL;
337		−
338		− /* IOMMU core ensures argsz is more than the start of the union */
339		− if (data->argsz < offsetofend(struct iommu_gpasid_bind_data, vendor.vtd))
340		− return -EINVAL;
341		−
342		− /* Make sure no undefined flags are used in vendor data */
343		− if (data->vendor.vtd.flags & ~(IOMMU_SVA_VTD_GPASID_LAST - 1))
344		− return -EINVAL;
345		−
346		− if (!dev_is_pci(dev))
347		− return -ENOTSUPP;
348		−
349		− /* VT-d supports devices with full 20 bit PASIDs only */
350		− if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX)
351		− return -EINVAL;
352		−
353		− /*
354		− * We only check host PASID range, we have no knowledge to check
355		− * guest PASID range.
356		− */
357		− if (data->hpasid <= 0 \|\| data->hpasid >= PASID_MAX)
358		− return -EINVAL;
359		−
360		− info = get_domain_info(dev);
361		− if (!info)
362		− return -EINVAL;
363		−
364		− dmar_domain = to_dmar_domain(domain);
365		−
366		− mutex_lock(&pasid_mutex);
367		− ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev);
368		− if (ret)
369		− goto out;
370		−
371		− if (sdev) {
372		− /*
373		− * Do not allow multiple bindings of the same device-PASID since
374		− * there is only one SL page tables per PASID. We may revisit
375		− * once sharing PGD across domains are supported.
376		− */
377		− dev_warn_ratelimited(dev, "Already bound with PASID %u\n",
378		− svm->pasid);
379		− ret = -EBUSY;
380		− goto out;
381		− }
382		−
383		− if (!svm) {
384		− /* We come here when PASID has never been bond to a device. */
385		− svm = kzalloc(sizeof(*svm), GFP_KERNEL);
386		− if (!svm) {
387		− ret = -ENOMEM;
388		− goto out;
389		− }
390		− /* REVISIT: upper layer/VFIO can track host process that bind
391		− * the PASID. ioasid_set = mm might be sufficient for vfio to
392		− * check pasid VMM ownership. We can drop the following line
393		− * once VFIO and IOASID set check is in place.
394		− */
395		− svm->mm = get_task_mm(current);
396		− svm->pasid = data->hpasid;
397		− if (data->flags & IOMMU_SVA_GPASID_VAL) {
398		− svm->gpasid = data->gpasid;
399		− svm->flags \|= SVM_FLAG_GUEST_PASID;
400		− }
401		− pasid_private_add(data->hpasid, svm);
402		− INIT_LIST_HEAD_RCU(&svm->devs);
403		− mmput(svm->mm);
404		− }
405		− sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
406		− if (!sdev) {
407		− ret = -ENOMEM;
408		− goto out;
409		− }
410		− sdev->dev = dev;
411		− sdev->sid = PCI_DEVID(info->bus, info->devfn);
412		− sdev->iommu = iommu;
413		−
414		− /* Only count users if device has aux domains */
415		− if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
416		− sdev->users = 1;
417		−
418		− /* Set up device context entry for PASID if not enabled already */
419		− ret = intel_iommu_enable_pasid(iommu, sdev->dev);
420		− if (ret) {
421		− dev_err_ratelimited(dev, "Failed to enable PASID capability\n");
422		− kfree(sdev);
423		− goto out;
424		− }
425		−
426		− /*
427		− * PASID table is per device for better security. Therefore, for
428		− * each bind of a new device even with an existing PASID, we need to
429		− * call the nested mode setup function here.
430		− */
431		− spin_lock_irqsave(&iommu->lock, iflags);
432		− ret = intel_pasid_setup_nested(iommu, dev,
433		− (pgd_t *)(uintptr_t)data->gpgd,
434		− data->hpasid, &data->vendor.vtd, dmar_domain,
435		− data->addr_width);
436		− spin_unlock_irqrestore(&iommu->lock, iflags);
437		− if (ret) {
438		− dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n",
439		− data->hpasid, ret);
440		− /*
441		− * PASID entry should be in cleared state if nested mode
442		− * set up failed. So we only need to clear IOASID tracking
443		− * data such that free call will succeed.
444		− */
445		− kfree(sdev);
446		− goto out;
447		− }
448		−
449		− svm->flags \|= SVM_FLAG_GUEST_MODE;
450		−
451		− init_rcu_head(&sdev->rcu);
452		− list_add_rcu(&sdev->list, &svm->devs);
453		− out:
454		− if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) {
455		− pasid_private_remove(data->hpasid);
456		− kfree(svm);
457		− }
458		−
459		− mutex_unlock(&pasid_mutex);
460		− return ret;
461		−}
462		−
463		−int intel_svm_unbind_gpasid(struct device *dev, u32 pasid)
464		−{
465		− struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
466		− struct intel_svm_dev *sdev;
467		− struct intel_svm *svm;
468		− int ret;
469		−
470		− if (WARN_ON(!iommu))
471		− return -EINVAL;
472		−
473		− mutex_lock(&pasid_mutex);
474		− ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);
475		− if (ret)
476		− goto out;
477		−
478		− if (sdev) {
479		− if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
480		− sdev->users--;
481		− if (!sdev->users) {
482		− list_del_rcu(&sdev->list);
483		− intel_pasid_tear_down_entry(iommu, dev,
484		− svm->pasid, false);
485		− intel_svm_drain_prq(dev, svm->pasid);
486		− kfree_rcu(sdev, rcu);
487		−
488		− if (list_empty(&svm->devs)) {
489		− /*
490		− * We do not free the IOASID here in that
491		− * IOMMU driver did not allocate it.
492		− * Unlike native SVM, IOASID for guest use was
493		− * allocated prior to the bind call.
494		− * In any case, if the free call comes before
495		− * the unbind, IOMMU driver will get notified
496		− * and perform cleanup.
497		− */
498		− pasid_private_remove(pasid);
499		− kfree(svm);
500		− }
501		− }
502		− }
503		−out:
504		− mutex_unlock(&pasid_mutex);
505		− return ret;
506		−}
507		−
508	316	static int intel_svm_alloc_pasid(struct device dev, struct mm_struct mm,
509	317	unsigned int flags)
510	318	{
@@ -519,7 +327,7 @@ static struct iommu_sva intel_svm_bind_mm(struct intel_iommu iommu, · 8 unmodified lines
519	327	struct mm_struct *mm,
520	328	unsigned int flags)
521	329	{
522		− struct device_domain_info *info = get_domain_info(dev);
	330	+ struct device_domain_info *info = dev_iommu_priv_get(dev);
523	331	unsigned long iflags, sflags;
524	332	struct intel_svm_dev *sdev;
525	333	struct intel_svm *svm;
@@ -725,7 +533,7 @@ static void intel_svm_drain_prq(struct device *dev, u32 pasid) · 199 unmodified lines
725	533	u16 sid, did;
726	534	int qdep;
727	535
728		− info = get_domain_info(dev);
	536	+ info = dev_iommu_priv_get(dev);
729	537	if (WARN_ON(!info \|\| !dev_is_pci(dev)))
730	538	return;
731	539
@@ -1116,28 +924,6 @@ int intel_svm_page_response(struct device *dev, · 384 unmodified lines
1116	924	goto out;
1117	925	}
1118	926
1119		− /*
1120		− * For responses from userspace, need to make sure that the
1121		− * pasid has been bound to its mm.
1122		− */
1123		− if (svm->flags & SVM_FLAG_GUEST_MODE) {
1124		− struct mm_struct *mm;
1125		−
1126		− mm = get_task_mm(current);
1127		− if (!mm) {
1128		− ret = -EINVAL;
1129		− goto out;
1130		− }
1131		−
1132		− if (mm != svm->mm) {
1133		− ret = -ENODEV;
1134		− mmput(mm);
1135		− goto out;
1136		− }
1137		−
1138		− mmput(mm);
1139		− }
1140		−
1141	927	/*
1142	928	* Per VT-d spec. v3.0 ch7.7, system software must respond
1143	929	* with page group response if private data is present (PDP)

drivers/iommu/iommu.c+39−300

@@ -323,13 +323,14 @@ err_out: · 322 unmodified lines
323	323
324	324	void iommu_release_device(struct device *dev)
325	325	{
326		− const struct iommu_ops *ops = dev->bus->iommu_ops;
	326	+ const struct iommu_ops *ops;
327	327
328	328	if (!dev->iommu)
329	329	return;
330	330
331	331	iommu_device_unlink(dev->iommu->iommu_dev, dev);
332	332
	333	+ ops = dev_iommu_ops(dev);
333	334	ops->release_device(dev);
334	335
335	336	iommu_group_remove_device(dev);
@@ -790,9 +791,6 @@ static int iommu_create_device_direct_mappings(struct iommu_group *group, · 454 unmodified lines
790	791	dma_addr_t start, end, addr;
791	792	size_t map_size = 0;
792	793
793		− if (domain->ops->apply_resv_region)
794		− domain->ops->apply_resv_region(dev, domain, entry);
795		−
796	794	start = ALIGN(entry->start, pg_size);
797	795	end = ALIGN(entry->start + entry->length, pg_size);
798	796
@@ -833,11 +831,12 @@ out: · 34 unmodified lines
833	831	return ret;
834	832	}
835	833
836		−static bool iommu_is_attach_deferred(struct iommu_domain *domain,
837		− struct device *dev)
	834	+static bool iommu_is_attach_deferred(struct device *dev)
838	835	{
839		− if (domain->ops->is_attach_deferred)
840		− return domain->ops->is_attach_deferred(domain, dev);
	836	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
	837	+
	838	+ if (ops->is_attach_deferred)
	839	+ return ops->is_attach_deferred(dev);
841	840
842	841	return false;
843	842	}
@@ -894,7 +893,7 @@ rename: · 50 unmodified lines
894	893
895	894	mutex_lock(&group->mutex);
896	895	list_add_tail(&device->list, &group->devices);
897		− if (group->domain && !iommu_is_attach_deferred(group->domain, dev))
	896	+ if (group->domain && !iommu_is_attach_deferred(dev))
898	897	ret = __iommu_attach_device(group->domain, dev);
899	898	mutex_unlock(&group->mutex);
900	899	if (ret)
@@ -1255,10 +1254,10 @@ int iommu_page_response(struct device *dev, · 354 unmodified lines
1255	1254	struct iommu_fault_event *evt;
1256	1255	struct iommu_fault_page_request *prm;
1257	1256	struct dev_iommu *param = dev->iommu;
	1257	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
1258	1258	bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID;
1259		− struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1260	1259
1261		− if (!domain \|\| !domain->ops->page_response)
	1260	+ if (!ops->page_response)
1262	1261	return -ENODEV;
1263	1262
1264	1263	if (!param \|\| !param->fault_param)
@@ -1299,7 +1298,7 @@ int iommu_page_response(struct device *dev, · 34 unmodified lines
1299	1298	msg->pasid = 0;
1300	1299	}
1301	1300
1302		− ret = domain->ops->page_response(dev, evt, msg);
	1301	+ ret = ops->page_response(dev, evt, msg);
1303	1302	list_del(&evt->list);
1304	1303	kfree(evt);
1305	1304	break;
@@ -1524,7 +1523,7 @@ EXPORT_SYMBOL_GPL(fsl_mc_device_group); · 218 unmodified lines
1524	1523
1525	1524	static int iommu_get_def_domain_type(struct device *dev)
1526	1525	{
1527		− const struct iommu_ops *ops = dev->bus->iommu_ops;
	1526	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
1528	1527
1529	1528	if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
1530	1529	return IOMMU_DOMAIN_DMA;
@@ -1583,7 +1582,7 @@ static int iommu_alloc_default_domain(struct iommu_group *group, · 52 unmodified lines
1583	1582	*/
1584	1583	static struct iommu_group iommu_group_get_for_dev(struct device dev)
1585	1584	{
1586		− const struct iommu_ops *ops = dev->bus->iommu_ops;
	1585	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
1587	1586	struct iommu_group *group;
1588	1587	int ret;
1589	1588
@@ -1591,9 +1590,6 @@ static struct iommu_group iommu_group_get_for_dev(struct device dev) · 1 unmodified line
1591	1590	if (group)
1592	1591	return group;
1593	1592
1594		− if (!ops)
1595		− return ERR_PTR(-EINVAL);
1596		−
1597	1593	group = ops->device_group(dev);
1598	1594	if (WARN_ON_ONCE(group == NULL))
1599	1595	return ERR_PTR(-EINVAL);
@@ -1748,7 +1744,7 @@ static int iommu_group_do_dma_attach(struct device dev, void data) · 148 unmodified lines
1748	1744	struct iommu_domain *domain = data;
1749	1745	int ret = 0;
1750	1746
1751		− if (!iommu_is_attach_deferred(domain, dev))
	1747	+ if (!iommu_is_attach_deferred(dev))
1752	1748	ret = __iommu_attach_device(domain, dev);
1753	1749
1754	1750	return ret;
@@ -1762,10 +1758,10 @@ static int __iommu_group_dma_attach(struct iommu_group *group) · 7 unmodified lines
1762	1758
1763	1759	static int iommu_group_do_probe_finalize(struct device dev, void data)
1764	1760	{
1765		− struct iommu_domain *domain = data;
	1761	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
1766	1762
1767		− if (domain->ops->probe_finalize)
1768		− domain->ops->probe_finalize(dev);
	1763	+ if (ops->probe_finalize)
	1764	+ ops->probe_finalize(dev);
1769	1765
1770	1766	return 0;
1771	1767	}
@@ -1954,10 +1950,11 @@ static struct iommu_domain __iommu_domain_alloc(struct bus_type bus, · 182 unmodified lines
1954	1950	if (!domain)
1955	1951	return NULL;
1956	1952
1957		− domain->ops = bus->iommu_ops;
1958	1953	domain->type = type;
1959	1954	/* Assume all sizes by default; the driver may override this later */
1960		− domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
	1955	+ domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
	1956	+ if (!domain->ops)
	1957	+ domain->ops = bus->iommu_ops->default_domain_ops;
1961	1958
1962	1959	if (iommu_is_dma_domain(domain) && iommu_get_dma_cookie(domain)) {
1963	1960	iommu_domain_free(domain);
@@ -1975,7 +1972,7 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc); · 11 unmodified lines
1975	1972	void iommu_domain_free(struct iommu_domain *domain)
1976	1973	{
1977	1974	iommu_put_dma_cookie(domain);
1978		− domain->ops->domain_free(domain);
	1975	+ domain->ops->free(domain);
1979	1976	}
1980	1977	EXPORT_SYMBOL_GPL(iommu_domain_free);
1981	1978
@@ -2023,228 +2020,16 @@ EXPORT_SYMBOL_GPL(iommu_attach_device); · 41 unmodified lines
2023	2020
2024	2021	int iommu_deferred_attach(struct device dev, struct iommu_domain domain)
2025	2022	{
2026		− const struct iommu_ops *ops = domain->ops;
2027		−
2028		− if (ops->is_attach_deferred && ops->is_attach_deferred(domain, dev))
	2023	+ if (iommu_is_attach_deferred(dev))
2029	2024	return __iommu_attach_device(domain, dev);
2030	2025
2031	2026	return 0;
2032	2027	}
2033	2028
2034		−/*
2035		− * Check flags and other user provided data for valid combinations. We also
2036		− * make sure no reserved fields or unused flags are set. This is to ensure
2037		− * not breaking userspace in the future when these fields or flags are used.
2038		− */
2039		−static int iommu_check_cache_invl_data(struct iommu_cache_invalidate_info *info)
2040		−{
2041		− u32 mask;
2042		− int i;
2043		−
2044		− if (info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1)
2045		− return -EINVAL;
2046		−
2047		− mask = (1 << IOMMU_CACHE_INV_TYPE_NR) - 1;
2048		− if (info->cache & ~mask)
2049		− return -EINVAL;
2050		−
2051		− if (info->granularity >= IOMMU_INV_GRANU_NR)
2052		− return -EINVAL;
2053		−
2054		− switch (info->granularity) {
2055		− case IOMMU_INV_GRANU_ADDR:
2056		− if (info->cache & IOMMU_CACHE_INV_TYPE_PASID)
2057		− return -EINVAL;
2058		−
2059		− mask = IOMMU_INV_ADDR_FLAGS_PASID \|
2060		− IOMMU_INV_ADDR_FLAGS_ARCHID \|
2061		− IOMMU_INV_ADDR_FLAGS_LEAF;
2062		−
2063		− if (info->granu.addr_info.flags & ~mask)
2064		− return -EINVAL;
2065		− break;
2066		− case IOMMU_INV_GRANU_PASID:
2067		− mask = IOMMU_INV_PASID_FLAGS_PASID \|
2068		− IOMMU_INV_PASID_FLAGS_ARCHID;
2069		− if (info->granu.pasid_info.flags & ~mask)
2070		− return -EINVAL;
2071		−
2072		− break;
2073		− case IOMMU_INV_GRANU_DOMAIN:
2074		− if (info->cache & IOMMU_CACHE_INV_TYPE_DEV_IOTLB)
2075		− return -EINVAL;
2076		− break;
2077		− default:
2078		− return -EINVAL;
2079		− }
2080		−
2081		− /* Check reserved padding fields */
2082		− for (i = 0; i < sizeof(info->padding); i++) {
2083		− if (info->padding[i])
2084		− return -EINVAL;
2085		− }
2086		−
2087		− return 0;
2088		−}
2089		−
2090		−int iommu_uapi_cache_invalidate(struct iommu_domain domain, struct device dev,
2091		− void __user *uinfo)
2092		−{
2093		− struct iommu_cache_invalidate_info inv_info = { 0 };
2094		− u32 minsz;
2095		− int ret;
2096		−
2097		− if (unlikely(!domain->ops->cache_invalidate))
2098		− return -ENODEV;
2099		−
2100		− /*
2101		− * No new spaces can be added before the variable sized union, the
2102		− * minimum size is the offset to the union.
2103		− */
2104		− minsz = offsetof(struct iommu_cache_invalidate_info, granu);
2105		−
2106		− /* Copy minsz from user to get flags and argsz */
2107		− if (copy_from_user(&inv_info, uinfo, minsz))
2108		− return -EFAULT;
2109		−
2110		− /* Fields before the variable size union are mandatory */
2111		− if (inv_info.argsz < minsz)
2112		− return -EINVAL;
2113		−
2114		− /* PASID and address granu require additional info beyond minsz */
2115		− if (inv_info.granularity == IOMMU_INV_GRANU_PASID &&
2116		− inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.pasid_info))
2117		− return -EINVAL;
2118		−
2119		− if (inv_info.granularity == IOMMU_INV_GRANU_ADDR &&
2120		− inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.addr_info))
2121		− return -EINVAL;
2122		−
2123		− /*
2124		− * User might be using a newer UAPI header which has a larger data
2125		− * size, we shall support the existing flags within the current
2126		− * size. Copy the remaining user data _after_ minsz but not more
2127		− * than the current kernel supported size.
2128		− */
2129		− if (copy_from_user((void *)&inv_info + minsz, uinfo + minsz,
2130		− min_t(u32, inv_info.argsz, sizeof(inv_info)) - minsz))
2131		− return -EFAULT;
2132		−
2133		− /* Now the argsz is validated, check the content */
2134		− ret = iommu_check_cache_invl_data(&inv_info);
2135		− if (ret)
2136		− return ret;
2137		−
2138		− return domain->ops->cache_invalidate(domain, dev, &inv_info);
2139		−}
2140		−EXPORT_SYMBOL_GPL(iommu_uapi_cache_invalidate);
2141		−
2142		−static int iommu_check_bind_data(struct iommu_gpasid_bind_data *data)
2143		−{
2144		− u64 mask;
2145		− int i;
2146		−
2147		− if (data->version != IOMMU_GPASID_BIND_VERSION_1)
2148		− return -EINVAL;
2149		−
2150		− /* Check the range of supported formats */
2151		− if (data->format >= IOMMU_PASID_FORMAT_LAST)
2152		− return -EINVAL;
2153		−
2154		− /* Check all flags */
2155		− mask = IOMMU_SVA_GPASID_VAL;
2156		− if (data->flags & ~mask)
2157		− return -EINVAL;
2158		−
2159		− /* Check reserved padding fields */
2160		− for (i = 0; i < sizeof(data->padding); i++) {
2161		− if (data->padding[i])
2162		− return -EINVAL;
2163		− }
2164		−
2165		− return 0;
2166		−}
2167		−
2168		−static int iommu_sva_prepare_bind_data(void __user *udata,
2169		− struct iommu_gpasid_bind_data *data)
2170		−{
2171		− u32 minsz;
2172		−
2173		− /*
2174		− * No new spaces can be added before the variable sized union, the
2175		− * minimum size is the offset to the union.
2176		− */
2177		− minsz = offsetof(struct iommu_gpasid_bind_data, vendor);
2178		−
2179		− /* Copy minsz from user to get flags and argsz */
2180		− if (copy_from_user(data, udata, minsz))
2181		− return -EFAULT;
2182		−
2183		− /* Fields before the variable size union are mandatory */
2184		− if (data->argsz < minsz)
2185		− return -EINVAL;
2186		− /*
2187		− * User might be using a newer UAPI header, we shall let IOMMU vendor
2188		− * driver decide on what size it needs. Since the guest PASID bind data
2189		− * can be vendor specific, larger argsz could be the result of extension
2190		− * for one vendor but it should not affect another vendor.
2191		− * Copy the remaining user data _after_ minsz
2192		− */
2193		− if (copy_from_user((void *)data + minsz, udata + minsz,
2194		− min_t(u32, data->argsz, sizeof(*data)) - minsz))
2195		− return -EFAULT;
2196		−
2197		− return iommu_check_bind_data(data);
2198		−}
2199		−
2200		−int iommu_uapi_sva_bind_gpasid(struct iommu_domain domain, struct device dev,
2201		− void __user *udata)
2202		−{
2203		− struct iommu_gpasid_bind_data data = { 0 };
2204		− int ret;
2205		−
2206		− if (unlikely(!domain->ops->sva_bind_gpasid))
2207		− return -ENODEV;
2208		−
2209		− ret = iommu_sva_prepare_bind_data(udata, &data);
2210		− if (ret)
2211		− return ret;
2212		−
2213		− return domain->ops->sva_bind_gpasid(domain, dev, &data);
2214		−}
2215		−EXPORT_SYMBOL_GPL(iommu_uapi_sva_bind_gpasid);
2216		−
2217		−int iommu_sva_unbind_gpasid(struct iommu_domain domain, struct device dev,
2218		− ioasid_t pasid)
2219		−{
2220		− if (unlikely(!domain->ops->sva_unbind_gpasid))
2221		− return -ENODEV;
2222		−
2223		− return domain->ops->sva_unbind_gpasid(dev, pasid);
2224		−}
2225		−EXPORT_SYMBOL_GPL(iommu_sva_unbind_gpasid);
2226		−
2227		−int iommu_uapi_sva_unbind_gpasid(struct iommu_domain domain, struct device dev,
2228		− void __user *udata)
2229		−{
2230		− struct iommu_gpasid_bind_data data = { 0 };
2231		− int ret;
2232		−
2233		− if (unlikely(!domain->ops->sva_bind_gpasid))
2234		− return -ENODEV;
2235		−
2236		− ret = iommu_sva_prepare_bind_data(udata, &data);
2237		− if (ret)
2238		− return ret;
2239		−
2240		− return iommu_sva_unbind_gpasid(domain, dev, data.hpasid);
2241		−}
2242		−EXPORT_SYMBOL_GPL(iommu_uapi_sva_unbind_gpasid);
2243		−
2244	2029	static void __iommu_detach_device(struct iommu_domain *domain,
2245	2030	struct device *dev)
2246	2031	{
2247		− if (iommu_is_attach_deferred(domain, dev))
	2032	+ if (iommu_is_attach_deferred(dev))
2248	2033	return;
2249	2034
2250	2035	if (unlikely(domain->ops->detach_dev == NULL))
@@ -2458,7 +2243,7 @@ static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova, · 207 unmodified lines
2458	2243	phys_addr_t paddr, size_t size, int prot,
2459	2244	gfp_t gfp, size_t *mapped)
2460	2245	{
2461		− const struct iommu_ops *ops = domain->ops;
	2246	+ const struct iommu_domain_ops *ops = domain->ops;
2462	2247	size_t pgsize, count;
2463	2248	int ret;
2464	2249
@@ -2481,7 +2266,7 @@ static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova, · 16 unmodified lines
2481	2266	static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
2482	2267	phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2483	2268	{
2484		− const struct iommu_ops *ops = domain->ops;
	2269	+ const struct iommu_domain_ops *ops = domain->ops;
2485	2270	unsigned long orig_iova = iova;
2486	2271	unsigned int min_pagesz;
2487	2272	size_t orig_size = size;
@@ -2541,7 +2326,7 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova, · 53 unmodified lines
2541	2326	static int _iommu_map(struct iommu_domain *domain, unsigned long iova,
2542	2327	phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2543	2328	{
2544		− const struct iommu_ops *ops = domain->ops;
	2329	+ const struct iommu_domain_ops *ops = domain->ops;
2545	2330	int ret;
2546	2331
2547	2332	ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
@@ -2570,7 +2355,7 @@ static size_t __iommu_unmap_pages(struct iommu_domain *domain, · 22 unmodified lines
2570	2355	unsigned long iova, size_t size,
2571	2356	struct iommu_iotlb_gather *iotlb_gather)
2572	2357	{
2573		− const struct iommu_ops *ops = domain->ops;
	2358	+ const struct iommu_domain_ops *ops = domain->ops;
2574	2359	size_t pgsize, count;
2575	2360
2576	2361	pgsize = iommu_pgsize(domain, iova, iova, size, &count);
@@ -2583,7 +2368,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain, · 6 unmodified lines
2583	2368	unsigned long iova, size_t size,
2584	2369	struct iommu_iotlb_gather *iotlb_gather)
2585	2370	{
2586		− const struct iommu_ops *ops = domain->ops;
	2371	+ const struct iommu_domain_ops *ops = domain->ops;
2587	2372	size_t unmapped_page, unmapped = 0;
2588	2373	unsigned long orig_iova = iova;
2589	2374	unsigned int min_pagesz;
@@ -2659,7 +2444,7 @@ static ssize_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, · 69 unmodified lines
2659	2444	struct scatterlist *sg, unsigned int nents, int prot,
2660	2445	gfp_t gfp)
2661	2446	{
2662		− const struct iommu_ops *ops = domain->ops;
	2447	+ const struct iommu_domain_ops *ops = domain->ops;
2663	2448	size_t len = 0, mapped = 0;
2664	2449	phys_addr_t start;
2665	2450	unsigned int i = 0;
@@ -2792,17 +2577,17 @@ EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks); · 126 unmodified lines
2792	2577
2793	2578	void iommu_get_resv_regions(struct device dev, struct list_head list)
2794	2579	{
2795		− const struct iommu_ops *ops = dev->bus->iommu_ops;
	2580	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
2796	2581
2797		− if (ops && ops->get_resv_regions)
	2582	+ if (ops->get_resv_regions)
2798	2583	ops->get_resv_regions(dev, list);
2799	2584	}
2800	2585
2801	2586	void iommu_put_resv_regions(struct device dev, struct list_head list)
2802	2587	{
2803		− const struct iommu_ops *ops = dev->bus->iommu_ops;
	2588	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
2804	2589
2805		− if (ops && ops->put_resv_regions)
	2590	+ if (ops->put_resv_regions)
2806	2591	ops->put_resv_regions(dev, list);
2807	2592	}
2808	2593
@@ -2959,8 +2744,6 @@ EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); · 150 unmodified lines
2959	2744
2960	2745	/*
2961	2746	* The device drivers should do the necessary cleanups before calling this.
2962		− * For example, before disabling the aux-domain feature, the device driver
2963		− * should detach all aux-domains. Otherwise, this will return -EBUSY.
2964	2747	*/
2965	2748	int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
2966	2749	{
@@ -2988,50 +2771,6 @@ bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) · 21 unmodified lines
2988	2771	}
2989	2772	EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled);
2990	2773
2991		−/*
2992		− * Aux-domain specific attach/detach.
2993		− *
2994		− * Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns
2995		− * true. Also, as long as domains are attached to a device through this
2996		− * interface, any tries to call iommu_attach_device() should fail
2997		− * (iommu_detach_device() can't fail, so we fail when trying to re-attach).
2998		− * This should make us safe against a device being attached to a guest as a
2999		− * whole while there are still pasid users on it (aux and sva).
3000		− */
3001		−int iommu_aux_attach_device(struct iommu_domain domain, struct device dev)
3002		−{
3003		− int ret = -ENODEV;
3004		−
3005		− if (domain->ops->aux_attach_dev)
3006		− ret = domain->ops->aux_attach_dev(domain, dev);
3007		−
3008		− if (!ret)
3009		− trace_attach_device_to_domain(dev);
3010		−
3011		− return ret;
3012		−}
3013		−EXPORT_SYMBOL_GPL(iommu_aux_attach_device);
3014		−
3015		−void iommu_aux_detach_device(struct iommu_domain domain, struct device dev)
3016		−{
3017		− if (domain->ops->aux_detach_dev) {
3018		− domain->ops->aux_detach_dev(domain, dev);
3019		− trace_detach_device_from_domain(dev);
3020		− }
3021		−}
3022		−EXPORT_SYMBOL_GPL(iommu_aux_detach_device);
3023		−
3024		−int iommu_aux_get_pasid(struct iommu_domain domain, struct device dev)
3025		−{
3026		− int ret = -ENODEV;
3027		−
3028		− if (domain->ops->aux_get_pasid)
3029		− ret = domain->ops->aux_get_pasid(domain, dev);
3030		−
3031		− return ret;
3032		−}
3033		−EXPORT_SYMBOL_GPL(iommu_aux_get_pasid);
3034		−
3035	2774	/**
3036	2775	* iommu_sva_bind_device() - Bind a process address space to a device
3037	2776	* @dev: the device
@@ -3053,9 +2792,9 @@ iommu_sva_bind_device(struct device dev, struct mm_struct mm, void *drvdata) · 15 unmodified lines
3053	2792	{
3054	2793	struct iommu_group *group;
3055	2794	struct iommu_sva *handle = ERR_PTR(-EINVAL);
3056		− const struct iommu_ops *ops = dev->bus->iommu_ops;
	2795	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
3057	2796
3058		− if (!ops \|\| !ops->sva_bind)
	2797	+ if (!ops->sva_bind)
3059	2798	return ERR_PTR(-ENODEV);
3060	2799
3061	2800	group = iommu_group_get(dev);
@@ -3096,9 +2835,9 @@ void iommu_sva_unbind_device(struct iommu_sva *handle) · 34 unmodified lines
3096	2835	{
3097	2836	struct iommu_group *group;
3098	2837	struct device *dev = handle->dev;
3099		− const struct iommu_ops *ops = dev->bus->iommu_ops;
	2838	+ const struct iommu_ops *ops = dev_iommu_ops(dev);
3100	2839
3101		− if (!ops \|\| !ops->sva_unbind)
	2840	+ if (!ops->sva_unbind)
3102	2841	return;
3103	2842
3104	2843	group = iommu_group_get(dev);
@@ -3115,9 +2854,9 @@ EXPORT_SYMBOL_GPL(iommu_sva_unbind_device); · 10 unmodified lines
3115	2854
3116	2855	u32 iommu_sva_get_pasid(struct iommu_sva *handle)
3117	2856	{
3118		− const struct iommu_ops *ops = handle->dev->bus->iommu_ops;
	2857	+ const struct iommu_ops *ops = dev_iommu_ops(handle->dev);
3119	2858
3120		− if (!ops \|\| !ops->sva_get_pasid)
	2859	+ if (!ops->sva_get_pasid)
3121	2860	return IOMMU_PASID_INVALID;
3122	2861
3123	2862	return ops->sva_get_pasid(handle);

drivers/iommu/iova.c+62−16

@@ -15,13 +15,14 @@ · 14 unmodified lines
15	15	/* The anchor node sits above the top of the usable address space */
16	16	#define IOVA_ANCHOR ~0UL
17	17
	18	+#define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */
	19	+
18	20	static bool iova_rcache_insert(struct iova_domain *iovad,
19	21	unsigned long pfn,
20	22	unsigned long size);
21	23	static unsigned long iova_rcache_get(struct iova_domain *iovad,
22	24	unsigned long size,
23	25	unsigned long limit_pfn);
24		−static void init_iova_rcaches(struct iova_domain *iovad);
25	26	static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
26	27	static void free_iova_rcaches(struct iova_domain *iovad);
27	28
@@ -64,8 +65,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, · 36 unmodified lines
64	65	iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
65	66	rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
66	67	rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
67		− cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, &iovad->cpuhp_dead);
68		− init_iova_rcaches(iovad);
69	68	}
70	69	EXPORT_SYMBOL_GPL(init_iova_domain);
71	70
@@ -95,10 +94,11 @@ __cached_rbnode_delete_update(struct iova_domain iovad, struct iova free) · 23 unmodified lines
95	94	cached_iova = to_iova(iovad->cached32_node);
96	95	if (free == cached_iova \|\|
97	96	(free->pfn_hi < iovad->dma_32bit_pfn &&
98		− free->pfn_lo >= cached_iova->pfn_lo)) {
	97	+ free->pfn_lo >= cached_iova->pfn_lo))
99	98	iovad->cached32_node = rb_next(&free->node);
	99	+
	100	+ if (free->pfn_lo < iovad->dma_32bit_pfn)
100	101	iovad->max32_alloc_size = iovad->dma_32bit_pfn;
101		− }
102	102
103	103	cached_iova = to_iova(iovad->cached_node);
104	104	if (free->pfn_lo >= cached_iova->pfn_lo)
@@ -488,6 +488,13 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) · 383 unmodified lines
488	488	}
489	489	EXPORT_SYMBOL_GPL(free_iova_fast);
490	490
	491	+static void iova_domain_free_rcaches(struct iova_domain *iovad)
	492	+{
	493	+ cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
	494	+ &iovad->cpuhp_dead);
	495	+ free_iova_rcaches(iovad);
	496	+}
	497	+
491	498	/**
492	499	* put_iova_domain - destroys the iova domain
493	500	* @iovad: - iova domain in question.
@@ -497,9 +504,9 @@ void put_iova_domain(struct iova_domain *iovad) · 3 unmodified lines
497	504	{
498	505	struct iova iova, tmp;
499	506
500		− cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
501		− &iovad->cpuhp_dead);
502		− free_iova_rcaches(iovad);
	507	+ if (iovad->rcaches)
	508	+ iova_domain_free_rcaches(iovad);
	509	+
503	510	rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
504	511	free_iova_mem(iova);
505	512	}
@@ -608,6 +615,7 @@ EXPORT_SYMBOL_GPL(reserve_iova); · 102 unmodified lines
608	615	*/
609	616
610	617	#define IOVA_MAG_SIZE 128
	618	+#define MAX_GLOBAL_MAGS 32 /* magazines per bin */
611	619
612	620	struct iova_magazine {
613	621	unsigned long size;
@@ -620,6 +628,13 @@ struct iova_cpu_rcache { · 6 unmodified lines
620	628	struct iova_magazine *prev;
621	629	};
622	630
	631	+struct iova_rcache {
	632	+ spinlock_t lock;
	633	+ unsigned long depot_size;
	634	+ struct iova_magazine *depot[MAX_GLOBAL_MAGS];
	635	+ struct iova_cpu_rcache __percpu *cpu_rcaches;
	636	+};
	637	+
623	638	static struct iova_magazine *iova_magazine_alloc(gfp_t flags)
624	639	{
625	640	return kzalloc(sizeof(struct iova_magazine), flags);
@@ -693,28 +708,54 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn) · 67 unmodified lines
693	708	mag->pfns[mag->size++] = pfn;
694	709	}
695	710
696		−static void init_iova_rcaches(struct iova_domain *iovad)
	711	+int iova_domain_init_rcaches(struct iova_domain *iovad)
697	712	{
698		− struct iova_cpu_rcache *cpu_rcache;
699		− struct iova_rcache *rcache;
700	713	unsigned int cpu;
701		− int i;
	714	+ int i, ret;
	715	+
	716	+ iovad->rcaches = kcalloc(IOVA_RANGE_CACHE_MAX_SIZE,
	717	+ sizeof(struct iova_rcache),
	718	+ GFP_KERNEL);
	719	+ if (!iovad->rcaches)
	720	+ return -ENOMEM;
702	721
703	722	for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
	723	+ struct iova_cpu_rcache *cpu_rcache;
	724	+ struct iova_rcache *rcache;
	725	+
704	726	rcache = &iovad->rcaches[i];
705	727	spin_lock_init(&rcache->lock);
706	728	rcache->depot_size = 0;
707		− rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
708		− if (WARN_ON(!rcache->cpu_rcaches))
709		− continue;
	729	+ rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache),
	730	+ cache_line_size());
	731	+ if (!rcache->cpu_rcaches) {
	732	+ ret = -ENOMEM;
	733	+ goto out_err;
	734	+ }
710	735	for_each_possible_cpu(cpu) {
711	736	cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
	737	+
712	738	spin_lock_init(&cpu_rcache->lock);
713	739	cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);
714	740	cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
	741	+ if (!cpu_rcache->loaded \|\| !cpu_rcache->prev) {
	742	+ ret = -ENOMEM;
	743	+ goto out_err;
	744	+ }
715	745	}
716	746	}
	747	+
	748	+ ret = cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
	749	+ &iovad->cpuhp_dead);
	750	+ if (ret)
	751	+ goto out_err;
	752	+ return 0;
	753	+
	754	+out_err:
	755	+ free_iova_rcaches(iovad);
	756	+ return ret;
717	757	}
	758	+EXPORT_SYMBOL_GPL(iova_domain_init_rcaches);
718	759
719	760	/*
720	761	* Try inserting IOVA range starting with 'iova_pfn' into 'rcache', and
@@ -831,7 +872,7 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, · 110 unmodified lines
831	872	{
832	873	unsigned int log_size = order_base_2(size);
833	874
834		− if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
	875	+ if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE \|\| !iovad->rcaches)
835	876	return 0;
836	877
837	878	return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);
@@ -849,6 +890,8 @@ static void free_iova_rcaches(struct iova_domain *iovad) · 11 unmodified lines
849	890
850	891	for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
851	892	rcache = &iovad->rcaches[i];
	893	+ if (!rcache->cpu_rcaches)
	894	+ break;
852	895	for_each_possible_cpu(cpu) {
853	896	cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
854	897	iova_magazine_free(cpu_rcache->loaded);
@@ -858,6 +901,9 @@ static void free_iova_rcaches(struct iova_domain *iovad) · 3 unmodified lines
858	901	for (j = 0; j < rcache->depot_size; ++j)
859	902	iova_magazine_free(rcache->depot[j]);
860	903	}
	904	+
	905	+ kfree(iovad->rcaches);
	906	+ iovad->rcaches = NULL;
861	907	}
862	908
863	909	/*

drivers/iommu/ipmmu-vmsa.c+20−12

@@ -719,6 +719,7 @@ static int ipmmu_init_platform_device(struct device *dev, · 718 unmodified lines
719	719
720	720	static const struct soc_device_attribute soc_needs_opt_in[] = {
721	721	{ .family = "R-Car Gen3", },
	722	+ { .family = "R-Car Gen4", },
722	723	{ .family = "RZ/G2", },
723	724	{ /* sentinel */ }
724	725	};
@@ -743,7 +744,7 @@ static bool ipmmu_device_is_allowed(struct device *dev) · 18 unmodified lines
743	744	unsigned int i;
744	745
745	746	/*
746		− * R-Car Gen3 and RZ/G2 use the allow list to opt-in devices.
	747	+ * R-Car Gen3/4 and RZ/G2 use the allow list to opt-in devices.
747	748	* For Other SoCs, this returns true anyway.
748	749	*/
749	750	if (!soc_device_match(soc_needs_opt_in))
@@ -868,14 +869,6 @@ static struct iommu_group ipmmu_find_group(struct device dev) · 118 unmodified lines
868	869
869	870	static const struct iommu_ops ipmmu_ops = {
870	871	.domain_alloc = ipmmu_domain_alloc,
871		− .domain_free = ipmmu_domain_free,
872		− .attach_dev = ipmmu_attach_device,
873		− .detach_dev = ipmmu_detach_device,
874		− .map = ipmmu_map,
875		− .unmap = ipmmu_unmap,
876		− .flush_iotlb_all = ipmmu_flush_iotlb_all,
877		− .iotlb_sync = ipmmu_iotlb_sync,
878		− .iova_to_phys = ipmmu_iova_to_phys,
879	872	.probe_device = ipmmu_probe_device,
880	873	.release_device = ipmmu_release_device,
881	874	.probe_finalize = ipmmu_probe_finalize,
@@ -883,6 +876,16 @@ static const struct iommu_ops ipmmu_ops = { · 1 unmodified line
883	876	? generic_device_group : ipmmu_find_group,
884	877	.pgsize_bitmap = SZ_1G \| SZ_2M \| SZ_4K,
885	878	.of_xlate = ipmmu_of_xlate,
	879	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	880	+ .attach_dev = ipmmu_attach_device,
	881	+ .detach_dev = ipmmu_detach_device,
	882	+ .map = ipmmu_map,
	883	+ .unmap = ipmmu_unmap,
	884	+ .flush_iotlb_all = ipmmu_flush_iotlb_all,
	885	+ .iotlb_sync = ipmmu_iotlb_sync,
	886	+ .iova_to_phys = ipmmu_iova_to_phys,
	887	+ .free = ipmmu_domain_free,
	888	+ }
886	889	};
887	890
888	891	/* -----------------------------------------------------------------------------
@@ -926,7 +929,7 @@ static const struct ipmmu_features ipmmu_features_rcar_gen3 = { · 37 unmodified lines
926	929	.utlb_offset_base = 0,
927	930	};
928	931
929		−static const struct ipmmu_features ipmmu_features_r8a779a0 = {
	932	+static const struct ipmmu_features ipmmu_features_rcar_gen4 = {
930	933	.use_ns_alias_offset = false,
931	934	.has_cache_leaf_nodes = true,
932	935	.number_of_contexts = 16,
@@ -982,7 +985,10 @@ static const struct of_device_id ipmmu_of_ids[] = { · 49 unmodified lines
982	985	.data = &ipmmu_features_rcar_gen3,
983	986	}, {
984	987	.compatible = "renesas,ipmmu-r8a779a0",
985		− .data = &ipmmu_features_r8a779a0,
	988	+ .data = &ipmmu_features_rcar_gen4,
	989	+ }, {
	990	+ .compatible = "renesas,rcar-gen4-ipmmu",
	991	+ .data = &ipmmu_features_rcar_gen4,
986	992	}, {
987	993	/* Terminator */
988	994	},
@@ -1006,7 +1012,9 @@ static int ipmmu_probe(struct platform_device *pdev) · 17 unmodified lines
1006	1012	bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);
1007	1013	mmu->features = of_device_get_match_data(&pdev->dev);
1008	1014	memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);
1009		− dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
	1015	+ ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
	1016	+ if (ret)
	1017	+ return ret;
1010	1018
1011	1019	/* Map I/O memory and request IRQ. */
1012	1020	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);

drivers/iommu/msm_iommu.c+27−47

@@ -558,11 +558,6 @@ fail: · 557 unmodified lines
558	558	return ret;
559	559	}
560	560
561		−static bool msm_iommu_capable(enum iommu_cap cap)
562		−{
563		− return false;
564		−}
565		−
566	561	static void print_ctx_regs(void __iomem *base, int ctx)
567	562	{
568	563	unsigned int fsr = GET_FSR(base, ctx);
@@ -672,27 +667,28 @@ fail: · 103 unmodified lines
672	667	}
673	668
674	669	static struct iommu_ops msm_iommu_ops = {
675		− .capable = msm_iommu_capable,
676	670	.domain_alloc = msm_iommu_domain_alloc,
677		− .domain_free = msm_iommu_domain_free,
678		− .attach_dev = msm_iommu_attach_dev,
679		− .detach_dev = msm_iommu_detach_dev,
680		− .map = msm_iommu_map,
681		− .unmap = msm_iommu_unmap,
682		− /*
683		− * Nothing is needed here, the barrier to guarantee
684		− * completion of the tlb sync operation is implicitly
685		− * taken care when the iommu client does a writel before
686		− * kick starting the other master.
687		− */
688		− .iotlb_sync = NULL,
689		− .iotlb_sync_map = msm_iommu_sync_map,
690		− .iova_to_phys = msm_iommu_iova_to_phys,
691	671	.probe_device = msm_iommu_probe_device,
692	672	.release_device = msm_iommu_release_device,
693	673	.device_group = generic_device_group,
694	674	.pgsize_bitmap = MSM_IOMMU_PGSIZES,
695	675	.of_xlate = qcom_iommu_of_xlate,
	676	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	677	+ .attach_dev = msm_iommu_attach_dev,
	678	+ .detach_dev = msm_iommu_detach_dev,
	679	+ .map = msm_iommu_map,
	680	+ .unmap = msm_iommu_unmap,
	681	+ /*
	682	+ * Nothing is needed here, the barrier to guarantee
	683	+ * completion of the tlb sync operation is implicitly
	684	+ * taken care when the iommu client does a writel before
	685	+ * kick starting the other master.
	686	+ */
	687	+ .iotlb_sync = NULL,
	688	+ .iotlb_sync_map = msm_iommu_sync_map,
	689	+ .iova_to_phys = msm_iommu_iova_to_phys,
	690	+ .free = msm_iommu_domain_free,
	691	+ }
696	692	};
697	693
698	694	static int msm_iommu_probe(struct platform_device *pdev)
@@ -710,36 +706,32 @@ static int msm_iommu_probe(struct platform_device *pdev) · 11 unmodified lines
710	706	INIT_LIST_HEAD(&iommu->ctx_list);
711	707
712	708	iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk");
713		− if (IS_ERR(iommu->pclk)) {
714		− dev_err(iommu->dev, "could not get smmu_pclk\n");
715		− return PTR_ERR(iommu->pclk);
716		− }
	709	+ if (IS_ERR(iommu->pclk))
	710	+ return dev_err_probe(iommu->dev, PTR_ERR(iommu->pclk),
	711	+ "could not get smmu_pclk\n");
717	712
718	713	ret = clk_prepare(iommu->pclk);
719		− if (ret) {
720		− dev_err(iommu->dev, "could not prepare smmu_pclk\n");
721		− return ret;
722		− }
	714	+ if (ret)
	715	+ return dev_err_probe(iommu->dev, ret,
	716	+ "could not prepare smmu_pclk\n");
723	717
724	718	iommu->clk = devm_clk_get(iommu->dev, "iommu_clk");
725	719	if (IS_ERR(iommu->clk)) {
726		− dev_err(iommu->dev, "could not get iommu_clk\n");
727	720	clk_unprepare(iommu->pclk);
728		− return PTR_ERR(iommu->clk);
	721	+ return dev_err_probe(iommu->dev, PTR_ERR(iommu->clk),
	722	+ "could not get iommu_clk\n");
729	723	}
730	724
731	725	ret = clk_prepare(iommu->clk);
732	726	if (ret) {
733		− dev_err(iommu->dev, "could not prepare iommu_clk\n");
734	727	clk_unprepare(iommu->pclk);
735		− return ret;
	728	+ return dev_err_probe(iommu->dev, ret, "could not prepare iommu_clk\n");
736	729	}
737	730
738	731	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
739	732	iommu->base = devm_ioremap_resource(iommu->dev, r);
740	733	if (IS_ERR(iommu->base)) {
741		− dev_err(iommu->dev, "could not get iommu base\n");
742		− ret = PTR_ERR(iommu->base);
	734	+ ret = dev_err_probe(iommu->dev, PTR_ERR(iommu->base), "could not get iommu base\n");
743	735	goto fail;
744	736	}
745	737	ioaddr = r->start;
@@ -831,16 +823,4 @@ static struct platform_driver msm_iommu_driver = { · 85 unmodified lines
831	823	.probe = msm_iommu_probe,
832	824	.remove = msm_iommu_remove,
833	825	};
834		−
835		−static int __init msm_iommu_driver_init(void)
836		−{
837		− int ret;
838		−
839		− ret = platform_driver_register(&msm_iommu_driver);
840		− if (ret != 0)
841		− pr_err("Failed to register IOMMU driver\n");
842		−
843		− return ret;
844		−}
845		−subsys_initcall(msm_iommu_driver_init);
846		−
	826	+builtin_platform_driver(msm_iommu_driver);

drivers/iommu/mtk_iommu.c+33−29

@@ -210,33 +210,27 @@ static struct mtk_iommu_domain to_mtk_domain(struct iommu_domain dom) · 209 unmodified lines
210	210
211	211	static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data)
212	212	{
213		− for_each_m4u(data) {
214		− if (pm_runtime_get_if_in_use(data->dev) <= 0)
215		− continue;
216		−
217		− writel_relaxed(F_INVLD_EN1 \| F_INVLD_EN0,
218		− data->base + data->plat_data->inv_sel_reg);
219		− writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
220		− wmb(); /* Make sure the tlb flush all done */
	213	+ unsigned long flags;
221	214
222		− pm_runtime_put(data->dev);
223		− }
	215	+ spin_lock_irqsave(&data->tlb_lock, flags);
	216	+ writel_relaxed(F_INVLD_EN1 \| F_INVLD_EN0,
	217	+ data->base + data->plat_data->inv_sel_reg);
	218	+ writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
	219	+ wmb(); /* Make sure the tlb flush all done */
	220	+ spin_unlock_irqrestore(&data->tlb_lock, flags);
224	221	}
225	222
226	223	static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size,
227	224	size_t granule,
228	225	struct mtk_iommu_data *data)
229	226	{
230		− bool has_pm = !!data->dev->pm_domain;
231	227	unsigned long flags;
232	228	int ret;
233	229	u32 tmp;
234	230
235	231	for_each_m4u(data) {
236		− if (has_pm) {
237		− if (pm_runtime_get_if_in_use(data->dev) <= 0)
238		− continue;
239		− }
	232	+ if (pm_runtime_get_if_in_use(data->dev) <= 0)
	233	+ continue;
240	234
241	235	spin_lock_irqsave(&data->tlb_lock, flags);
242	236	writel_relaxed(F_INVLD_EN1 \| F_INVLD_EN0,
@@ -252,17 +246,18 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, · 9 unmodified lines
252	246	/* tlb sync */
253	247	ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
254	248	tmp, tmp != 0, 10, 1000);
	249	+
	250	+ /* Clear the CPE status */
	251	+ writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
	252	+ spin_unlock_irqrestore(&data->tlb_lock, flags);
	253	+
255	254	if (ret) {
256	255	dev_warn(data->dev,
257	256	"Partial TLB flush timed out, falling back to full flush\n");
258	257	mtk_iommu_tlb_flush_all(data);
259	258	}
260		− /* Clear the CPE status */
261		− writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
262		− spin_unlock_irqrestore(&data->tlb_lock, flags);
263	259
264		− if (has_pm)
265		− pm_runtime_put(data->dev);
	260	+ pm_runtime_put(data->dev);
266	261	}
267	262	}
268	263
@@ -688,15 +683,6 @@ static void mtk_iommu_get_resv_regions(struct device *dev, · 419 unmodified lines
688	683
689	684	static const struct iommu_ops mtk_iommu_ops = {
690	685	.domain_alloc = mtk_iommu_domain_alloc,
691		− .domain_free = mtk_iommu_domain_free,
692		− .attach_dev = mtk_iommu_attach_device,
693		− .detach_dev = mtk_iommu_detach_device,
694		− .map = mtk_iommu_map,
695		− .unmap = mtk_iommu_unmap,
696		− .flush_iotlb_all = mtk_iommu_flush_iotlb_all,
697		− .iotlb_sync = mtk_iommu_iotlb_sync,
698		− .iotlb_sync_map = mtk_iommu_sync_map,
699		− .iova_to_phys = mtk_iommu_iova_to_phys,
700	686	.probe_device = mtk_iommu_probe_device,
701	687	.release_device = mtk_iommu_release_device,
702	688	.device_group = mtk_iommu_device_group,
@@ -705,6 +691,17 @@ static const struct iommu_ops mtk_iommu_ops = { · 2 unmodified lines
705	691	.put_resv_regions = generic_iommu_put_resv_regions,
706	692	.pgsize_bitmap = SZ_4K \| SZ_64K \| SZ_1M \| SZ_16M,
707	693	.owner = THIS_MODULE,
	694	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	695	+ .attach_dev = mtk_iommu_attach_device,
	696	+ .detach_dev = mtk_iommu_detach_device,
	697	+ .map = mtk_iommu_map,
	698	+ .unmap = mtk_iommu_unmap,
	699	+ .flush_iotlb_all = mtk_iommu_flush_iotlb_all,
	700	+ .iotlb_sync = mtk_iommu_iotlb_sync,
	701	+ .iotlb_sync_map = mtk_iommu_sync_map,
	702	+ .iova_to_phys = mtk_iommu_iova_to_phys,
	703	+ .free = mtk_iommu_domain_free,
	704	+ }
708	705	};
709	706
710	707	static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
@@ -1014,6 +1011,13 @@ static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) · 303 unmodified lines
1014	1011	writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
1015	1012	writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);
1016	1013	writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR);
	1014	+
	1015	+ /*
	1016	+ * Users may allocate dma buffer before they call pm_runtime_get,
	1017	+ * in which case it will lack the necessary tlb flush.
	1018	+ * Thus, make sure to update the tlb after each PM resume.
	1019	+ */
	1020	+ mtk_iommu_tlb_flush_all(data);
1017	1021	return 0;
1018	1022	}
1019	1023

drivers/iommu/mtk_iommu_v1.c+8−6

@@ -550,12 +550,6 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) · 549 unmodified lines
550	550
551	551	static const struct iommu_ops mtk_iommu_ops = {
552	552	.domain_alloc = mtk_iommu_domain_alloc,
553		− .domain_free = mtk_iommu_domain_free,
554		− .attach_dev = mtk_iommu_attach_device,
555		− .detach_dev = mtk_iommu_detach_device,
556		− .map = mtk_iommu_map,
557		− .unmap = mtk_iommu_unmap,
558		− .iova_to_phys = mtk_iommu_iova_to_phys,
559	553	.probe_device = mtk_iommu_probe_device,
560	554	.probe_finalize = mtk_iommu_probe_finalize,
561	555	.release_device = mtk_iommu_release_device,
@@ -563,6 +557,14 @@ static const struct iommu_ops mtk_iommu_ops = { · 1 unmodified line
563	557	.device_group = generic_device_group,
564	558	.pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT,
565	559	.owner = THIS_MODULE,
	560	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	561	+ .attach_dev = mtk_iommu_attach_device,
	562	+ .detach_dev = mtk_iommu_detach_device,
	563	+ .map = mtk_iommu_map,
	564	+ .unmap = mtk_iommu_unmap,
	565	+ .iova_to_phys = mtk_iommu_iova_to_phys,
	566	+ .free = mtk_iommu_domain_free,
	567	+ }
566	568	};
567	569
568	570	static const struct of_device_id mtk_iommu_of_ids[] = {

drivers/iommu/omap-iommu.c+8−6

@@ -1734,16 +1734,18 @@ static struct iommu_group omap_iommu_device_group(struct device dev) · 1733 unmodified lines
1734	1734
1735	1735	static const struct iommu_ops omap_iommu_ops = {
1736	1736	.domain_alloc = omap_iommu_domain_alloc,
1737		− .domain_free = omap_iommu_domain_free,
1738		− .attach_dev = omap_iommu_attach_dev,
1739		− .detach_dev = omap_iommu_detach_dev,
1740		− .map = omap_iommu_map,
1741		− .unmap = omap_iommu_unmap,
1742		− .iova_to_phys = omap_iommu_iova_to_phys,
1743	1737	.probe_device = omap_iommu_probe_device,
1744	1738	.release_device = omap_iommu_release_device,
1745	1739	.device_group = omap_iommu_device_group,
1746	1740	.pgsize_bitmap = OMAP_IOMMU_PGSIZES,
	1741	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	1742	+ .attach_dev = omap_iommu_attach_dev,
	1743	+ .detach_dev = omap_iommu_detach_dev,
	1744	+ .map = omap_iommu_map,
	1745	+ .unmap = omap_iommu_unmap,
	1746	+ .iova_to_phys = omap_iommu_iova_to_phys,
	1747	+ .free = omap_iommu_domain_free,
	1748	+ }
1747	1749	};
1748	1750
1749	1751	static int __init omap_iommu_init(void)

drivers/iommu/rockchip-iommu.c+9−12

@@ -1187,17 +1187,19 @@ static int rk_iommu_of_xlate(struct device *dev, · 1186 unmodified lines
1187	1187
1188	1188	static const struct iommu_ops rk_iommu_ops = {
1189	1189	.domain_alloc = rk_iommu_domain_alloc,
1190		− .domain_free = rk_iommu_domain_free,
1191		− .attach_dev = rk_iommu_attach_device,
1192		− .detach_dev = rk_iommu_detach_device,
1193		− .map = rk_iommu_map,
1194		− .unmap = rk_iommu_unmap,
1195	1190	.probe_device = rk_iommu_probe_device,
1196	1191	.release_device = rk_iommu_release_device,
1197		− .iova_to_phys = rk_iommu_iova_to_phys,
1198	1192	.device_group = rk_iommu_device_group,
1199	1193	.pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,
1200	1194	.of_xlate = rk_iommu_of_xlate,
	1195	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	1196	+ .attach_dev = rk_iommu_attach_device,
	1197	+ .detach_dev = rk_iommu_detach_device,
	1198	+ .map = rk_iommu_map,
	1199	+ .unmap = rk_iommu_unmap,
	1200	+ .iova_to_phys = rk_iommu_iova_to_phys,
	1201	+ .free = rk_iommu_domain_free,
	1202	+ }
1201	1203	};
1202	1204
1203	1205	static int rk_iommu_probe(struct platform_device *pdev)
@@ -1407,9 +1409,4 @@ static struct platform_driver rk_iommu_driver = { · 203 unmodified lines
1407	1409	.suppress_bind_attrs = true,
1408	1410	},
1409	1411	};
1410		−
1411		−static int __init rk_iommu_init(void)
1412		−{
1413		− return platform_driver_register(&rk_iommu_driver);
1414		−}
1415		−subsys_initcall(rk_iommu_init);
	1412	+builtin_platform_driver(rk_iommu_driver);

drivers/iommu/s390-iommu.c+8−6

@@ -363,16 +363,18 @@ void zpci_destroy_iommu(struct zpci_dev *zdev) · 362 unmodified lines
363	363	static const struct iommu_ops s390_iommu_ops = {
364	364	.capable = s390_iommu_capable,
365	365	.domain_alloc = s390_domain_alloc,
366		− .domain_free = s390_domain_free,
367		− .attach_dev = s390_iommu_attach_device,
368		− .detach_dev = s390_iommu_detach_device,
369		− .map = s390_iommu_map,
370		− .unmap = s390_iommu_unmap,
371		− .iova_to_phys = s390_iommu_iova_to_phys,
372	366	.probe_device = s390_iommu_probe_device,
373	367	.release_device = s390_iommu_release_device,
374	368	.device_group = generic_device_group,
375	369	.pgsize_bitmap = S390_IOMMU_PGSIZES,
	370	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	371	+ .attach_dev = s390_iommu_attach_device,
	372	+ .detach_dev = s390_iommu_detach_device,
	373	+ .map = s390_iommu_map,
	374	+ .unmap = s390_iommu_unmap,
	375	+ .iova_to_phys = s390_iommu_iova_to_phys,
	376	+ .free = s390_domain_free,
	377	+ }
376	378	};
377	379
378	380	static int __init s390_iommu_init(void)

drivers/iommu/sprd-iommu.c+10−8

@@ -416,20 +416,22 @@ static int sprd_iommu_of_xlate(struct device dev, struct of_phandle_args args) · 415 unmodified lines
416	416
417	417	static const struct iommu_ops sprd_iommu_ops = {
418	418	.domain_alloc = sprd_iommu_domain_alloc,
419		− .domain_free = sprd_iommu_domain_free,
420		− .attach_dev = sprd_iommu_attach_device,
421		− .detach_dev = sprd_iommu_detach_device,
422		− .map = sprd_iommu_map,
423		− .unmap = sprd_iommu_unmap,
424		− .iotlb_sync_map = sprd_iommu_sync_map,
425		− .iotlb_sync = sprd_iommu_sync,
426		− .iova_to_phys = sprd_iommu_iova_to_phys,
427	419	.probe_device = sprd_iommu_probe_device,
428	420	.release_device = sprd_iommu_release_device,
429	421	.device_group = sprd_iommu_device_group,
430	422	.of_xlate = sprd_iommu_of_xlate,
431	423	.pgsize_bitmap = ~0UL << SPRD_IOMMU_PAGE_SHIFT,
432	424	.owner = THIS_MODULE,
	425	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	426	+ .attach_dev = sprd_iommu_attach_device,
	427	+ .detach_dev = sprd_iommu_detach_device,
	428	+ .map = sprd_iommu_map,
	429	+ .unmap = sprd_iommu_unmap,
	430	+ .iotlb_sync_map = sprd_iommu_sync_map,
	431	+ .iotlb_sync = sprd_iommu_sync,
	432	+ .iova_to_phys = sprd_iommu_iova_to_phys,
	433	+ .free = sprd_iommu_domain_free,
	434	+ }
433	435	};
434	436
435	437	static const struct of_device_id sprd_iommu_of_match[] = {

drivers/iommu/sun50i-iommu.c+10−8

@@ -760,19 +760,21 @@ static int sun50i_iommu_of_xlate(struct device *dev, · 759 unmodified lines
760	760
761	761	static const struct iommu_ops sun50i_iommu_ops = {
762	762	.pgsize_bitmap = SZ_4K,
763		− .attach_dev = sun50i_iommu_attach_device,
764		− .detach_dev = sun50i_iommu_detach_device,
765	763	.device_group = sun50i_iommu_device_group,
766	764	.domain_alloc = sun50i_iommu_domain_alloc,
767		− .domain_free = sun50i_iommu_domain_free,
768		− .flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
769		− .iotlb_sync = sun50i_iommu_iotlb_sync,
770		− .iova_to_phys = sun50i_iommu_iova_to_phys,
771		− .map = sun50i_iommu_map,
772	765	.of_xlate = sun50i_iommu_of_xlate,
773	766	.probe_device = sun50i_iommu_probe_device,
774	767	.release_device = sun50i_iommu_release_device,
775		− .unmap = sun50i_iommu_unmap,
	768	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	769	+ .attach_dev = sun50i_iommu_attach_device,
	770	+ .detach_dev = sun50i_iommu_detach_device,
	771	+ .flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
	772	+ .iotlb_sync = sun50i_iommu_iotlb_sync,
	773	+ .iova_to_phys = sun50i_iommu_iova_to_phys,
	774	+ .map = sun50i_iommu_map,
	775	+ .unmap = sun50i_iommu_unmap,
	776	+ .free = sun50i_iommu_domain_free,
	777	+ }
776	778	};
777	779
778	780	static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,

drivers/iommu/tegra-gart.c+10−14

@@ -238,11 +238,6 @@ static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain, · 237 unmodified lines
238	238	return pte & GART_PAGE_MASK;
239	239	}
240	240
241		−static bool gart_iommu_capable(enum iommu_cap cap)
242		−{
243		− return false;
244		−}
245		−
246	241	static struct iommu_device gart_iommu_probe_device(struct device dev)
247	242	{
248	243	if (!dev_iommu_fwspec_get(dev))
@@ -276,21 +271,22 @@ static void gart_iommu_sync(struct iommu_domain *domain, · 27 unmodified lines
276	271	}
277	272
278	273	static const struct iommu_ops gart_iommu_ops = {
279		− .capable = gart_iommu_capable,
280	274	.domain_alloc = gart_iommu_domain_alloc,
281		− .domain_free = gart_iommu_domain_free,
282		− .attach_dev = gart_iommu_attach_dev,
283		− .detach_dev = gart_iommu_detach_dev,
284	275	.probe_device = gart_iommu_probe_device,
285	276	.release_device = gart_iommu_release_device,
286	277	.device_group = generic_device_group,
287		− .map = gart_iommu_map,
288		− .unmap = gart_iommu_unmap,
289		− .iova_to_phys = gart_iommu_iova_to_phys,
290	278	.pgsize_bitmap = GART_IOMMU_PGSIZES,
291	279	.of_xlate = gart_iommu_of_xlate,
292		− .iotlb_sync_map = gart_iommu_sync_map,
293		− .iotlb_sync = gart_iommu_sync,
	280	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	281	+ .attach_dev = gart_iommu_attach_dev,
	282	+ .detach_dev = gart_iommu_detach_dev,
	283	+ .map = gart_iommu_map,
	284	+ .unmap = gart_iommu_unmap,
	285	+ .iova_to_phys = gart_iommu_iova_to_phys,
	286	+ .iotlb_sync_map = gart_iommu_sync_map,
	287	+ .iotlb_sync = gart_iommu_sync,
	288	+ .free = gart_iommu_domain_free,
	289	+ }
294	290	};
295	291
296	292	int tegra_gart_suspend(struct gart_device *gart)

drivers/iommu/tegra-smmu.c+8−12

@@ -272,11 +272,6 @@ static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id) · 271 unmodified lines
272	272	clear_bit(id, smmu->asids);
273	273	}
274	274
275		−static bool tegra_smmu_capable(enum iommu_cap cap)
276		−{
277		− return false;
278		−}
279		−
280	275	static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
281	276	{
282	277	struct tegra_smmu_as *as;
@@ -969,19 +964,20 @@ static int tegra_smmu_of_xlate(struct device *dev, · 686 unmodified lines
969	964	}
970	965
971	966	static const struct iommu_ops tegra_smmu_ops = {
972		− .capable = tegra_smmu_capable,
973	967	.domain_alloc = tegra_smmu_domain_alloc,
974		− .domain_free = tegra_smmu_domain_free,
975		− .attach_dev = tegra_smmu_attach_dev,
976		− .detach_dev = tegra_smmu_detach_dev,
977	968	.probe_device = tegra_smmu_probe_device,
978	969	.release_device = tegra_smmu_release_device,
979	970	.device_group = tegra_smmu_device_group,
980		− .map = tegra_smmu_map,
981		− .unmap = tegra_smmu_unmap,
982		− .iova_to_phys = tegra_smmu_iova_to_phys,
983	971	.of_xlate = tegra_smmu_of_xlate,
984	972	.pgsize_bitmap = SZ_4K,
	973	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	974	+ .attach_dev = tegra_smmu_attach_dev,
	975	+ .detach_dev = tegra_smmu_detach_dev,
	976	+ .map = tegra_smmu_map,
	977	+ .unmap = tegra_smmu_unmap,
	978	+ .iova_to_phys = tegra_smmu_iova_to_phys,
	979	+ .free = tegra_smmu_domain_free,
	980	+ }
985	981	};
986	982
987	983	static void tegra_smmu_ahb_enable(void)

drivers/iommu/virtio-iommu.c+8−6

@@ -1008,12 +1008,6 @@ static int viommu_of_xlate(struct device dev, struct of_phandle_args args) · 1007 unmodified lines
1008	1008
1009	1009	static struct iommu_ops viommu_ops = {
1010	1010	.domain_alloc = viommu_domain_alloc,
1011		− .domain_free = viommu_domain_free,
1012		− .attach_dev = viommu_attach_dev,
1013		− .map = viommu_map,
1014		− .unmap = viommu_unmap,
1015		− .iova_to_phys = viommu_iova_to_phys,
1016		− .iotlb_sync = viommu_iotlb_sync,
1017	1011	.probe_device = viommu_probe_device,
1018	1012	.probe_finalize = viommu_probe_finalize,
1019	1013	.release_device = viommu_release_device,
@@ -1022,6 +1016,14 @@ static struct iommu_ops viommu_ops = { · 2 unmodified lines
1022	1016	.put_resv_regions = generic_iommu_put_resv_regions,
1023	1017	.of_xlate = viommu_of_xlate,
1024	1018	.owner = THIS_MODULE,
	1019	+ .default_domain_ops = &(const struct iommu_domain_ops) {
	1020	+ .attach_dev = viommu_attach_dev,
	1021	+ .map = viommu_map,
	1022	+ .unmap = viommu_unmap,
	1023	+ .iova_to_phys = viommu_iova_to_phys,
	1024	+ .iotlb_sync = viommu_iotlb_sync,
	1025	+ .free = viommu_domain_free,
	1026	+ }
1025	1027	};
1026	1028
1027	1029	static int viommu_init_vqs(struct viommu_dev *viommu)

drivers/perf/arm_smmuv3_pmu.c+3−1

@@ -654,6 +654,7 @@ static int smmu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 653 unmodified lines
654	654	static irqreturn_t smmu_pmu_handle_irq(int irq_num, void *data)
655	655	{
656	656	struct smmu_pmu *smmu_pmu = data;
	657	+ DECLARE_BITMAP(ovs, BITS_PER_TYPE(u64));
657	658	u64 ovsr;
658	659	unsigned int idx;
659	660
@@ -663,7 +664,8 @@ static irqreturn_t smmu_pmu_handle_irq(int irq_num, void *data) · 3 unmodified lines
663	664
664	665	writeq(ovsr, smmu_pmu->reloc_base + SMMU_PMCG_OVSCLR0);
665	666
666		− for_each_set_bit(idx, (unsigned long *)&ovsr, smmu_pmu->num_counters) {
	667	+ bitmap_from_u64(ovs, ovsr);
	668	+ for_each_set_bit(idx, ovs, smmu_pmu->num_counters) {
667	669	struct perf_event *event = smmu_pmu->events[idx];
668	670	struct hw_perf_event *hwc;
669	671

drivers/vdpa/vdpa_user/iova_domain.c+11−0

@@ -480,6 +480,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size) · 479 unmodified lines
480	480	struct file *file;
481	481	struct vduse_bounce_map *map;
482	482	unsigned long pfn, bounce_pfns;
	483	+ int ret;
483	484
484	485	bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
485	486	if (iova_limit <= bounce_size)
@@ -513,10 +514,20 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size) · 27 unmodified lines
513	514	spin_lock_init(&domain->iotlb_lock);
514	515	init_iova_domain(&domain->stream_iovad,
515	516	PAGE_SIZE, IOVA_START_PFN);
	517	+ ret = iova_domain_init_rcaches(&domain->stream_iovad);
	518	+ if (ret)
	519	+ goto err_iovad_stream;
516	520	init_iova_domain(&domain->consistent_iovad,
517	521	PAGE_SIZE, bounce_pfns);
	522	+ ret = iova_domain_init_rcaches(&domain->consistent_iovad);
	523	+ if (ret)
	524	+ goto err_iovad_consistent;
518	525
519	526	return domain;
	527	+err_iovad_consistent:
	528	+ put_iova_domain(&domain->stream_iovad);
	529	+err_iovad_stream:
	530	+ fput(file);
520	531	err_file:
521	532	vfree(domain->bounce_maps);
522	533	err_map:

include/linux/intel-iommu.h+2−31

@@ -525,12 +525,6 @@ struct context_entry { · 524 unmodified lines
525	525	*/
526	526	#define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(1)
527	527
528		−/*
529		− * Domain represents a virtual machine which demands iommu nested
530		− * translation mode support.
531		− */
532		−#define DOMAIN_FLAG_NESTING_MODE BIT(2)
533		−
534	528	struct dmar_domain {
535	529	int nid; /* node id */
536	530
@@ -548,7 +542,6 @@ struct dmar_domain { · 11 unmodified lines
548	542	u8 iommu_snooping: 1; /* indicate snooping control feature */
549	543
550	544	struct list_head devices; /* all devices' list */
551		− struct list_head subdevices; /* all subdevices' list */
552	545	struct iova_domain iovad; /* iova's that belong to this domain */
553	546
554	547	struct dma_pte pgd; / virtual address */
@@ -563,11 +556,6 @@ struct dmar_domain { · 8 unmodified lines
563	556	2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
564	557	u64 max_addr; /* maximum mapped address */
565	558
566		− u32 default_pasid; /*
567		− * The default pasid used for non-SVM
568		− * traffic on mediated devices.
569		− */
570		−
571	559	struct iommu_domain domain; /* generic domain data structure for
572	560	iommu core */
573	561	};
@@ -590,7 +578,6 @@ struct intel_iommu { · 16 unmodified lines
590	578
591	579	#ifdef CONFIG_INTEL_IOMMU
592	580	unsigned long domain_ids; / bitmap of domains */
593		− struct dmar_domain **domains; / ptr to domains */
594	581	spinlock_t lock; /* protect context, domain ids */
595	582	struct root_entry root_entry; / virtual address */
596	583
@@ -620,21 +607,11 @@ struct intel_iommu { · 23 unmodified lines
620	607	void *perf_statistic;
621	608	};
622	609
623		−/* Per subdevice private data */
624		−struct subdev_domain_info {
625		− struct list_head link_phys; /* link to phys device siblings */
626		− struct list_head link_domain; /* link to domain siblings */
627		− struct device pdev; / physical device derived from */
628		− struct dmar_domain domain; / aux-domain */
629		− int users; /* user count */
630		−};
631		−
632	610	/* PCI domain-device relationship */
633	611	struct device_domain_info {
634	612	struct list_head link; /* link to domain siblings */
635	613	struct list_head global; /* link to global list */
636	614	struct list_head table; /* link to pasid table */
637		− struct list_head subdevices; /* subdevices sibling */
638	615	u32 segment; /* PCI segment number */
639	616	u8 bus; /* PCI bus number */
640	617	u8 devfn; /* PCI devfn number */
@@ -645,7 +622,6 @@ struct device_domain_info { · 4 unmodified lines
645	622	u8 pri_enabled:1;
646	623	u8 ats_supported:1;
647	624	u8 ats_enabled:1;
648		− u8 auxd_enabled:1; /* Multiple domains per device */
649	625	u8 ats_qdep;
650	626	struct device dev; / it's NULL for PCIe-to-PCI bridge */
651	627	struct intel_iommu iommu; / IOMMU used by this device */
@@ -717,7 +693,6 @@ static inline int nr_pte_to_next_page(struct dma_pte *pte) · 65 unmodified lines
717	693	}
718	694
719	695	extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev);
720		−extern int dmar_find_matched_atsr_unit(struct pci_dev *dev);
721	696
722	697	extern int dmar_enable_qi(struct intel_iommu *iommu);
723	698	extern void dmar_disable_qi(struct intel_iommu *iommu);
@@ -757,17 +732,12 @@ int for_each_device_domain(int (fn)(struct device_domain_info info, · 33 unmodified lines
757	732	void data), void data);
758	733	void iommu_flush_write_buffer(struct intel_iommu *iommu);
759	734	int intel_iommu_enable_pasid(struct intel_iommu iommu, struct device dev);
760		−struct dmar_domain find_domain(struct device dev);
761		−struct device_domain_info get_domain_info(struct device dev);
762	735	struct intel_iommu device_to_iommu(struct device dev, u8 bus, u8 devfn);
763	736
764	737	#ifdef CONFIG_INTEL_IOMMU_SVM
765	738	extern void intel_svm_check(struct intel_iommu *iommu);
766	739	extern int intel_svm_enable_prq(struct intel_iommu *iommu);
767	740	extern int intel_svm_finish_prq(struct intel_iommu *iommu);
768		−int intel_svm_bind_gpasid(struct iommu_domain domain, struct device dev,
769		− struct iommu_gpasid_bind_data *data);
770		−int intel_svm_unbind_gpasid(struct device *dev, u32 pasid);
771	741	struct iommu_sva intel_svm_bind(struct device dev, struct mm_struct *mm,
772	742	void *drvdata);
773	743	void intel_svm_unbind(struct iommu_sva *handle);
@@ -795,7 +765,6 @@ struct intel_svm { · 21 unmodified lines
795	765
796	766	unsigned int flags;
797	767	u32 pasid;
798		− int gpasid; /* In case that guest PASID is different from host PASID */
799	768	struct list_head devs;
800	769	};
801	770	#else
@@ -813,6 +782,8 @@ bool context_present(struct context_entry *context); · 11 unmodified lines
813	782	struct context_entry iommu_context_addr(struct intel_iommu iommu, u8 bus,
814	783	u8 devfn, int alloc);
815	784
	785	+extern const struct iommu_ops intel_iommu_ops;
	786	+
816	787	#ifdef CONFIG_INTEL_IOMMU
817	788	extern int iommu_calculate_agaw(struct intel_iommu *iommu);
818	789	extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu);

include/linux/intel-svm.h+0−12

@@ -25,17 +25,5 @@ · 24 unmodified lines
25	25	* do such IOTLB flushes automatically.
26	26	*/
27	27	#define SVM_FLAG_SUPERVISOR_MODE BIT(0)
28		−/*
29		− * The SVM_FLAG_GUEST_MODE flag is used when a PASID bind is for guest
30		− * processes. Compared to the host bind, the primary differences are:
31		− * 1. mm life cycle management
32		− * 2. fault reporting
33		− */
34		−#define SVM_FLAG_GUEST_MODE BIT(1)
35		−/*
36		− * The SVM_FLAG_GUEST_PASID flag is used when a guest has its own PASID space,
37		− * which requires guest and host PASID translation at both directions.
38		− */
39		−#define SVM_FLAG_GUEST_PASID BIT(2)
40	28
41	29	#endif /* __INTEL_SVM_H__ */

include/linux/iommu.h+65−116

@@ -37,6 +37,7 @@ struct iommu_group; · 36 unmodified lines
37	37	struct bus_type;
38	38	struct device;
39	39	struct iommu_domain;
	40	+struct iommu_domain_ops;
40	41	struct notifier_block;
41	42	struct iommu_sva;
42	43	struct iommu_fault_event;
@@ -88,7 +89,7 @@ struct iommu_domain_geometry { · 45 unmodified lines
88	89
89	90	struct iommu_domain {
90	91	unsigned type;
91		− const struct iommu_ops *ops;
	92	+ const struct iommu_domain_ops *ops;
92	93	unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */
93	94	iommu_fault_handler_t handler;
94	95	void *handler_token;
@@ -144,7 +145,6 @@ struct iommu_resv_region { · 49 unmodified lines
144	145
145	146	/**
146	147	* enum iommu_dev_features - Per device IOMMU features
147		− * @IOMMU_DEV_FEAT_AUX: Auxiliary domain feature
148	148	* @IOMMU_DEV_FEAT_SVA: Shared Virtual Addresses
149	149	* @IOMMU_DEV_FEAT_IOPF: I/O Page Faults such as PRI or Stall. Generally
150	150	* enabling %IOMMU_DEV_FEAT_SVA requires
@@ -157,7 +157,6 @@ struct iommu_resv_region { · 6 unmodified lines
157	157	* iommu_dev_has_feature(), and enable it using iommu_dev_enable_feature().
158	158	*/
159	159	enum iommu_dev_features {
160		− IOMMU_DEV_FEAT_AUX,
161	160	IOMMU_DEV_FEAT_SVA,
162	161	IOMMU_DEV_FEAT_IOPF,
163	162	};
@@ -194,48 +193,28 @@ struct iommu_iotlb_gather { · 30 unmodified lines
194	193	* struct iommu_ops - iommu ops and capabilities
195	194	* @capable: check capability
196	195	* @domain_alloc: allocate iommu domain
197		− * @domain_free: free iommu domain
198		− * @attach_dev: attach device to an iommu domain
199		− * @detach_dev: detach device from an iommu domain
200		− * @map: map a physically contiguous memory region to an iommu domain
201		− * @map_pages: map a physically contiguous set of pages of the same size to
202		− * an iommu domain.
203		− * @unmap: unmap a physically contiguous memory region from an iommu domain
204		− * @unmap_pages: unmap a number of pages of the same size from an iommu domain
205		− * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
206		− * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
207		− * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
208		− * queue
209		− * @iova_to_phys: translate iova to physical address
210	196	* @probe_device: Add device to iommu driver handling
211	197	* @release_device: Remove device from iommu driver handling
212	198	* @probe_finalize: Do final setup work after the device is added to an IOMMU
213	199	* group and attached to the groups domain
214	200	* @device_group: find iommu group for a particular device
215		− * @enable_nesting: Enable nesting
216		− * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)
217	201	* @get_resv_regions: Request list of reserved regions for a device
218	202	* @put_resv_regions: Free list of reserved regions for a device
219		− * @apply_resv_region: Temporary helper call-back for iova reserved ranges
220	203	* @of_xlate: add OF master IDs to iommu grouping
221	204	* @is_attach_deferred: Check if domain attach should be deferred from iommu
222	205	* driver init to device driver init (default no)
223	206	* @dev_has/enable/disable_feat: per device entries to check/enable/disable
224	207	* iommu specific features.
225	208	* @dev_feat_enabled: check enabled feature
226		− * @aux_attach/detach_dev: aux-domain specific attach/detach entries.
227		− * @aux_get_pasid: get the pasid given an aux-domain
228	209	* @sva_bind: Bind process address space to device
229	210	* @sva_unbind: Unbind process address space from device
230	211	* @sva_get_pasid: Get PASID associated to a SVA handle
231	212	* @page_response: handle page request response
232		− * @cache_invalidate: invalidate translation caches
233		− * @sva_bind_gpasid: bind guest pasid and mm
234		− * @sva_unbind_gpasid: unbind guest pasid and mm
235	213	* @def_domain_type: device default domain type, return value:
236	214	* - IOMMU_DOMAIN_IDENTITY: must use an identity domain
237	215	* - IOMMU_DOMAIN_DMA: must use a dma domain
238	216	* - 0: use the default setting
	217	+ * @default_domain_ops: the default ops for domains
239	218	* @pgsize_bitmap: bitmap of all possible supported page sizes
240	219	* @owner: Driver module providing these ops
241	220	*/
@@ -244,43 +223,18 @@ struct iommu_ops { · 2 unmodified lines
244	223
245	224	/* Domain allocation and freeing by the iommu driver */
246	225	struct iommu_domain (domain_alloc)(unsigned iommu_domain_type);
247		− void (domain_free)(struct iommu_domain );
248	226
249		− int (attach_dev)(struct iommu_domain domain, struct device *dev);
250		− void (detach_dev)(struct iommu_domain domain, struct device *dev);
251		− int (map)(struct iommu_domain domain, unsigned long iova,
252		− phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
253		− int (map_pages)(struct iommu_domain domain, unsigned long iova,
254		− phys_addr_t paddr, size_t pgsize, size_t pgcount,
255		− int prot, gfp_t gfp, size_t *mapped);
256		− size_t (unmap)(struct iommu_domain domain, unsigned long iova,
257		− size_t size, struct iommu_iotlb_gather *iotlb_gather);
258		− size_t (unmap_pages)(struct iommu_domain domain, unsigned long iova,
259		− size_t pgsize, size_t pgcount,
260		− struct iommu_iotlb_gather *iotlb_gather);
261		− void (flush_iotlb_all)(struct iommu_domain domain);
262		− void (iotlb_sync_map)(struct iommu_domain domain, unsigned long iova,
263		− size_t size);
264		− void (iotlb_sync)(struct iommu_domain domain,
265		− struct iommu_iotlb_gather *iotlb_gather);
266		− phys_addr_t (iova_to_phys)(struct iommu_domain domain, dma_addr_t iova);
267	227	struct iommu_device (probe_device)(struct device *dev);
268	228	void (release_device)(struct device dev);
269	229	void (probe_finalize)(struct device dev);
270	230	struct iommu_group (device_group)(struct device *dev);
271		− int (enable_nesting)(struct iommu_domain domain);
272		− int (set_pgtable_quirks)(struct iommu_domain domain,
273		− unsigned long quirks);
274	231
275	232	/* Request/Free a list of reserved regions for a device */
276	233	void (get_resv_regions)(struct device dev, struct list_head *list);
277	234	void (put_resv_regions)(struct device dev, struct list_head *list);
278		− void (apply_resv_region)(struct device dev,
279		− struct iommu_domain *domain,
280		− struct iommu_resv_region *region);
281	235
282	236	int (of_xlate)(struct device dev, struct of_phandle_args *args);
283		− bool (is_attach_deferred)(struct iommu_domain domain, struct device *dev);
	237	+ bool (is_attach_deferred)(struct device dev);
284	238
285	239	/* Per device IOMMU features */
286	240	bool (dev_has_feat)(struct device dev, enum iommu_dev_features f);
@@ -288,11 +242,6 @@ struct iommu_ops { · 1 unmodified line
288	242	int (dev_enable_feat)(struct device dev, enum iommu_dev_features f);
289	243	int (dev_disable_feat)(struct device dev, enum iommu_dev_features f);
290	244
291		− /* Aux-domain specific attach/detach entries */
292		− int (aux_attach_dev)(struct iommu_domain domain, struct device *dev);
293		− void (aux_detach_dev)(struct iommu_domain domain, struct device *dev);
294		− int (aux_get_pasid)(struct iommu_domain domain, struct device *dev);
295		−
296	245	struct iommu_sva (sva_bind)(struct device dev, struct mm_struct mm,
297	246	void *drvdata);
298	247	void (sva_unbind)(struct iommu_sva handle);
@@ -301,19 +250,63 @@ struct iommu_ops { · 2 unmodified lines
301	250	int (page_response)(struct device dev,
302	251	struct iommu_fault_event *evt,
303	252	struct iommu_page_response *msg);
304		− int (cache_invalidate)(struct iommu_domain domain, struct device *dev,
305		− struct iommu_cache_invalidate_info *inv_info);
306		− int (sva_bind_gpasid)(struct iommu_domain domain,
307		− struct device dev, struct iommu_gpasid_bind_data data);
308		−
309		− int (sva_unbind_gpasid)(struct device dev, u32 pasid);
310	253
311	254	int (def_domain_type)(struct device dev);
312	255
	256	+ const struct iommu_domain_ops *default_domain_ops;
313	257	unsigned long pgsize_bitmap;
314	258	struct module *owner;
315	259	};
316	260
	261	+/**
	262	+ * struct iommu_domain_ops - domain specific operations
	263	+ * @attach_dev: attach an iommu domain to a device
	264	+ * @detach_dev: detach an iommu domain from a device
	265	+ * @map: map a physically contiguous memory region to an iommu domain
	266	+ * @map_pages: map a physically contiguous set of pages of the same size to
	267	+ * an iommu domain.
	268	+ * @unmap: unmap a physically contiguous memory region from an iommu domain
	269	+ * @unmap_pages: unmap a number of pages of the same size from an iommu domain
	270	+ * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
	271	+ * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
	272	+ * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
	273	+ * queue
	274	+ * @iova_to_phys: translate iova to physical address
	275	+ * @enable_nesting: Enable nesting
	276	+ * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)
	277	+ * @free: Release the domain after use.
	278	+ */
	279	+struct iommu_domain_ops {
	280	+ int (attach_dev)(struct iommu_domain domain, struct device *dev);
	281	+ void (detach_dev)(struct iommu_domain domain, struct device *dev);
	282	+
	283	+ int (map)(struct iommu_domain domain, unsigned long iova,
	284	+ phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
	285	+ int (map_pages)(struct iommu_domain domain, unsigned long iova,
	286	+ phys_addr_t paddr, size_t pgsize, size_t pgcount,
	287	+ int prot, gfp_t gfp, size_t *mapped);
	288	+ size_t (unmap)(struct iommu_domain domain, unsigned long iova,
	289	+ size_t size, struct iommu_iotlb_gather *iotlb_gather);
	290	+ size_t (unmap_pages)(struct iommu_domain domain, unsigned long iova,
	291	+ size_t pgsize, size_t pgcount,
	292	+ struct iommu_iotlb_gather *iotlb_gather);
	293	+
	294	+ void (flush_iotlb_all)(struct iommu_domain domain);
	295	+ void (iotlb_sync_map)(struct iommu_domain domain, unsigned long iova,
	296	+ size_t size);
	297	+ void (iotlb_sync)(struct iommu_domain domain,
	298	+ struct iommu_iotlb_gather *iotlb_gather);
	299	+
	300	+ phys_addr_t (iova_to_phys)(struct iommu_domain domain,
	301	+ dma_addr_t iova);
	302	+
	303	+ int (enable_nesting)(struct iommu_domain domain);
	304	+ int (set_pgtable_quirks)(struct iommu_domain domain,
	305	+ unsigned long quirks);
	306	+
	307	+ void (free)(struct iommu_domain domain);
	308	+};
	309	+
317	310	/**
318	311	* struct iommu_device - IOMMU core representation of one IOMMU hardware
319	312	* instance
@@ -403,6 +396,17 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) · 83 unmodified lines
403	396	};
404	397	}
405	398
	399	+static inline const struct iommu_ops dev_iommu_ops(struct device dev)
	400	+{
	401	+ /*
	402	+ * Assume that valid ops must be installed if iommu_probe_device()
	403	+ * has succeeded. The device ops are essentially for internal use
	404	+ * within the IOMMU subsystem itself, so we should be able to trust
	405	+ * ourselves not to misuse the helper.
	406	+ */
	407	+ return dev->iommu->iommu_dev->ops;
	408	+}
	409	+
406	410	#define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */
407	411	#define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */
408	412	#define IOMMU_GROUP_NOTIFY_BIND_DRIVER 3 /* Pre Driver bind */
@@ -421,14 +425,6 @@ extern int iommu_attach_device(struct iommu_domain *domain, · 12 unmodified lines
421	425	struct device *dev);
422	426	extern void iommu_detach_device(struct iommu_domain *domain,
423	427	struct device *dev);
424		−extern int iommu_uapi_cache_invalidate(struct iommu_domain *domain,
425		− struct device *dev,
426		− void __user *uinfo);
427		−
428		−extern int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain,
429		− struct device dev, void __user udata);
430		−extern int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain,
431		− struct device dev, void __user udata);
432	428	extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
433	429	struct device *dev, ioasid_t pasid);
434	430	extern struct iommu_domain iommu_get_domain_for_dev(struct device dev);
@@ -672,9 +668,6 @@ void iommu_release_device(struct device *dev); · 237 unmodified lines
672	668	int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
673	669	int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
674	670	bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f);
675		−int iommu_aux_attach_device(struct iommu_domain domain, struct device dev);
676		−void iommu_aux_detach_device(struct iommu_domain domain, struct device dev);
677		−int iommu_aux_get_pasid(struct iommu_domain domain, struct device dev);
678	671
679	672	struct iommu_sva iommu_sva_bind_device(struct device dev,
680	673	struct mm_struct *mm,
@@ -1019,23 +1012,6 @@ iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) · 338 unmodified lines
1019	1012	return -ENODEV;
1020	1013	}
1021	1014
1022		−static inline int
1023		−iommu_aux_attach_device(struct iommu_domain domain, struct device dev)
1024		−{
1025		− return -ENODEV;
1026		−}
1027		−
1028		−static inline void
1029		−iommu_aux_detach_device(struct iommu_domain domain, struct device dev)
1030		−{
1031		−}
1032		−
1033		−static inline int
1034		−iommu_aux_get_pasid(struct iommu_domain domain, struct device dev)
1035		−{
1036		− return -ENODEV;
1037		−}
1038		−
1039	1015	static inline struct iommu_sva *
1040	1016	iommu_sva_bind_device(struct device dev, struct mm_struct mm, void *drvdata)
1041	1017	{
@@ -1051,33 +1027,6 @@ static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle) · 9 unmodified lines
1051	1027	return IOMMU_PASID_INVALID;
1052	1028	}
1053	1029
1054		−static inline int
1055		−iommu_uapi_cache_invalidate(struct iommu_domain *domain,
1056		− struct device *dev,
1057		− struct iommu_cache_invalidate_info *inv_info)
1058		−{
1059		− return -ENODEV;
1060		−}
1061		−
1062		−static inline int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain,
1063		− struct device dev, void __user udata)
1064		−{
1065		− return -ENODEV;
1066		−}
1067		−
1068		−static inline int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain,
1069		− struct device dev, void __user udata)
1070		−{
1071		− return -ENODEV;
1072		−}
1073		−
1074		−static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
1075		− struct device *dev,
1076		− ioasid_t pasid)
1077		−{
1078		− return -ENODEV;
1079		−}
1080		−
1081	1030	static inline struct iommu_fwspec dev_iommu_fwspec_get(struct device dev)
1082	1031	{
1083	1032	return NULL;

include/linux/iova.h+3−12

@@ -21,18 +21,8 @@ struct iova { · 20 unmodified lines
21	21	unsigned long pfn_lo; /* Lowest allocated pfn */
22	22	};
23	23
24		−struct iova_magazine;
25		−struct iova_cpu_rcache;
26	24
27		−#define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */
28		−#define MAX_GLOBAL_MAGS 32 /* magazines per bin */
29		−
30		−struct iova_rcache {
31		− spinlock_t lock;
32		− unsigned long depot_size;
33		− struct iova_magazine *depot[MAX_GLOBAL_MAGS];
34		− struct iova_cpu_rcache __percpu *cpu_rcaches;
35		−};
	25	+struct iova_rcache;
36	26
37	27	/* holds all the iova translations for a domain */
38	28	struct iova_domain {
@@ -46,7 +36,7 @@ struct iova_domain { · 7 unmodified lines
46	36	unsigned long max32_alloc_size; /* Size of last failed allocation */
47	37	struct iova anchor; /* rbtree lookup anchor */
48	38
49		− struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
	39	+ struct iova_rcache *rcaches;
50	40	struct hlist_node cpuhp_dead;
51	41	};
52	42
@@ -102,6 +92,7 @@ struct iova reserve_iova(struct iova_domain iovad, unsigned long pfn_lo, · 49 unmodified lines
102	92	unsigned long pfn_hi);
103	93	void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
104	94	unsigned long start_pfn);
	95	+int iova_domain_init_rcaches(struct iova_domain *iovad);
105	96	struct iova find_iova(struct iova_domain iovad, unsigned long pfn);
106	97	void put_iova_domain(struct iova_domain *iovad);
107	98	#else

include/uapi/linux/iommu.h+0−181

@@ -158,185 +158,4 @@ struct iommu_page_response { · 157 unmodified lines
158	158	__u32 code;
159	159	};
160	160
161		−/* defines the granularity of the invalidation */
162		−enum iommu_inv_granularity {
163		− IOMMU_INV_GRANU_DOMAIN, /* domain-selective invalidation */
164		− IOMMU_INV_GRANU_PASID, /* PASID-selective invalidation */
165		− IOMMU_INV_GRANU_ADDR, /* page-selective invalidation */
166		− IOMMU_INV_GRANU_NR, /* number of invalidation granularities */
167		−};
168		−
169		−/**
170		− * struct iommu_inv_addr_info - Address Selective Invalidation Structure
171		− *
172		− * @flags: indicates the granularity of the address-selective invalidation
173		− * - If the PASID bit is set, the @pasid field is populated and the invalidation
174		− * relates to cache entries tagged with this PASID and matching the address
175		− * range.
176		− * - If ARCHID bit is set, @archid is populated and the invalidation relates
177		− * to cache entries tagged with this architecture specific ID and matching
178		− * the address range.
179		− * - Both PASID and ARCHID can be set as they may tag different caches.
180		− * - If neither PASID or ARCHID is set, global addr invalidation applies.
181		− * - The LEAF flag indicates whether only the leaf PTE caching needs to be
182		− * invalidated and other paging structure caches can be preserved.
183		− * @pasid: process address space ID
184		− * @archid: architecture-specific ID
185		− * @addr: first stage/level input address
186		− * @granule_size: page/block size of the mapping in bytes
187		− * @nb_granules: number of contiguous granules to be invalidated
188		− */
189		−struct iommu_inv_addr_info {
190		−#define IOMMU_INV_ADDR_FLAGS_PASID (1 << 0)
191		−#define IOMMU_INV_ADDR_FLAGS_ARCHID (1 << 1)
192		−#define IOMMU_INV_ADDR_FLAGS_LEAF (1 << 2)
193		− __u32 flags;
194		− __u32 archid;
195		− __u64 pasid;
196		− __u64 addr;
197		− __u64 granule_size;
198		− __u64 nb_granules;
199		−};
200		−
201		−/**
202		− * struct iommu_inv_pasid_info - PASID Selective Invalidation Structure
203		− *
204		− * @flags: indicates the granularity of the PASID-selective invalidation
205		− * - If the PASID bit is set, the @pasid field is populated and the invalidation
206		− * relates to cache entries tagged with this PASID and matching the address
207		− * range.
208		− * - If the ARCHID bit is set, the @archid is populated and the invalidation
209		− * relates to cache entries tagged with this architecture specific ID and
210		− * matching the address range.
211		− * - Both PASID and ARCHID can be set as they may tag different caches.
212		− * - At least one of PASID or ARCHID must be set.
213		− * @pasid: process address space ID
214		− * @archid: architecture-specific ID
215		− */
216		−struct iommu_inv_pasid_info {
217		−#define IOMMU_INV_PASID_FLAGS_PASID (1 << 0)
218		−#define IOMMU_INV_PASID_FLAGS_ARCHID (1 << 1)
219		− __u32 flags;
220		− __u32 archid;
221		− __u64 pasid;
222		−};
223		−
224		−/**
225		− * struct iommu_cache_invalidate_info - First level/stage invalidation
226		− * information
227		− * @argsz: User filled size of this data
228		− * @version: API version of this structure
229		− * @cache: bitfield that allows to select which caches to invalidate
230		− * @granularity: defines the lowest granularity used for the invalidation:
231		− * domain > PASID > addr
232		− * @padding: reserved for future use (should be zero)
233		− * @pasid_info: invalidation data when @granularity is %IOMMU_INV_GRANU_PASID
234		− * @addr_info: invalidation data when @granularity is %IOMMU_INV_GRANU_ADDR
235		− *
236		− * Not all the combinations of cache/granularity are valid:
237		− *
238		− * +--------------+---------------+---------------+---------------+
239		− * \| type / \| DEV_IOTLB \| IOTLB \| PASID \|
240		− * \| granularity \| \| \| cache \|
241		− * +==============+===============+===============+===============+
242		− * \| DOMAIN \| N/A \| Y \| Y \|
243		− * +--------------+---------------+---------------+---------------+
244		− * \| PASID \| Y \| Y \| Y \|
245		− * +--------------+---------------+---------------+---------------+
246		− * \| ADDR \| Y \| Y \| N/A \|
247		− * +--------------+---------------+---------------+---------------+
248		− *
249		− * Invalidations by %IOMMU_INV_GRANU_DOMAIN don't take any argument other than
250		− * @version and @cache.
251		− *
252		− * If multiple cache types are invalidated simultaneously, they all
253		− * must support the used granularity.
254		− */
255		−struct iommu_cache_invalidate_info {
256		− __u32 argsz;
257		−#define IOMMU_CACHE_INVALIDATE_INFO_VERSION_1 1
258		− __u32 version;
259		−/* IOMMU paging structure cache */
260		−#define IOMMU_CACHE_INV_TYPE_IOTLB (1 << 0) /* IOMMU IOTLB */
261		−#define IOMMU_CACHE_INV_TYPE_DEV_IOTLB (1 << 1) /* Device IOTLB */
262		−#define IOMMU_CACHE_INV_TYPE_PASID (1 << 2) /* PASID cache */
263		−#define IOMMU_CACHE_INV_TYPE_NR (3)
264		− __u8 cache;
265		− __u8 granularity;
266		− __u8 padding[6];
267		− union {
268		− struct iommu_inv_pasid_info pasid_info;
269		− struct iommu_inv_addr_info addr_info;
270		− } granu;
271		−};
272		−
273		−/**
274		− * struct iommu_gpasid_bind_data_vtd - Intel VT-d specific data on device and guest
275		− * SVA binding.
276		− *
277		− * @flags: VT-d PASID table entry attributes
278		− * @pat: Page attribute table data to compute effective memory type
279		− * @emt: Extended memory type
280		− *
281		− * Only guest vIOMMU selectable and effective options are passed down to
282		− * the host IOMMU.
283		− */
284		−struct iommu_gpasid_bind_data_vtd {
285		−#define IOMMU_SVA_VTD_GPASID_SRE (1 << 0) /* supervisor request */
286		−#define IOMMU_SVA_VTD_GPASID_EAFE (1 << 1) /* extended access enable */
287		−#define IOMMU_SVA_VTD_GPASID_PCD (1 << 2) /* page-level cache disable */
288		−#define IOMMU_SVA_VTD_GPASID_PWT (1 << 3) /* page-level write through */
289		−#define IOMMU_SVA_VTD_GPASID_EMTE (1 << 4) /* extended mem type enable */
290		−#define IOMMU_SVA_VTD_GPASID_CD (1 << 5) /* PASID-level cache disable */
291		−#define IOMMU_SVA_VTD_GPASID_WPE (1 << 6) /* Write protect enable */
292		−#define IOMMU_SVA_VTD_GPASID_LAST (1 << 7)
293		− __u64 flags;
294		− __u32 pat;
295		− __u32 emt;
296		−};
297		−
298		−#define IOMMU_SVA_VTD_GPASID_MTS_MASK (IOMMU_SVA_VTD_GPASID_CD \| \
299		− IOMMU_SVA_VTD_GPASID_EMTE \| \
300		− IOMMU_SVA_VTD_GPASID_PCD \| \
301		− IOMMU_SVA_VTD_GPASID_PWT)
302		−
303		−/**
304		− * struct iommu_gpasid_bind_data - Information about device and guest PASID binding
305		− * @argsz: User filled size of this data
306		− * @version: Version of this data structure
307		− * @format: PASID table entry format
308		− * @flags: Additional information on guest bind request
309		− * @gpgd: Guest page directory base of the guest mm to bind
310		− * @hpasid: Process address space ID used for the guest mm in host IOMMU
311		− * @gpasid: Process address space ID used for the guest mm in guest IOMMU
312		− * @addr_width: Guest virtual address width
313		− * @padding: Reserved for future use (should be zero)
314		− * @vtd: Intel VT-d specific data
315		− *
316		− * Guest to host PASID mapping can be an identity or non-identity, where guest
317		− * has its own PASID space. For non-identify mapping, guest to host PASID lookup
318		− * is needed when VM programs guest PASID into an assigned device. VMM may
319		− * trap such PASID programming then request host IOMMU driver to convert guest
320		− * PASID to host PASID based on this bind data.
321		− */
322		−struct iommu_gpasid_bind_data {
323		− __u32 argsz;
324		−#define IOMMU_GPASID_BIND_VERSION_1 1
325		− __u32 version;
326		−#define IOMMU_PASID_FORMAT_INTEL_VTD 1
327		−#define IOMMU_PASID_FORMAT_LAST 2
328		− __u32 format;
329		− __u32 addr_width;
330		−#define IOMMU_SVA_GPASID_VAL (1 << 0) /* guest PASID valid */
331		− __u64 flags;
332		− __u64 gpgd;
333		− __u64 hpasid;
334		− __u64 gpasid;
335		− __u8 padding[8];
336		− /* Vendor specific data */
337		− union {
338		− struct iommu_gpasid_bind_data_vtd vtd;
339		− } vendor;
340		−};
341		−
342	161	#endif /* _UAPI_IOMMU_H */

Merge tag 'iommu-updates-v5.18' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code