lael-labs/linux

Documentation/admin-guide/kernel-parameters.txt+9−0

@@ -2675,6 +2675,15 @@ Kernel parameters · 2674 unmodified lines
2675	1 - Bypass the IOMMU for DMA.	2675	1 - Bypass the IOMMU for DMA.
2676	unset - Use value of CONFIG_IOMMU_DEFAULT_PASSTHROUGH.	2676	unset - Use value of CONFIG_IOMMU_DEFAULT_PASSTHROUGH.
2677		2677
		2678	iommu.debug_pagealloc=
		2679	[KNL,EARLY] When CONFIG_IOMMU_DEBUG_PAGEALLOC is set, this
		2680	parameter enables the feature at boot time. By default, it
		2681	is disabled and the system behaves the same way as a kernel
		2682	built without CONFIG_IOMMU_DEBUG_PAGEALLOC.
		2683	Format: { "0" \| "1" }
		2684	0 - Sanitizer disabled.
		2685	1 - Sanitizer enabled, expect runtime overhead.
		2686
2678	io7= [HW] IO7 for Marvel-based Alpha systems	2687	io7= [HW] IO7 for Marvel-based Alpha systems
2679	See comment before marvel_specify_io7 in	2688	See comment before marvel_specify_io7 in
2680	arch/alpha/kernel/core_marvel.c.	2689	arch/alpha/kernel/core_marvel.c.

MAINTAINERS+1−0

@@ -13251,6 +13251,7 @@ F: drivers/iommu/ · 13250 unmodified lines
13251	F: include/linux/iommu.h	13251	F: include/linux/iommu.h
13252	F: include/linux/iova.h	13252	F: include/linux/iova.h
13253	F: include/linux/of_iommu.h	13253	F: include/linux/of_iommu.h
		13254	F: rust/kernel/iommu/
13254		13255
13255	IOMMUFD	13256	IOMMUFD
13256	M: Jason Gunthorpe <jgg@nvidia.com>	13257	M: Jason Gunthorpe <jgg@nvidia.com>

drivers/iommu/Kconfig+19−0

@@ -384,6 +384,25 @@ config SPRD_IOMMU · 383 unmodified lines
384		384
385	Say Y here if you want to use the multimedia devices listed above.	385	Say Y here if you want to use the multimedia devices listed above.
386		386
		387	config IOMMU_DEBUG_PAGEALLOC
		388	bool "Debug IOMMU mappings against page allocations"
		389	depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION
		390	help
		391	This enables a consistency check between the kernel page allocator and
		392	the IOMMU subsystem. It verifies that pages being allocated or freed
		393	are not currently mapped in any IOMMU domain.
		394
		395	This helps detect DMA use-after-free bugs where a driver frees a page
		396	but forgets to unmap it from the IOMMU, potentially allowing a device
		397	to overwrite memory that the kernel has repurposed.
		398
		399	These checks are best-effort and may not detect all problems.
		400
		401	Due to performance overhead, this feature is disabled by default.
		402	You must enable "iommu.debug_pagealloc" from the kernel command
		403	line to activate the runtime checks.
		404
		405	If unsure, say N.
387	endif # IOMMU_SUPPORT	406	endif # IOMMU_SUPPORT
388		407
389	source "drivers/iommu/generic_pt/Kconfig"	408	source "drivers/iommu/generic_pt/Kconfig"

drivers/iommu/Makefile+1−0

@@ -36,3 +36,4 @@ obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o · 35 unmodified lines
36	obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o	36	obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o
37	obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o	37	obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
38	obj-$(CONFIG_APPLE_DART) += apple-dart.o	38	obj-$(CONFIG_APPLE_DART) += apple-dart.o
		39	obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o

drivers/iommu/amd/Kconfig+10−0

@@ -30,6 +30,16 @@ config AMD_IOMMU · 29 unmodified lines
30	your BIOS for an option to enable it or if you have an IVRS ACPI	30	your BIOS for an option to enable it or if you have an IVRS ACPI
31	table.	31	table.
32		32
		33	config AMD_IOMMU_IOMMUFD
		34	bool "Enable IOMMUFD features for AMD IOMMU (EXPERIMENTAL)"
		35	depends on IOMMUFD
		36	depends on AMD_IOMMU
		37	help
		38	Support for IOMMUFD features intended to support virtual machines
		39	with accelerated virtual IOMMUs.
		40
		41	Say Y here if you are doing development and testing on this feature.
		42
33	config AMD_IOMMU_DEBUGFS	43	config AMD_IOMMU_DEBUGFS
34	bool "Enable AMD IOMMU internals in DebugFS"	44	bool "Enable AMD IOMMU internals in DebugFS"
35	depends on AMD_IOMMU && IOMMU_DEBUGFS	45	depends on AMD_IOMMU && IOMMU_DEBUGFS

drivers/iommu/amd/Makefile+1−0

@@ -1,3 +1,4 @@
1	# SPDX-License-Identifier: GPL-2.0-only	1	# SPDX-License-Identifier: GPL-2.0-only
2	obj-y += iommu.o init.o quirks.o ppr.o pasid.o	2	obj-y += iommu.o init.o quirks.o ppr.o pasid.o
		3	obj-$(CONFIG_AMD_IOMMU_IOMMUFD) += iommufd.o nested.o
3	obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o	4	obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o

drivers/iommu/amd/amd_iommu.h+33−0

@@ -190,4 +190,37 @@ void amd_iommu_domain_set_pgtable(struct protection_domain *domain, · 189 unmodified lines
190	struct dev_table_entry get_dev_table(struct amd_iommu iommu);	190	struct dev_table_entry get_dev_table(struct amd_iommu iommu);
191	struct iommu_dev_data search_dev_data(struct amd_iommu iommu, u16 devid);	191	struct iommu_dev_data search_dev_data(struct amd_iommu iommu, u16 devid);
192		192
		193	void amd_iommu_set_dte_v1(struct iommu_dev_data *dev_data,
		194	struct protection_domain *domain, u16 domid,
		195	struct pt_iommu_amdv1_hw_info *pt_info,
		196	struct dev_table_entry *new);
		197	void amd_iommu_update_dte(struct amd_iommu *iommu,
		198	struct iommu_dev_data *dev_data,
		199	struct dev_table_entry *new);
		200
		201	static inline void
		202	amd_iommu_make_clear_dte(struct iommu_dev_data dev_data, struct dev_table_entry new)
		203	{
		204	struct dev_table_entry *initial_dte;
		205	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
		206
		207	/* All existing DTE must have V bit set */
		208	new->data128[0] = DTE_FLAG_V;
		209	new->data128[1] = 0;
		210
		211	/*
		212	* Restore cached persistent DTE bits, which can be set by information
		213	* in IVRS table. See set_dev_entry_from_acpi().
		214	*/
		215	initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid);
		216	if (initial_dte) {
		217	new->data128[0] \|= initial_dte->data128[0];
		218	new->data128[1] \|= initial_dte->data128[1];
		219	}
		220	}
		221
		222	/* NESTED */
		223	struct iommu_domain *
		224	amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags,
		225	const struct iommu_user_data *user_data);
193	#endif /* AMD_IOMMU_H */	226	#endif /* AMD_IOMMU_H */

drivers/iommu/amd/amd_iommu_types.h+48−2

 #include <linux/list.h>
 #include <linux/spinlock.h>
 #include <linux/pci.h>
+#include <linux/iommufd.h>
 #include <linux/irqreturn.h>
 #include <linux/generic_pt/iommu.h>
+#include <uapi/linux/iommufd.h>
 /*
  * Maximum number of IOMMUs supported
  */
 /* Extended Feature 2 Bits */
 #define FEATURE_SEVSNPIO_SUP	BIT_ULL(1)
+#define FEATURE_GCR3TRPMODE	BIT_ULL(3)
 #define FEATURE_SNPAVICSUP	GENMASK_ULL(7, 5)
 #define FEATURE_SNPAVICSUP_GAM(x) \
 	(FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
 #define CONTROL_EPH_EN		45
 #define CONTROL_XT_EN		50
 #define CONTROL_INTCAPXT_EN	51
+#define CONTROL_GCR3TRPMODE	58
 #define CONTROL_IRTCACHEDIS	59
 #define CONTROL_SNPAVIC_EN	61
 #define DTE_FLAG_V	BIT_ULL(0)
 #define DTE_FLAG_TV	BIT_ULL(1)
 #define DTE_FLAG_HAD	(3ULL << 7)
+#define DTE_MODE_MASK	GENMASK_ULL(11, 9)
+#define DTE_HOST_TRP	GENMASK_ULL(51, 12)
+#define DTE_FLAG_PPR	BIT_ULL(52)
 #define DTE_FLAG_GIOV	BIT_ULL(54)
 #define DTE_FLAG_GV	BIT_ULL(55)
 #define DTE_GLX		GENMASK_ULL(57, 56)
 #define DTE_FLAG_IOTLB	BIT_ULL(32)
 #define DTE_FLAG_MASK	(0x3ffULL << 32)
-#define DEV_DOMID_MASK	0xffffULL
+#define DTE_DOMID_MASK	GENMASK_ULL(15, 0)
 #define DTE_GCR3_14_12	GENMASK_ULL(60, 58)
 #define DTE_GCR3_30_15	GENMASK_ULL(31, 16)
 	u32 refcnt;	/* Count of attached dev/pasid per domain/IOMMU */
 };
+struct amd_iommu_viommu {
+	struct iommufd_viommu core;
+	struct protection_domain *parent; /* nest parent domain for this viommu */
+	struct list_head pdom_list;	  /* For protection_domain->viommu_list */
+	/*
+	 * Per-vIOMMU guest domain ID to host domain ID mapping.
+	 * Indexed by guest domain ID.
+	 */
+	struct xarray gdomid_array;
+};
+/*
+ * Contains guest domain ID mapping info,
+ * which is stored in the struct xarray gdomid_array.
+ */
+struct guest_domain_mapping_info {
+	refcount_t users;
+	u32 hdom_id;		/* Host domain ID */
+};
+/*
+ * Nested domain is specifically used for nested translation
+ */
+struct nested_domain {
+	struct iommu_domain domain; /* generic domain handle used by iommu core code */
+	u16 gdom_id;                /* domain ID from gDTE */
+	struct guest_domain_mapping_info *gdom_info;
+	struct iommu_hwpt_amd_guest gdte; /* Guest vIOMMU DTE */
+	struct amd_iommu_viommu *viommu;  /* AMD hw-viommu this nested domain belong to */
+};
 /*
  * This structure contains generic data for  IOMMU protection domains
  * independent of their use.
 	struct mmu_notifier mn;	/* mmu notifier for the SVA domain */
 	struct list_head dev_data_list; /* List of pdom_dev_data */
+	/*
+	 * Store reference to list of vIOMMUs, which use this protection domain.
+	 * This will be used to look up host domain ID when flushing this domain.
+	 */
+	struct list_head viommu_list;
 };
 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain);
 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain);
 	u32 flags;
 	volatile u64 *cmd_sem;
-	atomic64_t cmd_sem_val;
+	u64 cmd_sem_val;
 	/*
 	 * Track physical address to directly use it in build_completion_wait()
 	 * and avoid adding any special checks and handling for kdump.

drivers/iommu/amd/init.c+10−2

 		return;
 	iommu_feature_enable(iommu, CONTROL_GT_EN);
+	/*
+	 * This feature needs to be enabled prior to a call
+	 * to iommu_snp_enable(). Since this function is called
+	 * in early_enable_iommu(), it is safe to enable here.
+	 */
+	if (check_feature2(FEATURE_GCR3TRPMODE))
+		iommu_feature_enable(iommu, CONTROL_GCR3TRPMODE);
 }
 /* sets a specific bit in the device table entry. */
 	for (devid = 0; devid <= pci_seg->last_bdf; devid++) {
 		old_dev_tbl_entry = &pci_seg->old_dev_tbl_cpy[devid];
 		dte_v = FIELD_GET(DTE_FLAG_V, old_dev_tbl_entry->data[0]);
-		dom_id = FIELD_GET(DEV_DOMID_MASK, old_dev_tbl_entry->data[1]);
+		dom_id = FIELD_GET(DTE_DOMID_MASK, old_dev_tbl_entry->data[1]);
 		if (!dte_v || !dom_id)
 			continue;
 	iommu->pci_seg = pci_seg;
 	raw_spin_lock_init(&iommu->lock);
-	atomic64_set(&iommu->cmd_sem_val, 0);
+	iommu->cmd_sem_val = 0;
 	/* Add IOMMU to internal data structures */
 	list_add_tail(&iommu->list, &amd_iommu_list);

drivers/iommu/amd/iommu.c+190−117

@@ -43,6 +43,7 @@ · 42 unmodified lines
43	#include <linux/generic_pt/iommu.h>	43	#include <linux/generic_pt/iommu.h>
44		44
45	#include "amd_iommu.h"	45	#include "amd_iommu.h"
		46	#include "iommufd.h"
46	#include "../irq_remapping.h"	47	#include "../irq_remapping.h"
47	#include "../iommu-pages.h"	48	#include "../iommu-pages.h"
48		49
@@ -75,6 +76,8 @@ static void set_dte_entry(struct amd_iommu *iommu, · 26 unmodified lines
75	struct iommu_dev_data *dev_data,	76	struct iommu_dev_data *dev_data,
76	phys_addr_t top_paddr, unsigned int top_level);	77	phys_addr_t top_paddr, unsigned int top_level);
77		78
		79	static int device_flush_dte(struct iommu_dev_data *dev_data);
		80
78	static void amd_iommu_change_top(struct pt_iommu *iommu_table,	81	static void amd_iommu_change_top(struct pt_iommu *iommu_table,
79	phys_addr_t top_paddr, unsigned int top_level);	82	phys_addr_t top_paddr, unsigned int top_level);
80		83
@@ -85,6 +88,10 @@ static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain); · 4 unmodified lines
85	static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain,	88	static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain,
86	bool enable);	89	bool enable);
87		90
		91	static void clone_aliases(struct amd_iommu iommu, struct device dev);
		92
		93	static int iommu_completion_wait(struct amd_iommu *iommu);
		94
88	/****************************************************************************	95	/****************************************************************************
89	*	96	*
90	* Helper functions	97	* Helper functions
@@ -202,6 +209,16 @@ static void update_dte256(struct amd_iommu iommu, struct iommu_dev_data dev_da · 111 unmodified lines
202	spin_unlock_irqrestore(&dev_data->dte_lock, flags);	209	spin_unlock_irqrestore(&dev_data->dte_lock, flags);
203	}	210	}
204		211
		212	void amd_iommu_update_dte(struct amd_iommu *iommu,
		213	struct iommu_dev_data *dev_data,
		214	struct dev_table_entry *new)
		215	{
		216	update_dte256(iommu, dev_data, new);
		217	clone_aliases(iommu, dev_data->dev);
		218	device_flush_dte(dev_data);
		219	iommu_completion_wait(iommu);
		220	}
		221
205	static void get_dte256(struct amd_iommu iommu, struct iommu_dev_data dev_data,	222	static void get_dte256(struct amd_iommu iommu, struct iommu_dev_data dev_data,
206	struct dev_table_entry *dte)	223	struct dev_table_entry *dte)
207	{	224	{
@@ -1185,7 +1202,12 @@ static int wait_on_sem(struct amd_iommu *iommu, u64 data) · 977 unmodified lines
1185	{	1202	{
1186	int i = 0;	1203	int i = 0;
1187		1204
1188	while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) {	1205	/*
		1206	* cmd_sem holds a monotonically non-decreasing completion sequence
		1207	* number.
		1208	*/
		1209	while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 &&
		1210	i < LOOP_TIMEOUT) {
1189	udelay(1);	1211	udelay(1);
1190	i += 1;	1212	i += 1;
1191	}	1213	}
@@ -1417,6 +1439,12 @@ static int iommu_queue_command(struct amd_iommu iommu, struct iommu_cmd cmd) · 225 unmodified lines
1417	return iommu_queue_command_sync(iommu, cmd, true);	1439	return iommu_queue_command_sync(iommu, cmd, true);
1418	}	1440	}
1419		1441
		1442	static u64 get_cmdsem_val(struct amd_iommu *iommu)
		1443	{
		1444	lockdep_assert_held(&iommu->lock);
		1445	return ++iommu->cmd_sem_val;
		1446	}
		1447
1420	/*	1448	/*
1421	* This function queues a completion wait command into the command	1449	* This function queues a completion wait command into the command
1422	* buffer of an IOMMU	1450	* buffer of an IOMMU
@@ -1431,20 +1459,19 @@ static int iommu_completion_wait(struct amd_iommu *iommu) · 8 unmodified lines
1431	if (!iommu->need_sync)	1459	if (!iommu->need_sync)
1432	return 0;	1460	return 0;
1433		1461
1434	data = atomic64_inc_return(&iommu->cmd_sem_val);
1435	build_completion_wait(&cmd, iommu, data);
1436
1437	raw_spin_lock_irqsave(&iommu->lock, flags);	1462	raw_spin_lock_irqsave(&iommu->lock, flags);
1438		1463
		1464	data = get_cmdsem_val(iommu);
		1465	build_completion_wait(&cmd, iommu, data);
		1466
1439	ret = __iommu_queue_command_sync(iommu, &cmd, false);	1467	ret = __iommu_queue_command_sync(iommu, &cmd, false);
		1468	raw_spin_unlock_irqrestore(&iommu->lock, flags);
		1469
1440	if (ret)	1470	if (ret)
1441	goto out_unlock;	1471	return ret;
1442		1472
1443	ret = wait_on_sem(iommu, data);	1473	ret = wait_on_sem(iommu, data);
1444		1474
1445	out_unlock:
1446	raw_spin_unlock_irqrestore(&iommu->lock, flags);
1447
1448	return ret;	1475	return ret;
1449	}	1476	}
1450		1477
@@ -1522,6 +1549,32 @@ static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id) · 71 unmodified lines
1522	iommu_completion_wait(iommu);	1549	iommu_completion_wait(iommu);
1523	}	1550	}
1524		1551
		1552	static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 address, size_t size)
		1553	{
		1554	int ret = 0;
		1555	struct amd_iommu_viommu *aviommu;
		1556
		1557	list_for_each_entry(aviommu, &pdom->viommu_list, pdom_list) {
		1558	unsigned long i;
		1559	struct guest_domain_mapping_info *gdom_info;
		1560	struct amd_iommu *iommu = container_of(aviommu->core.iommu_dev,
		1561	struct amd_iommu, iommu);
		1562
		1563	xa_lock(&aviommu->gdomid_array);
		1564	xa_for_each(&aviommu->gdomid_array, i, gdom_info) {
		1565	struct iommu_cmd cmd;
		1566
		1567	pr_debug("%s: iommu=%#x, hdom_id=%#x\n", __func__,
		1568	iommu->devid, gdom_info->hdom_id);
		1569	build_inv_iommu_pages(&cmd, address, size, gdom_info->hdom_id,
		1570	IOMMU_NO_PASID, false);
		1571	ret \|= iommu_queue_command(iommu, &cmd);
		1572	}
		1573	xa_unlock(&aviommu->gdomid_array);
		1574	}
		1575	return ret;
		1576	}
		1577
1525	static void amd_iommu_flush_all(struct amd_iommu *iommu)	1578	static void amd_iommu_flush_all(struct amd_iommu *iommu)
1526	{	1579	{
1527	struct iommu_cmd cmd;	1580	struct iommu_cmd cmd;
@@ -1670,6 +1723,17 @@ static int domain_flush_pages_v1(struct protection_domain *pdom, · 142 unmodified lines
1670	ret \|= iommu_queue_command(pdom_iommu_info->iommu, &cmd);	1723	ret \|= iommu_queue_command(pdom_iommu_info->iommu, &cmd);
1671	}	1724	}
1672		1725
		1726	/*
		1727	* A domain w/ v1 table can be a nest parent, which can have
		1728	* multiple nested domains. Each nested domain has 1:1 mapping
		1729	* between gDomID and hDomID. Therefore, flush every hDomID
		1730	* associated to this nest parent domain.
		1731	*
		1732	* See drivers/iommu/amd/nested.c: amd_iommu_alloc_domain_nested()
		1733	*/
		1734	if (!list_empty(&pdom->viommu_list))
		1735	ret \|= iommu_flush_pages_v1_hdom_ids(pdom, address, size);
		1736
1673	return ret;	1737	return ret;
1674	}	1738	}
1675		1739
@@ -2010,127 +2074,112 @@ int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid) · 334 unmodified lines
2010	return ret;	2074	return ret;
2011	}	2075	}
2012		2076
2013	static void make_clear_dte(struct iommu_dev_data dev_data, struct dev_table_entry ptr,
2014	struct dev_table_entry *new)
2015	{
2016	/* All existing DTE must have V bit set */
2017	new->data128[0] = DTE_FLAG_V;
2018	new->data128[1] = 0;
2019	}
2020
2021	/*	2077	/*
2022	* Note:	2078	* Note:
2023	* The old value for GCR3 table and GPT have been cleared from caller.	2079	* The old value for GCR3 table and GPT have been cleared from caller.
2024	*/	2080	*/
2025	static void set_dte_gcr3_table(struct amd_iommu *iommu,	2081	static void set_dte_gcr3_table(struct iommu_dev_data *dev_data,
2026	struct iommu_dev_data *dev_data,	2082	struct dev_table_entry *new)
2027	struct dev_table_entry *target)
2028	{	2083	{
2029	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;	2084	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
2030	u64 gcr3;	2085	u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl);
2031
2032	if (!gcr3_info->gcr3_tbl)
2033	return;
2034
2035	pr_debug("%s: devid=%#x, glx=%#x, gcr3_tbl=%#llx\n",
2036	__func__, dev_data->devid, gcr3_info->glx,
2037	(unsigned long long)gcr3_info->gcr3_tbl);
2038		2086
2039	gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl);	2087	new->data[0] \|= DTE_FLAG_TV \|
		2088	(dev_data->ppr ? DTE_FLAG_PPR : 0) \|
		2089	(pdom_is_v2_pgtbl_mode(dev_data->domain) ? DTE_FLAG_GIOV : 0) \|
		2090	DTE_FLAG_GV \|
		2091	FIELD_PREP(DTE_GLX, gcr3_info->glx) \|
		2092	FIELD_PREP(DTE_GCR3_14_12, gcr3 >> 12) \|
		2093	DTE_FLAG_IR \| DTE_FLAG_IW;
2040		2094
2041	target->data[0] \|= DTE_FLAG_GV \|	2095	new->data[1] \|= FIELD_PREP(DTE_DOMID_MASK, dev_data->gcr3_info.domid) \|
2042	FIELD_PREP(DTE_GLX, gcr3_info->glx) \|	2096	FIELD_PREP(DTE_GCR3_30_15, gcr3 >> 15) \|
2043	FIELD_PREP(DTE_GCR3_14_12, gcr3 >> 12);	2097	(dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0) \|
2044	if (pdom_is_v2_pgtbl_mode(dev_data->domain))	2098	FIELD_PREP(DTE_GCR3_51_31, gcr3 >> 31);
2045	target->data[0] \|= DTE_FLAG_GIOV;
2046
2047	target->data[1] \|= FIELD_PREP(DTE_GCR3_30_15, gcr3 >> 15) \|
2048	FIELD_PREP(DTE_GCR3_51_31, gcr3 >> 31);
2049		2099
2050	/* Guest page table can only support 4 and 5 levels */	2100	/* Guest page table can only support 4 and 5 levels */
2051	if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL)	2101	if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL)
2052	target->data[2] \|= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_5_LEVEL);	2102	new->data[2] \|= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_5_LEVEL);
2053	else	2103	else
2054	target->data[2] \|= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_4_LEVEL);	2104	new->data[2] \|= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_4_LEVEL);
2055	}	2105	}
2056		2106
2057	static void set_dte_entry(struct amd_iommu *iommu,	2107	void amd_iommu_set_dte_v1(struct iommu_dev_data *dev_data,
2058	struct iommu_dev_data *dev_data,	2108	struct protection_domain *domain, u16 domid,
2059	phys_addr_t top_paddr, unsigned int top_level)	2109	struct pt_iommu_amdv1_hw_info *pt_info,
		2110	struct dev_table_entry *new)
2060	{	2111	{
2061	u16 domid;	2112	u64 host_pt_root = __sme_set(pt_info->host_pt_root);
2062	u32 old_domid;
2063	struct dev_table_entry *initial_dte;
2064	struct dev_table_entry new = {};
2065	struct protection_domain *domain = dev_data->domain;
2066	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
2067	struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];
2068	struct pt_iommu_amdv1_hw_info pt_info;
2069		2113
2070	make_clear_dte(dev_data, dte, &new);	2114	/* Note Dirty tracking is used for v1 table only for now */
2071		2115	new->data[0] \|= DTE_FLAG_TV \|
2072	if (gcr3_info && gcr3_info->gcr3_tbl)	2116	FIELD_PREP(DTE_MODE_MASK, pt_info->mode) \|
2073	domid = dev_data->gcr3_info.domid;	2117	(domain->dirty_tracking ? DTE_FLAG_HAD : 0) \|
2074	else {	2118	FIELD_PREP(DTE_HOST_TRP, host_pt_root >> 12) \|
2075	domid = domain->id;	2119	DTE_FLAG_IR \| DTE_FLAG_IW;
2076
2077	if (domain->domain.type & __IOMMU_DOMAIN_PAGING) {
2078	/*
2079	* When updating the IO pagetable, the new top and level
2080	* are provided as parameters. For other operations i.e.
2081	* device attach, retrieve the current pagetable info
2082	* via the IOMMU PT API.
2083	*/
2084	if (top_paddr) {
2085	pt_info.host_pt_root = top_paddr;
2086	pt_info.mode = top_level + 1;
2087	} else {
2088	WARN_ON(top_paddr \|\| top_level);
2089	pt_iommu_amdv1_hw_info(&domain->amdv1,
2090	&pt_info);
2091	}
2092		2120
2093	new.data[0] \|= __sme_set(pt_info.host_pt_root) \|	2121	new->data[1] \|= FIELD_PREP(DTE_DOMID_MASK, domid) \|
2094	(pt_info.mode & DEV_ENTRY_MODE_MASK)	2122	(dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0);
2095	<< DEV_ENTRY_MODE_SHIFT;	2123	}
2096	}
2097	}
2098		2124
2099	new.data[0] \|= DTE_FLAG_IR \| DTE_FLAG_IW;	2125	static void set_dte_v1(struct iommu_dev_data *dev_data,
		2126	struct protection_domain *domain, u16 domid,
		2127	phys_addr_t top_paddr, unsigned int top_level,
		2128	struct dev_table_entry *new)
		2129	{
		2130	struct pt_iommu_amdv1_hw_info pt_info;
2100		2131
2101	/*	2132	/*
2102	* When SNP is enabled, we can only support TV=1 with non-zero domain ID.	2133	* When updating the IO pagetable, the new top and level
2103	* This is prevented by the SNP-enable and IOMMU_DOMAIN_IDENTITY check in	2134	* are provided as parameters. For other operations i.e.
2104	* do_iommu_domain_alloc().	2135	* device attach, retrieve the current pagetable info
		2136	* via the IOMMU PT API.
2105	*/	2137	*/
2106	WARN_ON(amd_iommu_snp_en && (domid == 0));	2138	if (top_paddr) {
2107	new.data[0] \|= DTE_FLAG_TV;	2139	pt_info.host_pt_root = top_paddr;
		2140	pt_info.mode = top_level + 1;
		2141	} else {
		2142	WARN_ON(top_paddr \|\| top_level);
		2143	pt_iommu_amdv1_hw_info(&domain->amdv1, &pt_info);
		2144	}
2108		2145
2109	if (dev_data->ppr)	2146	amd_iommu_set_dte_v1(dev_data, domain, domid, &pt_info, new);
2110	new.data[0] \|= 1ULL << DEV_ENTRY_PPR;	2147	}
2111		2148
2112	if (domain->dirty_tracking)	2149	static void set_dte_passthrough(struct iommu_dev_data *dev_data,
2113	new.data[0] \|= DTE_FLAG_HAD;	2150	struct protection_domain *domain,
		2151	struct dev_table_entry *new)
		2152	{
		2153	new->data[0] \|= DTE_FLAG_TV \| DTE_FLAG_IR \| DTE_FLAG_IW;
2114		2154
2115	if (dev_data->ats_enabled)	2155	new->data[1] \|= FIELD_PREP(DTE_DOMID_MASK, domain->id) \|
2116	new.data[1] \|= DTE_FLAG_IOTLB;	2156	(dev_data->ats_enabled) ? DTE_FLAG_IOTLB : 0;
		2157	}
2117		2158
2118	old_domid = READ_ONCE(dte->data[1]) & DEV_DOMID_MASK;	2159	static void set_dte_entry(struct amd_iommu *iommu,
2119	new.data[1] \|= domid;	2160	struct iommu_dev_data *dev_data,
		2161	phys_addr_t top_paddr, unsigned int top_level)
		2162	{
		2163	u32 old_domid;
		2164	struct dev_table_entry new = {};
		2165	struct protection_domain *domain = dev_data->domain;
		2166	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
		2167	struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];
2120		2168
2121	/*	2169	amd_iommu_make_clear_dte(dev_data, &new);
2122	* Restore cached persistent DTE bits, which can be set by information
2123	* in IVRS table. See set_dev_entry_from_acpi().
2124	*/
2125	initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid);
2126	if (initial_dte) {
2127	new.data128[0] \|= initial_dte->data128[0];
2128	new.data128[1] \|= initial_dte->data128[1];
2129	}
2130		2170
2131	set_dte_gcr3_table(iommu, dev_data, &new);	2171	old_domid = READ_ONCE(dte->data[1]) & DTE_DOMID_MASK;
		2172	if (gcr3_info->gcr3_tbl)
		2173	set_dte_gcr3_table(dev_data, &new);
		2174	else if (domain->domain.type == IOMMU_DOMAIN_IDENTITY)
		2175	set_dte_passthrough(dev_data, domain, &new);
		2176	else if ((domain->domain.type & __IOMMU_DOMAIN_PAGING) &&
		2177	domain->pd_mode == PD_MODE_V1)
		2178	set_dte_v1(dev_data, domain, domain->id, top_paddr, top_level, &new);
		2179	else
		2180	WARN_ON(true);
2132		2181
2133	update_dte256(iommu, dev_data, &new);	2182	amd_iommu_update_dte(iommu, dev_data, &new);
2134		2183
2135	/*	2184	/*
2136	* A kdump kernel might be replacing a domain ID that was copied from	2185	* A kdump kernel might be replacing a domain ID that was copied from
@@ -2148,10 +2197,9 @@ static void set_dte_entry(struct amd_iommu *iommu, · 11 unmodified lines
2148	static void clear_dte_entry(struct amd_iommu iommu, struct iommu_dev_data dev_data)	2197	static void clear_dte_entry(struct amd_iommu iommu, struct iommu_dev_data dev_data)
2149	{	2198	{
2150	struct dev_table_entry new = {};	2199	struct dev_table_entry new = {};
2151	struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];
2152		2200
2153	make_clear_dte(dev_data, dte, &new);	2201	amd_iommu_make_clear_dte(dev_data, &new);
2154	update_dte256(iommu, dev_data, &new);	2202	amd_iommu_update_dte(iommu, dev_data, &new);
2155	}	2203	}
2156		2204
2157	/* Update and flush DTE for the given device */	2205	/* Update and flush DTE for the given device */
@@ -2163,10 +2211,6 @@ static void dev_update_dte(struct iommu_dev_data *dev_data, bool set) · 5 unmodified lines
2163	set_dte_entry(iommu, dev_data, 0, 0);	2211	set_dte_entry(iommu, dev_data, 0, 0);
2164	else	2212	else
2165	clear_dte_entry(iommu, dev_data);	2213	clear_dte_entry(iommu, dev_data);
2166
2167	clone_aliases(iommu, dev_data->dev);
2168	device_flush_dte(dev_data);
2169	iommu_completion_wait(iommu);
2170	}	2214	}
2171		2215
2172	/*	2216	/*
@@ -2499,6 +2543,7 @@ static void protection_domain_init(struct protection_domain *domain) · 326 unmodified lines
2499	spin_lock_init(&domain->lock);	2543	spin_lock_init(&domain->lock);
2500	INIT_LIST_HEAD(&domain->dev_list);	2544	INIT_LIST_HEAD(&domain->dev_list);
2501	INIT_LIST_HEAD(&domain->dev_data_list);	2545	INIT_LIST_HEAD(&domain->dev_data_list);
		2546	INIT_LIST_HEAD(&domain->viommu_list);
2502	xa_init(&domain->iommu_array);	2547	xa_init(&domain->iommu_array);
2503	}	2548	}
2504		2549
@@ -2760,6 +2805,14 @@ static struct iommu_domain amd_iommu_domain_alloc_paging_v2(struct device dev, · 255 unmodified lines
2760	return &domain->domain;	2805	return &domain->domain;
2761	}	2806	}
2762		2807
		2808	static inline bool is_nest_parent_supported(u32 flags)
		2809	{
		2810	/* Only allow nest parent when these features are supported */
		2811	return check_feature(FEATURE_GT) &&
		2812	check_feature(FEATURE_GIOSUP) &&
		2813	check_feature2(FEATURE_GCR3TRPMODE);
		2814	}
		2815
2763	static struct iommu_domain *	2816	static struct iommu_domain *
2764	amd_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,	2817	amd_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,
2765	const struct iommu_user_data *user_data)	2818	const struct iommu_user_data *user_data)
@@ -2767,16 +2820,28 @@ amd_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags, · 1 unmodified line
2767	{	2820	{
2768	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);	2821	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
2769	const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING \|	2822	const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING \|
2770	IOMMU_HWPT_ALLOC_PASID;	2823	IOMMU_HWPT_ALLOC_PASID \|
		2824	IOMMU_HWPT_ALLOC_NEST_PARENT;
2771		2825
2772	if ((flags & ~supported_flags) \|\| user_data)	2826	if ((flags & ~supported_flags) \|\| user_data)
2773	return ERR_PTR(-EOPNOTSUPP);	2827	return ERR_PTR(-EOPNOTSUPP);
2774		2828
2775	switch (flags & supported_flags) {	2829	switch (flags & supported_flags) {
2776	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING:	2830	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING:
2777	/* Allocate domain with v1 page table for dirty tracking */	2831	case IOMMU_HWPT_ALLOC_NEST_PARENT:
2778	if (!amd_iommu_hd_support(iommu))	2832	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING \| IOMMU_HWPT_ALLOC_NEST_PARENT:
		2833	/*
		2834	* Allocate domain with v1 page table for dirty tracking
		2835	* and/or Nest parent.
		2836	*/
		2837	if ((flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) &&
		2838	!amd_iommu_hd_support(iommu))
2779	break;	2839	break;
		2840
		2841	if ((flags & IOMMU_HWPT_ALLOC_NEST_PARENT) &&
		2842	!is_nest_parent_supported(flags))
		2843	break;
		2844
2780	return amd_iommu_domain_alloc_paging_v1(dev, flags);	2845	return amd_iommu_domain_alloc_paging_v1(dev, flags);
2781	case IOMMU_HWPT_ALLOC_PASID:	2846	case IOMMU_HWPT_ALLOC_PASID:
2782	/* Allocate domain with v2 page table if IOMMU supports PASID. */	2847	/* Allocate domain with v2 page table if IOMMU supports PASID. */
@@ -3078,6 +3143,7 @@ static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain) · 295 unmodified lines
3078		3143
3079	const struct iommu_ops amd_iommu_ops = {	3144	const struct iommu_ops amd_iommu_ops = {
3080	.capable = amd_iommu_capable,	3145	.capable = amd_iommu_capable,
		3146	.hw_info = amd_iommufd_hw_info,
3081	.blocked_domain = &blocked_domain,	3147	.blocked_domain = &blocked_domain,
3082	.release_domain = &blocked_domain,	3148	.release_domain = &blocked_domain,
3083	.identity_domain = &identity_domain.domain,	3149	.identity_domain = &identity_domain.domain,
@@ -3090,6 +3156,8 @@ const struct iommu_ops amd_iommu_ops = { · 6 unmodified lines
3090	.is_attach_deferred = amd_iommu_is_attach_deferred,	3156	.is_attach_deferred = amd_iommu_is_attach_deferred,
3091	.def_domain_type = amd_iommu_def_domain_type,	3157	.def_domain_type = amd_iommu_def_domain_type,
3092	.page_response = amd_iommu_page_response,	3158	.page_response = amd_iommu_page_response,
		3159	.get_viommu_size = amd_iommufd_get_viommu_size,
		3160	.viommu_init = amd_iommufd_viommu_init,
3093	};	3161	};
3094		3162
3095	#ifdef CONFIG_IRQ_REMAP	3163	#ifdef CONFIG_IRQ_REMAP
@@ -3114,18 +3182,23 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) · 18 unmodified lines
3114	return;	3182	return;
3115		3183
3116	build_inv_irt(&cmd, devid);	3184	build_inv_irt(&cmd, devid);
3117	data = atomic64_inc_return(&iommu->cmd_sem_val);
3118	build_completion_wait(&cmd2, iommu, data);
3119		3185
3120	raw_spin_lock_irqsave(&iommu->lock, flags);	3186	raw_spin_lock_irqsave(&iommu->lock, flags);
		3187	data = get_cmdsem_val(iommu);
		3188	build_completion_wait(&cmd2, iommu, data);
		3189
3121	ret = __iommu_queue_command_sync(iommu, &cmd, true);	3190	ret = __iommu_queue_command_sync(iommu, &cmd, true);
3122	if (ret)	3191	if (ret)
3123	goto out;	3192	goto out_err;
3124	ret = __iommu_queue_command_sync(iommu, &cmd2, false);	3193	ret = __iommu_queue_command_sync(iommu, &cmd2, false);
3125	if (ret)	3194	if (ret)
3126	goto out;	3195	goto out_err;
		3196	raw_spin_unlock_irqrestore(&iommu->lock, flags);
		3197
3127	wait_on_sem(iommu, data);	3198	wait_on_sem(iommu, data);
3128	out:	3199	return;
		3200
		3201	out_err:
3129	raw_spin_unlock_irqrestore(&iommu->lock, flags);	3202	raw_spin_unlock_irqrestore(&iommu->lock, flags);
3130	}	3203	}
3131		3204
@@ -3239,7 +3312,7 @@ static struct irq_remap_table alloc_irq_table(struct amd_iommu iommu, · 107 unmodified lines
3239	struct irq_remap_table *new_table = NULL;	3312	struct irq_remap_table *new_table = NULL;
3240	struct amd_iommu_pci_seg *pci_seg;	3313	struct amd_iommu_pci_seg *pci_seg;
3241	unsigned long flags;	3314	unsigned long flags;
3242	int nid = iommu && iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;	3315	int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
3243	u16 alias;	3316	u16 alias;
3244		3317
3245	spin_lock_irqsave(&iommu_table_lock, flags);	3318	spin_lock_irqsave(&iommu_table_lock, flags);

drivers/iommu/amd/iommufd.c+77−0

@@ -0,0 +1,77 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2025 Advanced Micro Devices, Inc.
	4	*/
	5
	6	#include <linux/iommu.h>
	7
	8	#include "iommufd.h"
	9	#include "amd_iommu.h"
	10	#include "amd_iommu_types.h"
	11
	12	static const struct iommufd_viommu_ops amd_viommu_ops;
	13
	14	void amd_iommufd_hw_info(struct device dev, u32 length, enum iommu_hw_info_type type)
	15	{
	16	struct iommu_hw_info_amd *hwinfo;
	17
	18	if (*type != IOMMU_HW_INFO_TYPE_DEFAULT &&
	19	*type != IOMMU_HW_INFO_TYPE_AMD)
	20	return ERR_PTR(-EOPNOTSUPP);
	21
	22	hwinfo = kzalloc(sizeof(*hwinfo), GFP_KERNEL);
	23	if (!hwinfo)
	24	return ERR_PTR(-ENOMEM);
	25
	26	length = sizeof(hwinfo);
	27	*type = IOMMU_HW_INFO_TYPE_AMD;
	28
	29	hwinfo->efr = amd_iommu_efr;
	30	hwinfo->efr2 = amd_iommu_efr2;
	31
	32	return hwinfo;
	33	}
	34
	35	size_t amd_iommufd_get_viommu_size(struct device *dev, enum iommu_viommu_type viommu_type)
	36	{
	37	return VIOMMU_STRUCT_SIZE(struct amd_iommu_viommu, core);
	38	}
	39
	40	int amd_iommufd_viommu_init(struct iommufd_viommu viommu, struct iommu_domain parent,
	41	const struct iommu_user_data *user_data)
	42	{
	43	unsigned long flags;
	44	struct protection_domain *pdom = to_pdomain(parent);
	45	struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
	46
	47	xa_init_flags(&aviommu->gdomid_array, XA_FLAGS_ALLOC1);
	48	aviommu->parent = pdom;
	49
	50	viommu->ops = &amd_viommu_ops;
	51
	52	spin_lock_irqsave(&pdom->lock, flags);
	53	list_add(&aviommu->pdom_list, &pdom->viommu_list);
	54	spin_unlock_irqrestore(&pdom->lock, flags);
	55
	56	return 0;
	57	}
	58
	59	static void amd_iommufd_viommu_destroy(struct iommufd_viommu *viommu)
	60	{
	61	unsigned long flags;
	62	struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
	63	struct protection_domain *pdom = aviommu->parent;
	64
	65	spin_lock_irqsave(&pdom->lock, flags);
	66	list_del(&aviommu->pdom_list);
	67	spin_unlock_irqrestore(&pdom->lock, flags);
	68	xa_destroy(&aviommu->gdomid_array);
	69	}
	70
	71	/*
	72	* See include/linux/iommufd.h
	73	* struct iommufd_viommu_ops - vIOMMU specific operations
	74	*/
	75	static const struct iommufd_viommu_ops amd_viommu_ops = {
	76	.destroy = amd_iommufd_viommu_destroy,
	77	};

drivers/iommu/amd/iommufd.h+20−0

@@ -0,0 +1,20 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/*
	3	* Copyright (C) 2025 Advanced Micro Devices, Inc.
	4	*/
	5
	6	#ifndef AMD_IOMMUFD_H
	7	#define AMD_IOMMUFD_H
	8
	9	#if IS_ENABLED(CONFIG_AMD_IOMMU_IOMMUFD)
	10	void amd_iommufd_hw_info(struct device dev, u32 length, enum iommu_hw_info_type type);
	11	size_t amd_iommufd_get_viommu_size(struct device *dev, enum iommu_viommu_type viommu_type);
	12	int amd_iommufd_viommu_init(struct iommufd_viommu viommu, struct iommu_domain parent,
	13	const struct iommu_user_data *user_data);
	14	#else
	15	#define amd_iommufd_hw_info NULL
	16	#define amd_iommufd_viommu_init NULL
	17	#define amd_iommufd_get_viommu_size NULL
	18	#endif /* CONFIG_AMD_IOMMU_IOMMUFD */
	19
	20	#endif /* AMD_IOMMUFD_H */

drivers/iommu/amd/nested.c+294−0

@@ -0,0 +1,294 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2025 Advanced Micro Devices, Inc.
	4	*/
	5
	6	#define dev_fmt(fmt) "AMD-Vi: " fmt
	7
	8	#include <linux/iommu.h>
	9	#include <linux/refcount.h>
	10	#include <uapi/linux/iommufd.h>
	11
	12	#include "amd_iommu.h"
	13
	14	static const struct iommu_domain_ops nested_domain_ops;
	15
	16	static inline struct nested_domain to_ndomain(struct iommu_domain dom)
	17	{
	18	return container_of(dom, struct nested_domain, domain);
	19	}
	20
	21	/*
	22	* Validate guest DTE to make sure that configuration for host (v1)
	23	* and guest (v2) page tables are valid when allocating nested domain.
	24	*/
	25	static int validate_gdte_nested(struct iommu_hwpt_amd_guest *gdte)
	26	{
	27	u32 gpt_level = FIELD_GET(DTE_GPT_LEVEL_MASK, gdte->dte[2]);
	28
	29	/* Must be zero: Mode, Host-TPR */
	30	if (FIELD_GET(DTE_MODE_MASK, gdte->dte[0]) != 0 \|\|
	31	FIELD_GET(DTE_HOST_TRP, gdte->dte[0]) != 0)
	32	return -EINVAL;
	33
	34	/* GCR3 TRP must be non-zero if V, GV is set */
	35	if (FIELD_GET(DTE_FLAG_V, gdte->dte[0]) == 1 &&
	36	FIELD_GET(DTE_FLAG_GV, gdte->dte[0]) == 1 &&
	37	FIELD_GET(DTE_GCR3_14_12, gdte->dte[0]) == 0 &&
	38	FIELD_GET(DTE_GCR3_30_15, gdte->dte[1]) == 0 &&
	39	FIELD_GET(DTE_GCR3_51_31, gdte->dte[1]) == 0)
	40	return -EINVAL;
	41
	42	/* Valid Guest Paging Mode values are 0 and 1 */
	43	if (gpt_level != GUEST_PGTABLE_4_LEVEL &&
	44	gpt_level != GUEST_PGTABLE_5_LEVEL)
	45	return -EINVAL;
	46
	47	/* GLX = 3 is reserved */
	48	if (FIELD_GET(DTE_GLX, gdte->dte[0]) == 3)
	49	return -EINVAL;
	50
	51	/*
	52	* We need to check host capability before setting
	53	* the Guest Paging Mode
	54	*/
	55	if (gpt_level == GUEST_PGTABLE_5_LEVEL &&
	56	amd_iommu_gpt_level < PAGE_MODE_5_LEVEL)
	57	return -EOPNOTSUPP;
	58
	59	return 0;
	60	}
	61
	62	static void gdom_info_load_or_alloc_locked(struct xarray xa, unsigned long index)
	63	{
	64	struct guest_domain_mapping_info elm, res;
	65
	66	elm = xa_load(xa, index);
	67	if (elm)
	68	return elm;
	69
	70	xa_unlock(xa);
	71	elm = kzalloc(sizeof(struct guest_domain_mapping_info), GFP_KERNEL);
	72	xa_lock(xa);
	73	if (!elm)
	74	return ERR_PTR(-ENOMEM);
	75
	76	res = __xa_cmpxchg(xa, index, NULL, elm, GFP_KERNEL);
	77	if (xa_is_err(res))
	78	res = ERR_PTR(xa_err(res));
	79
	80	if (res) {
	81	kfree(elm);
	82	return res;
	83	}
	84
	85	refcount_set(&elm->users, 0);
	86	return elm;
	87	}
	88
	89	/*
	90	* This function is assigned to struct iommufd_viommu_ops.alloc_domain_nested()
	91	* during the call to struct iommu_ops.viommu_init().
	92	*/
	93	struct iommu_domain *
	94	amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags,
	95	const struct iommu_user_data *user_data)
	96	{
	97	int ret;
	98	struct nested_domain *ndom;
	99	struct guest_domain_mapping_info *gdom_info;
	100	struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
	101
	102	if (user_data->type != IOMMU_HWPT_DATA_AMD_GUEST)
	103	return ERR_PTR(-EOPNOTSUPP);
	104
	105	ndom = kzalloc(sizeof(*ndom), GFP_KERNEL);
	106	if (!ndom)
	107	return ERR_PTR(-ENOMEM);
	108
	109	ret = iommu_copy_struct_from_user(&ndom->gdte, user_data,
	110	IOMMU_HWPT_DATA_AMD_GUEST,
	111	dte);
	112	if (ret)
	113	goto out_err;
	114
	115	ret = validate_gdte_nested(&ndom->gdte);
	116	if (ret)
	117	goto out_err;
	118
	119	ndom->gdom_id = FIELD_GET(DTE_DOMID_MASK, ndom->gdte.dte[1]);
	120	ndom->domain.ops = &nested_domain_ops;
	121	ndom->domain.type = IOMMU_DOMAIN_NESTED;
	122	ndom->viommu = aviommu;
	123
	124	/*
	125	* Normally, when a guest has multiple pass-through devices,
	126	* the IOMMU driver setup DTEs with the same stage-2 table and
	127	* use the same host domain ID (hDomId). In case of nested translation,
	128	* if the guest setup different stage-1 tables with same PASID,
	129	* IOMMU would use the same TLB tag. This will results in TLB
	130	* aliasing issue.
	131	*
	132	* The guest is assigning gDomIDs based on its own algorithm for managing
	133	* cache tags of (DomID, PASID). Within a single viommu, the nest parent domain
	134	* (w/ S2 table) is used by all DTEs. But we need to consistently map the gDomID
	135	* to a single hDomID. This is done using an xarray in the vIOMMU to
	136	* keep track of the gDomID mapping. When the S2 is changed, the INVALIDATE_IOMMU_PAGES
	137	* command must be issued for each hDomID in the xarray.
	138	*/
	139	xa_lock(&aviommu->gdomid_array);
	140
	141	gdom_info = gdom_info_load_or_alloc_locked(&aviommu->gdomid_array, ndom->gdom_id);
	142	if (IS_ERR(gdom_info)) {
	143	xa_unlock(&aviommu->gdomid_array);
	144	ret = PTR_ERR(gdom_info);
	145	goto out_err;
	146	}
	147
	148	/* Check if gDomID exist */
	149	if (refcount_inc_not_zero(&gdom_info->users)) {
	150	ndom->gdom_info = gdom_info;
	151	xa_unlock(&aviommu->gdomid_array);
	152
	153	pr_debug("%s: Found gdom_id=%#x, hdom_id=%#x\n",
	154	__func__, ndom->gdom_id, gdom_info->hdom_id);
	155
	156	return &ndom->domain;
	157	}
	158
	159	/* The gDomID does not exist. We allocate new hdom_id */
	160	gdom_info->hdom_id = amd_iommu_pdom_id_alloc();
	161	if (gdom_info->hdom_id <= 0) {
	162	__xa_cmpxchg(&aviommu->gdomid_array,
	163	ndom->gdom_id, gdom_info, NULL, GFP_ATOMIC);
	164	xa_unlock(&aviommu->gdomid_array);
	165	ret = -ENOSPC;
	166	goto out_err_gdom_info;
	167	}
	168
	169	ndom->gdom_info = gdom_info;
	170	refcount_set(&gdom_info->users, 1);
	171
	172	xa_unlock(&aviommu->gdomid_array);
	173
	174	pr_debug("%s: Allocate gdom_id=%#x, hdom_id=%#x\n",
	175	__func__, ndom->gdom_id, gdom_info->hdom_id);
	176
	177	return &ndom->domain;
	178
	179	out_err_gdom_info:
	180	kfree(gdom_info);
	181	out_err:
	182	kfree(ndom);
	183	return ERR_PTR(ret);
	184	}
	185
	186	static void set_dte_nested(struct amd_iommu iommu, struct iommu_domain dom,
	187	struct iommu_dev_data dev_data, struct dev_table_entry new)
	188	{
	189	struct protection_domain *parent;
	190	struct nested_domain *ndom = to_ndomain(dom);
	191	struct iommu_hwpt_amd_guest *gdte = &ndom->gdte;
	192	struct pt_iommu_amdv1_hw_info pt_info;
	193
	194	/*
	195	* The nest parent domain is attached during the call to the
	196	* struct iommu_ops.viommu_init(), which will be stored as part
	197	* of the struct amd_iommu_viommu.parent.
	198	*/
	199	if (WARN_ON(!ndom->viommu \|\| !ndom->viommu->parent))
	200	return;
	201
	202	parent = ndom->viommu->parent;
	203	amd_iommu_make_clear_dte(dev_data, new);
	204
	205	/* Retrieve the current pagetable info via the IOMMU PT API. */
	206	pt_iommu_amdv1_hw_info(&parent->amdv1, &pt_info);
	207
	208	/*
	209	* Use domain ID from nested domain to program DTE.
	210	* See amd_iommu_alloc_domain_nested().
	211	*/
	212	amd_iommu_set_dte_v1(dev_data, parent, ndom->gdom_info->hdom_id,
	213	&pt_info, new);
	214
	215	/* GV is required for nested page table */
	216	new->data[0] \|= DTE_FLAG_GV;
	217
	218	/* Guest PPR */
	219	new->data[0] \|= gdte->dte[0] & DTE_FLAG_PPR;
	220
	221	/* Guest translation stuff */
	222	new->data[0] \|= gdte->dte[0] & (DTE_GLX \| DTE_FLAG_GIOV);
	223
	224	/* GCR3 table */
	225	new->data[0] \|= gdte->dte[0] & DTE_GCR3_14_12;
	226	new->data[1] \|= gdte->dte[1] & (DTE_GCR3_30_15 \| DTE_GCR3_51_31);
	227
	228	/* Guest paging mode */
	229	new->data[2] \|= gdte->dte[2] & DTE_GPT_LEVEL_MASK;
	230	}
	231
	232	static int nested_attach_device(struct iommu_domain dom, struct device dev,
	233	struct iommu_domain *old)
	234	{
	235	struct dev_table_entry new = {0};
	236	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
	237	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
	238	int ret = 0;
	239
	240	/*
	241	* Needs to make sure PASID is not enabled
	242	* for this attach path.
	243	*/
	244	if (WARN_ON(dev_data->pasid_enabled))
	245	return -EINVAL;
	246
	247	mutex_lock(&dev_data->mutex);
	248
	249	set_dte_nested(iommu, dom, dev_data, &new);
	250
	251	amd_iommu_update_dte(iommu, dev_data, &new);
	252
	253	mutex_unlock(&dev_data->mutex);
	254
	255	return ret;
	256	}
	257
	258	static void nested_domain_free(struct iommu_domain *dom)
	259	{
	260	struct guest_domain_mapping_info *curr;
	261	struct nested_domain *ndom = to_ndomain(dom);
	262	struct amd_iommu_viommu *aviommu = ndom->viommu;
	263
	264	xa_lock(&aviommu->gdomid_array);
	265
	266	if (!refcount_dec_and_test(&ndom->gdom_info->users)) {
	267	xa_unlock(&aviommu->gdomid_array);
	268	return;
	269	}
	270
	271	/*
	272	* The refcount for the gdom_id to hdom_id mapping is zero.
	273	* It is now safe to remove the mapping.
	274	*/
	275	curr = __xa_cmpxchg(&aviommu->gdomid_array, ndom->gdom_id,
	276	ndom->gdom_info, NULL, GFP_ATOMIC);
	277
	278	xa_unlock(&aviommu->gdomid_array);
	279	if (WARN_ON(!curr \|\| xa_err(curr)))
	280	return;
	281
	282	/* success */
	283	pr_debug("%s: Free gdom_id=%#x, hdom_id=%#x\n",
	284	__func__, ndom->gdom_id, curr->hdom_id);
	285
	286	amd_iommu_pdom_id_free(ndom->gdom_info->hdom_id);
	287	kfree(curr);
	288	kfree(ndom);
	289	}
	290
	291	static const struct iommu_domain_ops nested_domain_ops = {
	292	.attach_dev = nested_attach_device,
	293	.free = nested_domain_free,
	294	};

drivers/iommu/arm/Kconfig+0−1

@@ -121,7 +121,6 @@ config ARM_SMMU_V3_KUNIT_TEST · 120 unmodified lines
121		121
122	config TEGRA241_CMDQV	122	config TEGRA241_CMDQV
123	bool "NVIDIA Tegra241 CMDQ-V extension support for ARM SMMUv3"	123	bool "NVIDIA Tegra241 CMDQ-V extension support for ARM SMMUv3"
124	depends on ACPI
125	help	124	help
126	Support for NVIDIA CMDQ-Virtualization extension for ARM SMMUv3. The	125	Support for NVIDIA CMDQ-Virtualization extension for ARM SMMUv3. The
127	CMDQ-V extension is similar to v3.3 ECMDQ for multi command queues	126	CMDQ-V extension is similar to v3.3 ECMDQ for multi command queues

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c+3−1

 	 * config bit here base this off the EATS value in the STE. If the EATS
 	 * is set then the VM must generate ATC flushes.
 	 */
-	state.disable_ats = !nested_domain->enable_ats;
+	if (FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(nested_domain->ste[0])) ==
+	    STRTAB_STE_0_CFG_S1_TRANS)
+		state.disable_ats = !nested_domain->enable_ats;
 	ret = arm_smmu_attach_prepare(&state, domain);
 	if (ret) {
 		mutex_unlock(&arm_smmu_asid_lock);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c+75−3

@@ -33,18 +33,25 @@ static struct mm_struct sva_mm = { · 32 unmodified lines
33	enum arm_smmu_test_master_feat {	33	enum arm_smmu_test_master_feat {
34	ARM_SMMU_MASTER_TEST_ATS = BIT(0),	34	ARM_SMMU_MASTER_TEST_ATS = BIT(0),
35	ARM_SMMU_MASTER_TEST_STALL = BIT(1),	35	ARM_SMMU_MASTER_TEST_STALL = BIT(1),
		36	ARM_SMMU_MASTER_TEST_NESTED = BIT(2),
36	};	37	};
37		38
		39	static void arm_smmu_test_make_s2_ste(struct arm_smmu_ste *ste,
		40	enum arm_smmu_test_master_feat feat);
		41
38	static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry,	42	static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry,
39	const __le64 *used_bits,	43	const __le64 *used_bits,
40	const __le64 *target,	44	const __le64 *target,
		45	const __le64 *safe,
41	unsigned int length)	46	unsigned int length)
42	{	47	{
43	bool differs = false;	48	bool differs = false;
44	unsigned int i;	49	unsigned int i;
45		50
46	for (i = 0; i < length; i++) {	51	for (i = 0; i < length; i++) {
47	if ((entry[i] & used_bits[i]) != target[i])	52	__le64 used = used_bits[i] & ~safe[i];
		53
		54	if ((entry[i] & used) != (target[i] & used))
48	differs = true;	55	differs = true;
49	}	56	}
50	return differs;	57	return differs;
@@ -56,12 +63,24 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) · 5 unmodified lines
56	struct arm_smmu_test_writer *test_writer =	63	struct arm_smmu_test_writer *test_writer =
57	container_of(writer, struct arm_smmu_test_writer, writer);	64	container_of(writer, struct arm_smmu_test_writer, writer);
58	__le64 *entry_used_bits;	65	__le64 *entry_used_bits;
		66	__le64 *safe_target;
		67	__le64 *safe_init;
59		68
60	entry_used_bits = kunit_kzalloc(	69	entry_used_bits = kunit_kzalloc(
61	test_writer->test, sizeof(entry_used_bits) NUM_ENTRY_QWORDS,	70	test_writer->test, sizeof(entry_used_bits) NUM_ENTRY_QWORDS,
62	GFP_KERNEL);	71	GFP_KERNEL);
63	KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits);	72	KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits);
64		73
		74	safe_target = kunit_kzalloc(test_writer->test,
		75	sizeof(safe_target) NUM_ENTRY_QWORDS,
		76	GFP_KERNEL);
		77	KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_target);
		78
		79	safe_init = kunit_kzalloc(test_writer->test,
		80	sizeof(safe_init) NUM_ENTRY_QWORDS,
		81	GFP_KERNEL);
		82	KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_init);
		83
65	pr_debug("STE value is now set to: ");	84	pr_debug("STE value is now set to: ");
66	print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8,	85	print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8,
67	test_writer->entry,	86	test_writer->entry,
@@ -79,14 +98,23 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) · 11 unmodified lines
79	* configuration.	98	* configuration.
80	*/	99	*/
81	writer->ops->get_used(test_writer->entry, entry_used_bits);	100	writer->ops->get_used(test_writer->entry, entry_used_bits);
		101	if (writer->ops->get_update_safe)
		102	writer->ops->get_update_safe(test_writer->entry,
		103	test_writer->init_entry,
		104	safe_init);
		105	if (writer->ops->get_update_safe)
		106	writer->ops->get_update_safe(test_writer->entry,
		107	test_writer->target_entry,
		108	safe_target);
82	KUNIT_EXPECT_FALSE(	109	KUNIT_EXPECT_FALSE(
83	test_writer->test,	110	test_writer->test,
84	arm_smmu_entry_differs_in_used_bits(	111	arm_smmu_entry_differs_in_used_bits(
85	test_writer->entry, entry_used_bits,	112	test_writer->entry, entry_used_bits,
86	test_writer->init_entry, NUM_ENTRY_QWORDS) &&	113	test_writer->init_entry, safe_init,
		114	NUM_ENTRY_QWORDS) &&
87	arm_smmu_entry_differs_in_used_bits(	115	arm_smmu_entry_differs_in_used_bits(
88	test_writer->entry, entry_used_bits,	116	test_writer->entry, entry_used_bits,
89	test_writer->target_entry,	117	test_writer->target_entry, safe_target,
90	NUM_ENTRY_QWORDS));	118	NUM_ENTRY_QWORDS));
91	}	119	}
92	}	120	}
@@ -106,6 +134,7 @@ arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, · 13 unmodified lines
106	static const struct arm_smmu_entry_writer_ops test_ste_ops = {	134	static const struct arm_smmu_entry_writer_ops test_ste_ops = {
107	.sync = arm_smmu_test_writer_record_syncs,	135	.sync = arm_smmu_test_writer_record_syncs,
108	.get_used = arm_smmu_get_ste_used,	136	.get_used = arm_smmu_get_ste_used,
		137	.get_update_safe = arm_smmu_get_ste_update_safe,
109	};	138	};
110		139
111	static const struct arm_smmu_entry_writer_ops test_cd_ops = {	140	static const struct arm_smmu_entry_writer_ops test_cd_ops = {
@@ -185,6 +214,18 @@ static void arm_smmu_test_make_cdtable_ste(struct arm_smmu_ste *ste, · 73 unmodified lines
185	};	214	};
186		215
187	arm_smmu_make_cdtable_ste(ste, &master, ats_enabled, s1dss);	216	arm_smmu_make_cdtable_ste(ste, &master, ats_enabled, s1dss);
		217	if (feat & ARM_SMMU_MASTER_TEST_NESTED) {
		218	struct arm_smmu_ste s2ste;
		219	int i;
		220
		221	arm_smmu_test_make_s2_ste(&s2ste,
		222	feat & ~ARM_SMMU_MASTER_TEST_NESTED);
		223	ste->data[0] \|= cpu_to_le64(
		224	FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_NESTED));
		225	ste->data[1] \|= cpu_to_le64(STRTAB_STE_1_MEV);
		226	for (i = 2; i < NUM_ENTRY_QWORDS; i++)
		227	ste->data[i] = s2ste.data[i];
		228	}
188	}	229	}
189		230
190	static void arm_smmu_v3_write_ste_test_bypass_to_abort(struct kunit *test)	231	static void arm_smmu_v3_write_ste_test_bypass_to_abort(struct kunit *test)
@@ -542,6 +583,35 @@ static void arm_smmu_v3_write_ste_test_s2_to_s1_stall(struct kunit *test) · 351 unmodified lines
542	NUM_EXPECTED_SYNCS(3));	583	NUM_EXPECTED_SYNCS(3));
543	}	584	}
544		585
		586	static void
		587	arm_smmu_v3_write_ste_test_nested_s1dssbypass_to_s1bypass(struct kunit *test)
		588	{
		589	struct arm_smmu_ste s1_ste;
		590	struct arm_smmu_ste s2_ste;
		591
		592	arm_smmu_test_make_cdtable_ste(
		593	&s1_ste, STRTAB_STE_1_S1DSS_BYPASS, fake_cdtab_dma_addr,
		594	ARM_SMMU_MASTER_TEST_ATS \| ARM_SMMU_MASTER_TEST_NESTED);
		595	arm_smmu_test_make_s2_ste(&s2_ste, 0);
		596	/* Expect an additional sync to unset ignored bits: EATS and MEV */
		597	arm_smmu_v3_test_ste_expect_hitless_transition(test, &s1_ste, &s2_ste,
		598	NUM_EXPECTED_SYNCS(3));
		599	}
		600
		601	static void
		602	arm_smmu_v3_write_ste_test_nested_s1bypass_to_s1dssbypass(struct kunit *test)
		603	{
		604	struct arm_smmu_ste s1_ste;
		605	struct arm_smmu_ste s2_ste;
		606
		607	arm_smmu_test_make_cdtable_ste(
		608	&s1_ste, STRTAB_STE_1_S1DSS_BYPASS, fake_cdtab_dma_addr,
		609	ARM_SMMU_MASTER_TEST_ATS \| ARM_SMMU_MASTER_TEST_NESTED);
		610	arm_smmu_test_make_s2_ste(&s2_ste, 0);
		611	arm_smmu_v3_test_ste_expect_hitless_transition(test, &s2_ste, &s1_ste,
		612	NUM_EXPECTED_SYNCS(2));
		613	}
		614
545	static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test)	615	static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test)
546	{	616	{
547	struct arm_smmu_cd cd = {};	617	struct arm_smmu_cd cd = {};
@@ -588,6 +658,8 @@ static struct kunit_case arm_smmu_v3_test_cases[] = { · 40 unmodified lines
588	KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_change_asid),	658	KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_change_asid),
589	KUNIT_CASE(arm_smmu_v3_write_ste_test_s1_to_s2_stall),	659	KUNIT_CASE(arm_smmu_v3_write_ste_test_s1_to_s2_stall),
590	KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_s1_stall),	660	KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_s1_stall),
		661	KUNIT_CASE(arm_smmu_v3_write_ste_test_nested_s1dssbypass_to_s1bypass),
		662	KUNIT_CASE(arm_smmu_v3_write_ste_test_nested_s1bypass_to_s1dssbypass),
591	KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear),	663	KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear),
592	KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release),	664	KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release),
593	{},	665	{},

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c+120−33

  */
 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
 {
-	int val;
 	/*
-	 * We can try to avoid the cmpxchg() loop by simply incrementing the
+	 * When held in exclusive state, the lock counter is set to INT_MIN
-	 * lock counter. When held in exclusive state, the lock counter is set
+	 * so these increments won't hurt as the value will remain negative.
-	 * to INT_MIN so these increments won't hurt as the value will remain
+	 * The increment will also signal the exclusive locker that there are
-	 * negative.
+	 * shared waiters.
 	 */
 	if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
 		return;
-	do {
+	/*
-		val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
+	 * Someone else is holding the lock in exclusive state, so wait
-	} while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
+	 * for them to finish. Since we already incremented the lock counter,
+	 * no exclusive lock can be acquired until we finish. We don't need
+	 * the return value since we only care that the exclusive lock is
+	 * released (i.e. the lock counter is non-negative).
+	 * Once the exclusive locker releases the lock, the sign bit will
+	 * be cleared and our increment will make the lock counter positive,
+	 * allowing us to proceed.
+	 */
+	atomic_cond_read_relaxed(&cmdq->lock, VAL > 0);
 }
 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
 	__ret;								\
 })
+/*
+ * Only clear the sign bit when releasing the exclusive lock this will
+ * allow any shared_lock() waiters to proceed without the possibility
+ * of entering the exclusive lock in a tight loop.
+ */
 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags)		\
 ({									\
-	atomic_set_release(&cmdq->lock, 0);				\
+	atomic_fetch_andnot_release(INT_MIN, &cmdq->lock);		\
 	local_irq_restore(flags);					\
 })
 }
 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used);
+VISIBLE_IF_KUNIT
+void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
+				  __le64 *safe_bits)
+{
+	const __le64 eats_s1chk =
+		FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK);
+	const __le64 eats_trans =
+		FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS);
+	/*
+	 * When an STE changes EATS_TRANS, the sequencing code in the attach
+	 * logic already will have the PCI cap for ATS disabled. Thus at this
+	 * moment we can expect that the device will not generate ATS queries
+	 * and so we don't care about the sequencing of EATS. The purpose of
+	 * EATS_TRANS is to protect the system from hostile untrusted devices
+	 * that issue ATS when the PCI config space is disabled. However, if
+	 * EATS_TRANS is being changed, then we must have already trusted the
+	 * device as the EATS_TRANS security block is being disabled.
+	 *
+	 *  Note: now the EATS_TRANS update is moved to the first entry_set().
+	 *  Changing S2S and EATS might transiently result in S2S=1 and EATS=1
+	 *  which is a bad STE (see "5.2 Stream Table Entry"). In such a case,
+	 *  we can't do a hitless update. Also, it should not be added to the
+	 *  safe bits with STRTAB_STE_1_EATS_S1CHK, because EATS=0b11 would be
+	 *  effectively an errant 0b00 configuration.
+	 */
+	if (!((cur[1] | target[1]) & cpu_to_le64(eats_s1chk)) &&
+	    !((cur[2] | target[2]) & cpu_to_le64(STRTAB_STE_2_S2S)))
+		safe_bits[1] |= cpu_to_le64(eats_trans);
+	/*
+	 * MEV does not meaningfully impact the operation of the HW, it only
+	 * changes how many fault events are generated, thus we can relax it
+	 * when computing the ordering. The spec notes the device can act like
+	 * MEV=1 anyhow:
+	 *
+	 *  Note: Software must expect, and be able to deal with, coalesced
+	 *  fault records even when MEV == 0.
+	 */
+	safe_bits[1] |= cpu_to_le64(STRTAB_STE_1_MEV);
+}
+EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_update_safe);
 /*
  * Figure out if we can do a hitless update of entry to become target. Returns a
  * bit mask where 1 indicates that qword needs to be set disruptively.
 {
 	__le64 target_used[NUM_ENTRY_QWORDS] = {};
 	__le64 cur_used[NUM_ENTRY_QWORDS] = {};
+	__le64 safe[NUM_ENTRY_QWORDS] = {};
 	u8 used_qword_diff = 0;
 	unsigned int i;
 	writer->ops->get_used(entry, cur_used);
 	writer->ops->get_used(target, target_used);
+	if (writer->ops->get_update_safe)
+		writer->ops->get_update_safe(entry, target, safe);
 	for (i = 0; i != NUM_ENTRY_QWORDS; i++) {
+		/*
+		 * Safe is only used for bits that are used by both entries,
+		 * otherwise it is sequenced according to the unused entry.
+		 */
+		safe[i] &= target_used[i] & cur_used[i];
 		/*
 		 * Check that masks are up to date, the make functions are not
 		 * allowed to set a bit to 1 if the used function doesn't say it
 		WARN_ON_ONCE(target[i] & ~target_used[i]);
 		/* Bits can change because they are not currently being used */
+		cur_used[i] &= ~safe[i];
 		unused_update[i] = (entry[i] & cur_used[i]) |
 				   (target[i] & ~cur_used[i]);
 		/*
 	return used_qword_diff;
 }
-static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
+static void entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
 		      const __le64 *target, unsigned int start,
 		      unsigned int len)
 {
 	if (changed)
 		writer->ops->sync(writer);
-	return changed;
 }
 /*
 		entry_set(writer, entry, target, 0, 1);
 	} else {
 		/*
-		 * No inuse bit changed. Sanity check that all unused bits are 0
+		 * No inuse bit changed, though safe bits may have changed.
-		 * in the entry. The target was already sanity checked by
-		 * compute_qword_diff().
 		 */
-		WARN_ON_ONCE(
+		entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
-			entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS));
 	}
 }
 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry);
 static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
 	.sync = arm_smmu_ste_writer_sync_entry,
 	.get_used = arm_smmu_get_ste_used,
+	.get_update_safe = arm_smmu_get_ste_update_safe,
 };
 static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
 				     ARM_SMMU_FEAT_VAX) ? 52 : 48;
 		pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS);
-		pgtbl_cfg.oas = smmu->ias;
+		pgtbl_cfg.oas = smmu->oas;
 		if (enable_dirty)
 			pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD;
 		fmt = ARM_64_LPAE_S1;
 	case ARM_SMMU_DOMAIN_S2:
 		if (enable_dirty)
 			return -EOPNOTSUPP;
-		pgtbl_cfg.ias = smmu->ias;
+		pgtbl_cfg.ias = smmu->oas;
 		pgtbl_cfg.oas = smmu->oas;
 		fmt = ARM_64_LPAE_S2;
 		finalise_stage_fn = arm_smmu_domain_finalise_s2;
 		       struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
 		       struct arm_smmu_cd *cd, struct iommu_domain *old)
 {
-	struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
+	struct iommu_domain *sid_domain =
+		iommu_driver_get_domain_for_dev(master->dev);
 	struct arm_smmu_attach_state state = {
 		.master = master,
 		.ssid = pasid,
 	 */
 	if (!arm_smmu_ssids_in_use(&master->cd_table)) {
 		struct iommu_domain *sid_domain =
-			iommu_get_domain_for_dev(master->dev);
+			iommu_driver_get_domain_for_dev(master->dev);
 		if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
 		    sid_domain->type == IOMMU_DOMAIN_BLOCKED)
 	}
 	/* We only support the AArch64 table format at present */
-	switch (FIELD_GET(IDR0_TTF, reg)) {
+	if (!(FIELD_GET(IDR0_TTF, reg) & IDR0_TTF_AARCH64)) {
-	case IDR0_TTF_AARCH32_64:
-		smmu->ias = 40;
-		fallthrough;
-	case IDR0_TTF_AARCH64:
-		break;
-	default:
 		dev_err(smmu->dev, "AArch64 table format not supported!\n");
 		return -ENXIO;
 	}
 		dev_warn(smmu->dev,
 			 "failed to set DMA mask for table walker\n");
-	smmu->ias = max(smmu->ias, smmu->oas);
 	if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
 	    (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 		smmu->features |= ARM_SMMU_FEAT_NESTING;
 	if (arm_smmu_sva_supported(smmu))
 		smmu->features |= ARM_SMMU_FEAT_SVA;
-	dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
+	dev_info(smmu->dev, "oas %lu-bit (features 0x%08x)\n",
-		 smmu->ias, smmu->oas, smmu->features);
+		 smmu->oas, smmu->features);
 	return 0;
 }
+#ifdef CONFIG_TEGRA241_CMDQV
+static void tegra_cmdqv_dt_probe(struct device_node *smmu_node,
+				 struct arm_smmu_device *smmu)
+{
+	struct platform_device *pdev;
+	struct device_node *np;
+	np = of_parse_phandle(smmu_node, "nvidia,cmdqv", 0);
+	if (!np)
+		return;
+	/* Tegra241 CMDQV driver is responsible for put_device() */
+	pdev = of_find_device_by_node(np);
+	of_node_put(np);
+	if (!pdev)
+		return;
+	smmu->impl_dev = &pdev->dev;
+	smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
+	dev_dbg(smmu->dev, "found companion CMDQV device: %s\n",
+		dev_name(smmu->impl_dev));
+}
+#else
+static void tegra_cmdqv_dt_probe(struct device_node *smmu_node,
+				 struct arm_smmu_device *smmu)
+{
+}
+#endif
 #ifdef CONFIG_ACPI
 #ifdef CONFIG_TEGRA241_CMDQV
 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
 	adev = acpi_dev_get_first_match_dev("NVDA200C", uid, -1);
 	if (adev) {
 		/* Tegra241 CMDQV driver is responsible for put_device() */
-		smmu->impl_dev = &adev->dev;
+		smmu->impl_dev = get_device(acpi_get_first_physical_node(adev));
 		smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
 		dev_info(smmu->dev, "found companion CMDQV device: %s\n",
 			 dev_name(smmu->impl_dev));
+		acpi_dev_put(adev);
 	}
 	kfree(uid);
 }
 	if (of_dma_is_coherent(dev->of_node))
 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
+	if (of_device_is_compatible(dev->of_node, "nvidia,tegra264-smmu"))
+		tegra_cmdqv_dt_probe(dev->of_node, smmu);
 	return ret;
 }

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h+4−2

 #define IDR0_COHACC			(1 << 4)
 #define IDR0_TTF			GENMASK(3, 2)
 #define IDR0_TTF_AARCH64		2
-#define IDR0_TTF_AARCH32_64		3
 #define IDR0_S1P			(1 << 1)
 #define IDR0_S2P			(1 << 0)
 	int				gerr_irq;
 	int				combined_irq;
-	unsigned long			ias; /* IPA */
 	unsigned long			oas; /* PA */
 	unsigned long			pgsize_bitmap;
 struct arm_smmu_entry_writer_ops {
 	void (*get_used)(const __le64 *entry, __le64 *used);
+	void (*get_update_safe)(const __le64 *cur, const __le64 *target,
+				__le64 *safe_bits);
 	void (*sync)(struct arm_smmu_entry_writer *writer);
 };
 #if IS_ENABLED(CONFIG_KUNIT)
 void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits);
+void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
+				  __le64 *safe_bits);
 void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur,
 			  const __le64 *target);
 void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits);

drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c+12−72

@@ -3,17 +3,15 @@ · 2 unmodified lines
3		3
4	#define dev_fmt(fmt) "tegra241_cmdqv: " fmt	4	#define dev_fmt(fmt) "tegra241_cmdqv: " fmt
5		5
6	#include <linux/acpi.h>
7	#include <linux/debugfs.h>	6	#include <linux/debugfs.h>
8	#include <linux/dma-mapping.h>	7	#include <linux/dma-mapping.h>
9	#include <linux/interrupt.h>	8	#include <linux/interrupt.h>
10	#include <linux/iommu.h>	9	#include <linux/iommu.h>
11	#include <linux/iommufd.h>	10	#include <linux/iommufd.h>
12	#include <linux/iopoll.h>	11	#include <linux/iopoll.h>
		12	#include <linux/platform_device.h>
13	#include <uapi/linux/iommufd.h>	13	#include <uapi/linux/iommufd.h>
14		14
15	#include <acpi/acpixf.h>
16
17	#include "arm-smmu-v3.h"	15	#include "arm-smmu-v3.h"
18		16
19	/* CMDQV register page base and size defines */	17	/* CMDQV register page base and size defines */
@@ -854,69 +852,6 @@ static struct arm_smmu_impl_ops tegra241_cmdqv_impl_ops = { · 834 unmodified lines
854		852
855	/* Probe Functions */	853	/* Probe Functions */
856		854
857	static int tegra241_cmdqv_acpi_is_memory(struct acpi_resource res, void data)
858	{
859	struct resource_win win;
860
861	return !acpi_dev_resource_address_space(res, &win);
862	}
863
864	static int tegra241_cmdqv_acpi_get_irqs(struct acpi_resource ares, void data)
865	{
866	struct resource r;
867	int *irq = data;
868
869	if (*irq <= 0 && acpi_dev_resource_interrupt(ares, 0, &r))
870	*irq = r.start;
871	return 1; /* No need to add resource to the list */
872	}
873
874	static struct resource *
875	tegra241_cmdqv_find_acpi_resource(struct device dev, int irq)
876	{
877	struct acpi_device *adev = to_acpi_device(dev);
878	struct list_head resource_list;
879	struct resource_entry *rentry;
880	struct resource *res = NULL;
881	int ret;
882
883	INIT_LIST_HEAD(&resource_list);
884	ret = acpi_dev_get_resources(adev, &resource_list,
885	tegra241_cmdqv_acpi_is_memory, NULL);
886	if (ret < 0) {
887	dev_err(dev, "failed to get memory resource: %d\n", ret);
888	return NULL;
889	}
890
891	rentry = list_first_entry_or_null(&resource_list,
892	struct resource_entry, node);
893	if (!rentry) {
894	dev_err(dev, "failed to get memory resource entry\n");
895	goto free_list;
896	}
897
898	/* Caller must free the res */
899	res = kzalloc(sizeof(*res), GFP_KERNEL);
900	if (!res)
901	goto free_list;
902
903	res = rentry->res;
904
905	acpi_dev_free_resource_list(&resource_list);
906
907	INIT_LIST_HEAD(&resource_list);
908
909	if (irq)
910	ret = acpi_dev_get_resources(adev, &resource_list,
911	tegra241_cmdqv_acpi_get_irqs, irq);
912	if (ret < 0 \|\| !irq \|\| *irq <= 0)
913	dev_warn(dev, "no interrupt. errors will not be reported\n");
914
915	free_list:
916	acpi_dev_free_resource_list(&resource_list);
917	return res;
918	}
919
920	static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)	855	static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)
921	{	856	{
922	struct tegra241_cmdqv *cmdqv =	857	struct tegra241_cmdqv *cmdqv =
@@ -1042,18 +977,23 @@ iounmap: · 119 unmodified lines
1042		977
1043	struct arm_smmu_device tegra241_cmdqv_probe(struct arm_smmu_device smmu)	978	struct arm_smmu_device tegra241_cmdqv_probe(struct arm_smmu_device smmu)
1044	{	979	{
		980	struct platform_device *pdev = to_platform_device(smmu->impl_dev);
1045	struct arm_smmu_device *new_smmu;	981	struct arm_smmu_device *new_smmu;
1046	struct resource *res = NULL;	982	struct resource *res;
1047	int irq;	983	int irq;
1048		984
1049	if (!smmu->dev->of_node)	985	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1050	res = tegra241_cmdqv_find_acpi_resource(smmu->impl_dev, &irq);	986	if (!res) {
1051	if (!res)	987	dev_err(&pdev->dev, "no memory resource found for CMDQV\n");
1052	goto out_fallback;	988	goto out_fallback;
		989	}
1053		990
1054	new_smmu = __tegra241_cmdqv_probe(smmu, res, irq);	991	irq = platform_get_irq_optional(pdev, 0);
1055	kfree(res);	992	if (irq <= 0)
		993	dev_warn(&pdev->dev,
		994	"no interrupt. errors will not be reported\n");
1056		995
		996	new_smmu = __tegra241_cmdqv_probe(smmu, res, irq);
1057	if (new_smmu)	997	if (new_smmu)
1058	return new_smmu;	998	return new_smmu;
1059		999

drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c+28−0

@@ -41,12 +41,40 @@ static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { · 40 unmodified lines
41	.data = (const void *) (PREFETCH_DEEP \| CPRE \| CMTLB) },	41	.data = (const void *) (PREFETCH_DEEP \| CPRE \| CMTLB) },
42	{ .compatible = "qcom,fastrpc",	42	{ .compatible = "qcom,fastrpc",
43	.data = (const void *) (PREFETCH_DEEP \| CPRE \| CMTLB) },	43	.data = (const void *) (PREFETCH_DEEP \| CPRE \| CMTLB) },
		44	{ .compatible = "qcom,qcm2290-mdss",
		45	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		46	{ .compatible = "qcom,sa8775p-mdss",
		47	.data = (const void *) (PREFETCH_DEFAULT \| CMTLB) },
44	{ .compatible = "qcom,sc7280-mdss",	48	{ .compatible = "qcom,sc7280-mdss",
45	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },	49	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
46	{ .compatible = "qcom,sc7280-venus",	50	{ .compatible = "qcom,sc7280-venus",
47	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },	51	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		52	{ .compatible = "qcom,sc8180x-mdss",
		53	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		54	{ .compatible = "qcom,sc8280xp-mdss",
		55	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		56	{ .compatible = "qcom,sm6115-mdss",
		57	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		58	{ .compatible = "qcom,sm6125-mdss",
		59	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		60	{ .compatible = "qcom,sm6350-mdss",
		61	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		62	{ .compatible = "qcom,sm8150-mdss",
		63	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		64	{ .compatible = "qcom,sm8250-mdss",
		65	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		66	{ .compatible = "qcom,sm8350-mdss",
		67	.data = (const void *) (PREFETCH_SHALLOW \| CPRE \| CMTLB) },
		68	{ .compatible = "qcom,sm8450-mdss",
		69	.data = (const void *) (PREFETCH_DEFAULT \| CMTLB) },
48	{ .compatible = "qcom,sm8550-mdss",	70	{ .compatible = "qcom,sm8550-mdss",
49	.data = (const void *) (PREFETCH_DEFAULT \| CMTLB) },	71	.data = (const void *) (PREFETCH_DEFAULT \| CMTLB) },
		72	{ .compatible = "qcom,sm8650-mdss",
		73	.data = (const void *) (PREFETCH_DEFAULT \| CMTLB) },
		74	{ .compatible = "qcom,sm8750-mdss",
		75	.data = (const void *) (PREFETCH_DEFAULT \| CMTLB) },
		76	{ .compatible = "qcom,x1e80100-mdss",
		77	.data = (const void *) (PREFETCH_DEFAULT \| CMTLB) },
50	{ }	78	{ }
51	};	79	};
52		80

drivers/iommu/arm/arm-smmu/qcom_iommu.c+2−6

 static bool qcom_iommu_has_secure_context(struct qcom_iommu_dev *qcom_iommu)
 {
-	struct device_node *child;
+	for_each_child_of_node_scoped(qcom_iommu->dev->of_node, child) {
-	for_each_child_of_node(qcom_iommu->dev->of_node, child) {
 		if (of_device_is_compatible(child, "qcom,msm-iommu-v1-sec") ||
-		    of_device_is_compatible(child, "qcom,msm-iommu-v2-sec")) {
+		    of_device_is_compatible(child, "qcom,msm-iommu-v2-sec"))
-			of_node_put(child);
 			return true;
-		}
 	}
 	return false;

drivers/iommu/dma-iommu.c+1−3

@@ -2097,10 +2097,8 @@ void dma_iova_destroy(struct device dev, struct dma_iova_state state, · 2096 unmodified lines
2097	}	2097	}
2098	EXPORT_SYMBOL_GPL(dma_iova_destroy);	2098	EXPORT_SYMBOL_GPL(dma_iova_destroy);
2099		2099
2100	void iommu_setup_dma_ops(struct device *dev)	2100	void iommu_setup_dma_ops(struct device dev, struct iommu_domain domain)
2101	{	2101	{
2102	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
2103
2104	if (dev_is_pci(dev))	2102	if (dev_is_pci(dev))
2105	dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac;	2103	dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac;
2106		2104

drivers/iommu/dma-iommu.h+3−2

 #ifdef CONFIG_IOMMU_DMA
-void iommu_setup_dma_ops(struct device *dev);
+void iommu_setup_dma_ops(struct device *dev, struct iommu_domain *domain);
 int iommu_get_dma_cookie(struct iommu_domain *domain);
 void iommu_put_dma_cookie(struct iommu_domain *domain);
 #else /* CONFIG_IOMMU_DMA */
-static inline void iommu_setup_dma_ops(struct device *dev)
+static inline void iommu_setup_dma_ops(struct device *dev,
+				       struct iommu_domain *domain)
 {
 }

drivers/iommu/generic_pt/fmt/amdv1.h+2−1

@@ -354,7 +354,8 @@ static inline int amdv1pt_iommu_set_prot(struct pt_common *common, · 353 unmodified lines
354	* Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to	354	* Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to
355	* control this. For now if the tables use sme_set then so do the ptes.	355	* control this. For now if the tables use sme_set then so do the ptes.
356	*/	356	*/
357	if (pt_feature(common, PT_FEAT_AMDV1_ENCRYPT_TABLES))	357	if (pt_feature(common, PT_FEAT_AMDV1_ENCRYPT_TABLES) &&
		358	!(iommu_prot & IOMMU_MMIO))
358	pte = __sme_set(pte);	359	pte = __sme_set(pte);
359		360
360	attrs->descriptor_bits = pte;	361	attrs->descriptor_bits = pte;

drivers/iommu/generic_pt/fmt/x86_64.h+2−1

@@ -227,7 +227,8 @@ static inline int x86_64_pt_iommu_set_prot(struct pt_common *common, · 226 unmodified lines
227	* Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to	227	* Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to
228	* control this. For now if the tables use sme_set then so do the ptes.	228	* control this. For now if the tables use sme_set then so do the ptes.
229	*/	229	*/
230	if (pt_feature(common, PT_FEAT_X86_64_AMD_ENCRYPT_TABLES))	230	if (pt_feature(common, PT_FEAT_X86_64_AMD_ENCRYPT_TABLES) &&
		231	!(iommu_prot & IOMMU_MMIO))
231	pte = __sme_set(pte);	232	pte = __sme_set(pte);
232		233
233	attrs->descriptor_bits = pte;	234	attrs->descriptor_bits = pte;

drivers/iommu/generic_pt/iommu_pt.h+1−2

 		 * Note that the sync frees the gather's free list, so we must
 		 * not have any pages on that list that are covered by iova/len
 		 */
-	} else if (pt_feature(common, PT_FEAT_FLUSH_RANGE)) {
-		iommu_iotlb_gather_add_range(iotlb_gather, iova, len);
 	}
+	iommu_iotlb_gather_add_range(iotlb_gather, iova, len);
 	iommu_pages_list_splice(free_list, &iotlb_gather->freelist);
 }

drivers/iommu/intel/cache.c+8−1

 	qi_batch_increment_index(iommu, batch);
 }
+static bool intel_domain_use_piotlb(struct dmar_domain *domain)
+{
+	return domain->domain.type == IOMMU_DOMAIN_SVA ||
+			domain->domain.type == IOMMU_DOMAIN_NESTED ||
+			intel_domain_is_fs_paging(domain);
+}
 static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *tag,
 				  unsigned long addr, unsigned long pages,
 				  unsigned long mask, int ih)
 	struct intel_iommu *iommu = tag->iommu;
 	u64 type = DMA_TLB_PSI_FLUSH;
-	if (intel_domain_is_fs_paging(domain)) {
+	if (intel_domain_use_piotlb(domain)) {
 		qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr,
 				    pages, ih, domain->qi_batch);
 		return;

drivers/iommu/intel/iommu.c+15−18

@@ -1240,22 +1240,22 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 · 1239 unmodified lines
1240	}	1240	}
1241		1241
1242	did = context_domain_id(context);	1242	did = context_domain_id(context);
1243	context_clear_entry(context);	1243	context_clear_present(context);
1244	__iommu_flush_cache(iommu, context, sizeof(*context));	1244	__iommu_flush_cache(iommu, context, sizeof(*context));
1245	spin_unlock(&iommu->lock);	1245	spin_unlock(&iommu->lock);
1246	intel_context_flush_no_pasid(info, context, did);	1246	intel_context_flush_no_pasid(info, context, did);
		1247	context_clear_entry(context);
		1248	__iommu_flush_cache(iommu, context, sizeof(*context));
1247	}	1249	}
1248		1250
1249	int __domain_setup_first_level(struct intel_iommu iommu, struct device dev,	1251	int __domain_setup_first_level(struct intel_iommu iommu, struct device dev,
1250	ioasid_t pasid, u16 did, phys_addr_t fsptptr,	1252	ioasid_t pasid, u16 did, phys_addr_t fsptptr,
1251	int flags, struct iommu_domain *old)	1253	int flags, struct iommu_domain *old)
1252	{	1254	{
1253	if (!old)	1255	if (old)
1254	return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid,	1256	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
1255	did, flags);	1257
1256	return intel_pasid_replace_first_level(iommu, dev, fsptptr, pasid, did,	1258	return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, did, flags);
1257	iommu_domain_did(old, iommu),
1258	flags);
1259	}	1259	}
1260		1260
1261	static int domain_setup_second_level(struct intel_iommu *iommu,	1261	static int domain_setup_second_level(struct intel_iommu *iommu,
@@ -1263,23 +1263,20 @@ static int domain_setup_second_level(struct intel_iommu *iommu, · 1 unmodified line
1263	struct device *dev, ioasid_t pasid,	1263	struct device *dev, ioasid_t pasid,
1264	struct iommu_domain *old)	1264	struct iommu_domain *old)
1265	{	1265	{
1266	if (!old)	1266	if (old)
1267	return intel_pasid_setup_second_level(iommu, domain,	1267	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
1268	dev, pasid);	1268
1269	return intel_pasid_replace_second_level(iommu, domain, dev,	1269	return intel_pasid_setup_second_level(iommu, domain, dev, pasid);
1270	iommu_domain_did(old, iommu),
1271	pasid);
1272	}	1270	}
1273		1271
1274	static int domain_setup_passthrough(struct intel_iommu *iommu,	1272	static int domain_setup_passthrough(struct intel_iommu *iommu,
1275	struct device *dev, ioasid_t pasid,	1273	struct device *dev, ioasid_t pasid,
1276	struct iommu_domain *old)	1274	struct iommu_domain *old)
1277	{	1275	{
1278	if (!old)	1276	if (old)
1279	return intel_pasid_setup_pass_through(iommu, dev, pasid);	1277	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
1280	return intel_pasid_replace_pass_through(iommu, dev,	1278
1281	iommu_domain_did(old, iommu),	1279	return intel_pasid_setup_pass_through(iommu, dev, pasid);
1282	pasid);
1283	}	1280	}
1284		1281
1285	static int domain_setup_first_level(struct intel_iommu *iommu,	1282	static int domain_setup_first_level(struct intel_iommu *iommu,

drivers/iommu/intel/iommu.h+20−1

@@ -900,7 +900,26 @@ static inline int pfn_level_offset(u64 pfn, int level) · 899 unmodified lines
900		900
901	static inline void context_set_present(struct context_entry *context)	901	static inline void context_set_present(struct context_entry *context)
902	{	902	{
903	context->lo \|= 1;	903	u64 val;
		904
		905	dma_wmb();
		906	val = READ_ONCE(context->lo) \| 1;
		907	WRITE_ONCE(context->lo, val);
		908	}
		909
		910	/*
		911	* Clear the Present (P) bit (bit 0) of a context table entry. This initiates
		912	* the transition of the entry's ownership from hardware to software. The
		913	* caller is responsible for fulfilling the invalidation handshake recommended
		914	* by the VT-d spec, Section 6.5.3.3 (Guidance to Software for Invalidations).
		915	*/
		916	static inline void context_clear_present(struct context_entry *context)
		917	{
		918	u64 val;
		919
		920	val = READ_ONCE(context->lo) & GENMASK_ULL(63, 1);
		921	WRITE_ONCE(context->lo, val);
		922	dma_wmb();
904	}	923	}
905		924
906	static inline void context_set_fault_enable(struct context_entry *context)	925	static inline void context_set_fault_enable(struct context_entry *context)

drivers/iommu/intel/nested.c+4−5

@@ -136,11 +136,10 @@ static int domain_setup_nested(struct intel_iommu *iommu, · 135 unmodified lines
136	struct device *dev, ioasid_t pasid,	136	struct device *dev, ioasid_t pasid,
137	struct iommu_domain *old)	137	struct iommu_domain *old)
138	{	138	{
139	if (!old)	139	if (old)
140	return intel_pasid_setup_nested(iommu, dev, pasid, domain);	140	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
141	return intel_pasid_replace_nested(iommu, dev, pasid,	141
142	iommu_domain_did(old, iommu),	142	return intel_pasid_setup_nested(iommu, dev, pasid, domain);
143	domain);
144	}	143	}
145		144
146	static int intel_nested_set_dev_pasid(struct iommu_domain *domain,	145	static int intel_nested_set_dev_pasid(struct iommu_domain *domain,

drivers/iommu/intel/pasid.c+22−190

@@ -153,6 +153,9 @@ retry: · 152 unmodified lines
153	if (!entries)	153	if (!entries)
154	return NULL;	154	return NULL;
155		155
		156	if (!ecap_coherent(info->iommu->ecap))
		157	clflush_cache_range(entries, VTD_PAGE_SIZE);
		158
156	/*	159	/*
157	* The pasid directory table entry won't be freed after	160	* The pasid directory table entry won't be freed after
158	* allocation. No worry about the race with free and	161	* allocation. No worry about the race with free and
@@ -165,10 +168,8 @@ retry: · 6 unmodified lines
165	iommu_free_pages(entries);	168	iommu_free_pages(entries);
166	goto retry;	169	goto retry;
167	}	170	}
168	if (!ecap_coherent(info->iommu->ecap)) {	171	if (!ecap_coherent(info->iommu->ecap))
169	clflush_cache_range(entries, VTD_PAGE_SIZE);
170	clflush_cache_range(&dir[dir_index].val, sizeof(*dir));	172	clflush_cache_range(&dir[dir_index].val, sizeof(*dir));
171	}
172	}	173	}
173		174
174	return &entries[index];	175	return &entries[index];
@@ -218,7 +219,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, · 43 unmodified lines
218	if (!info \|\| !info->ats_enabled)	219	if (!info \|\| !info->ats_enabled)
219	return;	220	return;
220		221
221	if (pci_dev_is_disconnected(to_pci_dev(dev)))	222	if (!pci_device_is_present(to_pci_dev(dev)))
222	return;	223	return;
223		224
224	sid = PCI_DEVID(info->bus, info->devfn);	225	sid = PCI_DEVID(info->bus, info->devfn);
@@ -272,7 +273,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu iommu, struct device dev, · 47 unmodified lines
272		273
273	did = pasid_get_domain_id(pte);	274	did = pasid_get_domain_id(pte);
274	pgtt = pasid_pte_get_pgtt(pte);	275	pgtt = pasid_pte_get_pgtt(pte);
275	intel_pasid_clear_entry(dev, pasid, fault_ignore);	276	pasid_clear_present(pte);
276	spin_unlock(&iommu->lock);	277	spin_unlock(&iommu->lock);
277		278
278	if (!ecap_coherent(iommu->ecap))	279	if (!ecap_coherent(iommu->ecap))
@@ -286,6 +287,10 @@ void intel_pasid_tear_down_entry(struct intel_iommu iommu, struct device dev, · 7 unmodified lines
286	iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);	287	iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
287		288
288	devtlb_invalidation_with_pasid(iommu, dev, pasid);	289	devtlb_invalidation_with_pasid(iommu, dev, pasid);
		290	intel_pasid_clear_entry(dev, pasid, fault_ignore);
		291	if (!ecap_coherent(iommu->ecap))
		292	clflush_cache_range(pte, sizeof(*pte));
		293
289	if (!fault_ignore)	294	if (!fault_ignore)
290	intel_iommu_drain_pasid_prq(dev, pasid);	295	intel_iommu_drain_pasid_prq(dev, pasid);
291	}	296	}
@@ -412,50 +417,6 @@ int intel_pasid_setup_first_level(struct intel_iommu iommu, struct device dev, · 120 unmodified lines
412	return 0;	417	return 0;
413	}	418	}
414		419
415	int intel_pasid_replace_first_level(struct intel_iommu *iommu,
416	struct device *dev, phys_addr_t fsptptr,
417	u32 pasid, u16 did, u16 old_did,
418	int flags)
419	{
420	struct pasid_entry *pte, new_pte;
421
422	if (!ecap_flts(iommu->ecap)) {
423	pr_err("No first level translation support on %s\n",
424	iommu->name);
425	return -EINVAL;
426	}
427
428	if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) {
429	pr_err("No 5-level paging support for first-level on %s\n",
430	iommu->name);
431	return -EINVAL;
432	}
433
434	pasid_pte_config_first_level(iommu, &new_pte, fsptptr, did, flags);
435
436	spin_lock(&iommu->lock);
437	pte = intel_pasid_get_entry(dev, pasid);
438	if (!pte) {
439	spin_unlock(&iommu->lock);
440	return -ENODEV;
441	}
442
443	if (!pasid_pte_is_present(pte)) {
444	spin_unlock(&iommu->lock);
445	return -EINVAL;
446	}
447
448	WARN_ON(old_did != pasid_get_domain_id(pte));
449
450	*pte = new_pte;
451	spin_unlock(&iommu->lock);
452
453	intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
454	intel_iommu_drain_pasid_prq(dev, pasid);
455
456	return 0;
457	}
458
459	/*	420	/*
460	* Set up the scalable mode pasid entry for second only translation type.	421	* Set up the scalable mode pasid entry for second only translation type.
461	*/	422	*/
@@ -522,51 +483,6 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, · 60 unmodified lines
522	return 0;	483	return 0;
523	}	484	}
524		485
525	int intel_pasid_replace_second_level(struct intel_iommu *iommu,
526	struct dmar_domain *domain,
527	struct device *dev, u16 old_did,
528	u32 pasid)
529	{
530	struct pasid_entry *pte, new_pte;
531	u16 did;
532
533	/*
534	* If hardware advertises no support for second level
535	* translation, return directly.
536	*/
537	if (!ecap_slts(iommu->ecap)) {
538	pr_err("No second level translation support on %s\n",
539	iommu->name);
540	return -EINVAL;
541	}
542
543	did = domain_id_iommu(domain, iommu);
544
545	pasid_pte_config_second_level(iommu, &new_pte, domain, did);
546
547	spin_lock(&iommu->lock);
548	pte = intel_pasid_get_entry(dev, pasid);
549	if (!pte) {
550	spin_unlock(&iommu->lock);
551	return -ENODEV;
552	}
553
554	if (!pasid_pte_is_present(pte)) {
555	spin_unlock(&iommu->lock);
556	return -EINVAL;
557	}
558
559	WARN_ON(old_did != pasid_get_domain_id(pte));
560
561	*pte = new_pte;
562	spin_unlock(&iommu->lock);
563
564	intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
565	intel_iommu_drain_pasid_prq(dev, pasid);
566
567	return 0;
568	}
569
570	/*	486	/*
571	* Set up dirty tracking on a second only or nested translation type.	487	* Set up dirty tracking on a second only or nested translation type.
572	*/	488	*/
@@ -679,38 +595,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, · 106 unmodified lines
679	return 0;	595	return 0;
680	}	596	}
681		597
682	int intel_pasid_replace_pass_through(struct intel_iommu *iommu,
683	struct device *dev, u16 old_did,
684	u32 pasid)
685	{
686	struct pasid_entry *pte, new_pte;
687	u16 did = FLPT_DEFAULT_DID;
688
689	pasid_pte_config_pass_through(iommu, &new_pte, did);
690
691	spin_lock(&iommu->lock);
692	pte = intel_pasid_get_entry(dev, pasid);
693	if (!pte) {
694	spin_unlock(&iommu->lock);
695	return -ENODEV;
696	}
697
698	if (!pasid_pte_is_present(pte)) {
699	spin_unlock(&iommu->lock);
700	return -EINVAL;
701	}
702
703	WARN_ON(old_did != pasid_get_domain_id(pte));
704
705	*pte = new_pte;
706	spin_unlock(&iommu->lock);
707
708	intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
709	intel_iommu_drain_pasid_prq(dev, pasid);
710
711	return 0;
712	}
713
714	/*	598	/*
715	* Set the page snoop control for a pasid entry which has been set up.	599	* Set the page snoop control for a pasid entry which has been set up.
716	*/	600	*/
@@ -844,69 +728,6 @@ int intel_pasid_setup_nested(struct intel_iommu iommu, struct device dev, · 127 unmodified lines
844	return 0;	728	return 0;
845	}	729	}
846		730
847	int intel_pasid_replace_nested(struct intel_iommu *iommu,
848	struct device *dev, u32 pasid,
849	u16 old_did, struct dmar_domain *domain)
850	{
851	struct iommu_hwpt_vtd_s1 *s1_cfg = &domain->s1_cfg;
852	struct dmar_domain *s2_domain = domain->s2_domain;
853	u16 did = domain_id_iommu(domain, iommu);
854	struct pasid_entry *pte, new_pte;
855
856	/* Address width should match the address width supported by hardware */
857	switch (s1_cfg->addr_width) {
858	case ADDR_WIDTH_4LEVEL:
859	break;
860	case ADDR_WIDTH_5LEVEL:
861	if (!cap_fl5lp_support(iommu->cap)) {
862	dev_err_ratelimited(dev,
863	"5-level paging not supported\n");
864	return -EINVAL;
865	}
866	break;
867	default:
868	dev_err_ratelimited(dev, "Invalid stage-1 address width %d\n",
869	s1_cfg->addr_width);
870	return -EINVAL;
871	}
872
873	if ((s1_cfg->flags & IOMMU_VTD_S1_SRE) && !ecap_srs(iommu->ecap)) {
874	pr_err_ratelimited("No supervisor request support on %s\n",
875	iommu->name);
876	return -EINVAL;
877	}
878
879	if ((s1_cfg->flags & IOMMU_VTD_S1_EAFE) && !ecap_eafs(iommu->ecap)) {
880	pr_err_ratelimited("No extended access flag support on %s\n",
881	iommu->name);
882	return -EINVAL;
883	}
884
885	pasid_pte_config_nestd(iommu, &new_pte, s1_cfg, s2_domain, did);
886
887	spin_lock(&iommu->lock);
888	pte = intel_pasid_get_entry(dev, pasid);
889	if (!pte) {
890	spin_unlock(&iommu->lock);
891	return -ENODEV;
892	}
893
894	if (!pasid_pte_is_present(pte)) {
895	spin_unlock(&iommu->lock);
896	return -EINVAL;
897	}
898
899	WARN_ON(old_did != pasid_get_domain_id(pte));
900
901	*pte = new_pte;
902	spin_unlock(&iommu->lock);
903
904	intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
905	intel_iommu_drain_pasid_prq(dev, pasid);
906
907	return 0;
908	}
909
910	/*	731	/*
911	* Interfaces to setup or teardown a pasid table to the scalable-mode	732	* Interfaces to setup or teardown a pasid table to the scalable-mode
912	* context table entry:	733	* context table entry:
@@ -1019,7 +840,7 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) · 106 unmodified lines
1019	}	840	}
1020		841
1021	if (context_copied(iommu, bus, devfn)) {	842	if (context_copied(iommu, bus, devfn)) {
1022	context_clear_entry(context);	843	context_clear_present(context);
1023	__iommu_flush_cache(iommu, context, sizeof(*context));	844	__iommu_flush_cache(iommu, context, sizeof(*context));
1024		845
1025	/*	846	/*
@@ -1039,6 +860,9 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) · 13 unmodified lines
1039	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);	860	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1040	devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID);	861	devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID);
1041		862
		863	context_clear_entry(context);
		864	__iommu_flush_cache(iommu, context, sizeof(*context));
		865
1042	/*	866	/*
1043	* At this point, the device is supposed to finish reset at	867	* At this point, the device is supposed to finish reset at
1044	* its driver probe stage, so no in-flight DMA will exist,	868	* its driver probe stage, so no in-flight DMA will exist,
@@ -1102,6 +926,14 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info) · 57 unmodified lines
1102	if (!info->ats_enabled)	926	if (!info->ats_enabled)
1103	return;	927	return;
1104		928
		929	/*
		930	* Skip dev-IOTLB flush for inaccessible PCIe devices to prevent the
		931	* Intel IOMMU from waiting indefinitely for an ATS invalidation that
		932	* cannot complete.
		933	*/
		934	if (!pci_device_is_present(to_pci_dev(info->dev)))
		935	return;
		936
1105	qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn),	937	qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn),
1106	info->pfsid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH);	938	info->pfsid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH);
1107		939

drivers/iommu/intel/pasid.h+14−14

  */
 static inline void pasid_set_present(struct pasid_entry *pe)
 {
+	dma_wmb();
 	pasid_set_bits(&pe->val[0], 1 << 0, 1);
 }
+/*
+ * Clear the Present (P) bit (bit 0) of a scalable-mode PASID table entry.
+ * This initiates the transition of the entry's ownership from hardware
+ * to software. The caller is responsible for fulfilling the invalidation
+ * handshake recommended by the VT-d spec, Section 6.5.3.3 (Guidance to
+ * Software for Invalidations).
+ */
+static inline void pasid_clear_present(struct pasid_entry *pe)
+{
+	pasid_set_bits(&pe->val[0], 1 << 0, 0);
+	dma_wmb();
+}
 /*
  * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID
  * entry.
 				   struct device *dev, u32 pasid);
 int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
 			     u32 pasid, struct dmar_domain *domain);
-int intel_pasid_replace_first_level(struct intel_iommu *iommu,
-				    struct device *dev, phys_addr_t fsptptr,
-				    u32 pasid, u16 did, u16 old_did, int flags);
-int intel_pasid_replace_second_level(struct intel_iommu *iommu,
-				     struct dmar_domain *domain,
-				     struct device *dev, u16 old_did,
-				     u32 pasid);
-int intel_pasid_replace_pass_through(struct intel_iommu *iommu,
-				     struct device *dev, u16 old_did,
-				     u32 pasid);
-int intel_pasid_replace_nested(struct intel_iommu *iommu,
-			       struct device *dev, u32 pasid,
-			       u16 old_did, struct dmar_domain *domain);
 void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
 				 struct device *dev, u32 pasid,
 				 bool fault_ignore);

drivers/iommu/iommu-debug-pagealloc.c+164−0

@@ -0,0 +1,164 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2025 - Google Inc
	4	* Author: Mostafa Saleh <smostafa@google.com>
	5	* IOMMU API debug page alloc sanitizer
	6	*/
	7	#include <linux/atomic.h>
	8	#include <linux/iommu.h>
	9	#include <linux/iommu-debug-pagealloc.h>
	10	#include <linux/kernel.h>
	11	#include <linux/page_ext.h>
	12	#include <linux/page_owner.h>
	13
	14	#include "iommu-priv.h"
	15
	16	static bool needed;
	17	DEFINE_STATIC_KEY_FALSE(iommu_debug_initialized);
	18
	19	struct iommu_debug_metadata {
	20	atomic_t ref;
	21	};
	22
	23	static __init bool need_iommu_debug(void)
	24	{
	25	return needed;
	26	}
	27
	28	struct page_ext_operations page_iommu_debug_ops = {
	29	.size = sizeof(struct iommu_debug_metadata),
	30	.need = need_iommu_debug,
	31	};
	32
	33	static struct iommu_debug_metadata get_iommu_data(struct page_ext page_ext)
	34	{
	35	return page_ext_data(page_ext, &page_iommu_debug_ops);
	36	}
	37
	38	static void iommu_debug_inc_page(phys_addr_t phys)
	39	{
	40	struct page_ext *page_ext = page_ext_from_phys(phys);
	41	struct iommu_debug_metadata *d;
	42
	43	if (!page_ext)
	44	return;
	45
	46	d = get_iommu_data(page_ext);
	47	WARN_ON(atomic_inc_return_relaxed(&d->ref) <= 0);
	48	page_ext_put(page_ext);
	49	}
	50
	51	static void iommu_debug_dec_page(phys_addr_t phys)
	52	{
	53	struct page_ext *page_ext = page_ext_from_phys(phys);
	54	struct iommu_debug_metadata *d;
	55
	56	if (!page_ext)
	57	return;
	58
	59	d = get_iommu_data(page_ext);
	60	WARN_ON(atomic_dec_return_relaxed(&d->ref) < 0);
	61	page_ext_put(page_ext);
	62	}
	63
	64	/*
	65	* IOMMU page size doesn't have to match the CPU page size. So, we use
	66	* the smallest IOMMU page size to refcount the pages in the vmemmap.
	67	* That is important as both map and unmap has to use the same page size
	68	* to update the refcount to avoid double counting the same page.
	69	* And as we can't know from iommu_unmap() what was the original page size
	70	* used for map, we just use the minimum supported one for both.
	71	*/
	72	static size_t iommu_debug_page_size(struct iommu_domain *domain)
	73	{
	74	return 1UL << __ffs(domain->pgsize_bitmap);
	75	}
	76
	77	static bool iommu_debug_page_count(const struct page *page)
	78	{
	79	unsigned int ref;
	80	struct page_ext *page_ext = page_ext_get(page);
	81	struct iommu_debug_metadata *d = get_iommu_data(page_ext);
	82
	83	ref = atomic_read(&d->ref);
	84	page_ext_put(page_ext);
	85	return ref != 0;
	86	}
	87
	88	void __iommu_debug_check_unmapped(const struct page *page, int numpages)
	89	{
	90	while (numpages--) {
	91	if (WARN_ON(iommu_debug_page_count(page))) {
	92	pr_warn("iommu: Detected page leak!\n");
	93	dump_page_owner(page);
	94	}
	95	page++;
	96	}
	97	}
	98
	99	void __iommu_debug_map(struct iommu_domain *domain, phys_addr_t phys, size_t size)
	100	{
	101	size_t off, end;
	102	size_t page_size = iommu_debug_page_size(domain);
	103
	104	if (WARN_ON(!phys \|\| check_add_overflow(phys, size, &end)))
	105	return;
	106
	107	for (off = 0 ; off < size ; off += page_size)
	108	iommu_debug_inc_page(phys + off);
	109	}
	110
	111	static void __iommu_debug_update_iova(struct iommu_domain *domain,
	112	unsigned long iova, size_t size, bool inc)
	113	{
	114	size_t off, end;
	115	size_t page_size = iommu_debug_page_size(domain);
	116
	117	if (WARN_ON(check_add_overflow(iova, size, &end)))
	118	return;
	119
	120	for (off = 0 ; off < size ; off += page_size) {
	121	phys_addr_t phys = iommu_iova_to_phys(domain, iova + off);
	122
	123	if (!phys)
	124	continue;
	125
	126	if (inc)
	127	iommu_debug_inc_page(phys);
	128	else
	129	iommu_debug_dec_page(phys);
	130	}
	131	}
	132
	133	void __iommu_debug_unmap_begin(struct iommu_domain *domain,
	134	unsigned long iova, size_t size)
	135	{
	136	__iommu_debug_update_iova(domain, iova, size, false);
	137	}
	138
	139	void __iommu_debug_unmap_end(struct iommu_domain *domain,
	140	unsigned long iova, size_t size,
	141	size_t unmapped)
	142	{
	143	if ((unmapped == size) \|\| WARN_ON_ONCE(unmapped > size))
	144	return;
	145
	146	/* If unmap failed, re-increment the refcount. */
	147	__iommu_debug_update_iova(domain, iova + unmapped,
	148	size - unmapped, true);
	149	}
	150
	151	void iommu_debug_init(void)
	152	{
	153	if (!needed)
	154	return;
	155
	156	pr_info("iommu: Debugging page allocations, expect overhead or disable iommu.debug_pagealloc");
	157	static_branch_enable(&iommu_debug_initialized);
	158	}
	159
	160	static int __init iommu_debug_pagealloc(char *str)
	161	{
	162	return kstrtobool(str, &needed);
	163	}
	164	early_param("iommu.debug_pagealloc", iommu_debug_pagealloc);

drivers/iommu/iommu-priv.h+58−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	#define __LINUX_IOMMU_PRIV_H	5	#define __LINUX_IOMMU_PRIV_H
6		6
7	#include <linux/iommu.h>	7	#include <linux/iommu.h>
		8	#include <linux/iommu-debug-pagealloc.h>
8	#include <linux/msi.h>	9	#include <linux/msi.h>
9		10
10	static inline const struct iommu_ops dev_iommu_ops(struct device dev)	11	static inline const struct iommu_ops dev_iommu_ops(struct device dev)
@@ -65,4 +66,61 @@ static inline int iommufd_sw_msi(struct iommu_domain *domain, · 54 unmodified lines
65	int iommu_replace_device_pasid(struct iommu_domain *domain,	66	int iommu_replace_device_pasid(struct iommu_domain *domain,
66	struct device *dev, ioasid_t pasid,	67	struct device *dev, ioasid_t pasid,
67	struct iommu_attach_handle *handle);	68	struct iommu_attach_handle *handle);
		69
		70	#ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC
		71
		72	void __iommu_debug_map(struct iommu_domain *domain, phys_addr_t phys,
		73	size_t size);
		74	void __iommu_debug_unmap_begin(struct iommu_domain *domain,
		75	unsigned long iova, size_t size);
		76	void __iommu_debug_unmap_end(struct iommu_domain *domain,
		77	unsigned long iova, size_t size, size_t unmapped);
		78
		79	static inline void iommu_debug_map(struct iommu_domain *domain,
		80	phys_addr_t phys, size_t size)
		81	{
		82	if (static_branch_unlikely(&iommu_debug_initialized))
		83	__iommu_debug_map(domain, phys, size);
		84	}
		85
		86	static inline void iommu_debug_unmap_begin(struct iommu_domain *domain,
		87	unsigned long iova, size_t size)
		88	{
		89	if (static_branch_unlikely(&iommu_debug_initialized))
		90	__iommu_debug_unmap_begin(domain, iova, size);
		91	}
		92
		93	static inline void iommu_debug_unmap_end(struct iommu_domain *domain,
		94	unsigned long iova, size_t size,
		95	size_t unmapped)
		96	{
		97	if (static_branch_unlikely(&iommu_debug_initialized))
		98	__iommu_debug_unmap_end(domain, iova, size, unmapped);
		99	}
		100
		101	void iommu_debug_init(void);
		102
		103	#else
		104	static inline void iommu_debug_map(struct iommu_domain *domain,
		105	phys_addr_t phys, size_t size)
		106	{
		107	}
		108
		109	static inline void iommu_debug_unmap_begin(struct iommu_domain *domain,
		110	unsigned long iova, size_t size)
		111	{
		112	}
		113
		114	static inline void iommu_debug_unmap_end(struct iommu_domain *domain,
		115	unsigned long iova, size_t size,
		116	size_t unmapped)
		117	{
		118	}
		119
		120	static inline void iommu_debug_init(void)
		121	{
		122	}
		123
		124	#endif /* CONFIG_IOMMU_DEBUG_PAGEALLOC */
		125
68	#endif /* __LINUX_IOMMU_PRIV_H */	126	#endif /* __LINUX_IOMMU_PRIV_H */

drivers/iommu/iommu.c+224−10

@@ -61,6 +61,11 @@ struct iommu_group { · 60 unmodified lines
61	int id;	61	int id;
62	struct iommu_domain *default_domain;	62	struct iommu_domain *default_domain;
63	struct iommu_domain *blocking_domain;	63	struct iommu_domain *blocking_domain;
		64	/*
		65	* During a group device reset, @resetting_domain points to the physical
		66	* domain, while @domain points to the attached domain before the reset.
		67	*/
		68	struct iommu_domain *resetting_domain;
64	struct iommu_domain *domain;	69	struct iommu_domain *domain;
65	struct list_head entry;	70	struct list_head entry;
66	unsigned int owner_cnt;	71	unsigned int owner_cnt;
@@ -232,6 +237,8 @@ static int __init iommu_subsys_init(void) · 165 unmodified lines
232	if (!nb)	237	if (!nb)
233	return -ENOMEM;	238	return -ENOMEM;
234		239
		240	iommu_debug_init();
		241
235	for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {	242	for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
236	nb[i].notifier_call = iommu_bus_notifier;	243	nb[i].notifier_call = iommu_bus_notifier;
237	bus_register_notifier(iommu_buses[i], &nb[i]);	244	bus_register_notifier(iommu_buses[i], &nb[i]);
@@ -661,7 +668,7 @@ static int __iommu_probe_device(struct device dev, struct list_head group_list · 423 unmodified lines
661	}	668	}
662		669
663	if (group->default_domain)	670	if (group->default_domain)
664	iommu_setup_dma_ops(dev);	671	iommu_setup_dma_ops(dev, group->default_domain);
665		672
666	mutex_unlock(&group->mutex);	673	mutex_unlock(&group->mutex);
667		674
@@ -1173,12 +1180,11 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, · 505 unmodified lines
1173	struct device *dev)	1180	struct device *dev)
1174	{	1181	{
1175	struct iommu_resv_region *entry;	1182	struct iommu_resv_region *entry;
1176	struct list_head mappings;	1183	LIST_HEAD(mappings);
1177	unsigned long pg_size;	1184	unsigned long pg_size;
1178	int ret = 0;	1185	int ret = 0;
1179		1186
1180	pg_size = domain->pgsize_bitmap ? 1UL << __ffs(domain->pgsize_bitmap) : 0;	1187	pg_size = domain->pgsize_bitmap ? 1UL << __ffs(domain->pgsize_bitmap) : 0;
1181	INIT_LIST_HEAD(&mappings);
1182		1188
1183	if (WARN_ON_ONCE(iommu_is_dma_domain(domain) && !pg_size))	1189	if (WARN_ON_ONCE(iommu_is_dma_domain(domain) && !pg_size))
1184	return -EINVAL;	1190	return -EINVAL;
@@ -1949,7 +1955,7 @@ static int bus_iommu_probe(const struct bus_type *bus) · 764 unmodified lines
1949	return ret;	1955	return ret;
1950	}	1956	}
1951	for_each_group_device(group, gdev)	1957	for_each_group_device(group, gdev)
1952	iommu_setup_dma_ops(gdev->dev);	1958	iommu_setup_dma_ops(gdev->dev, group->default_domain);
1953	mutex_unlock(&group->mutex);	1959	mutex_unlock(&group->mutex);
1954		1960
1955	/*	1961	/*
@@ -2185,10 +2191,26 @@ EXPORT_SYMBOL_GPL(iommu_attach_device); · 229 unmodified lines
2185		2191
2186	int iommu_deferred_attach(struct device dev, struct iommu_domain domain)	2192	int iommu_deferred_attach(struct device dev, struct iommu_domain domain)
2187	{	2193	{
2188	if (dev->iommu && dev->iommu->attach_deferred)	2194	/*
2189	return __iommu_attach_device(domain, dev, NULL);	2195	* This is called on the dma mapping fast path so avoid locking. This is
		2196	* racy, but we have an expectation that the driver will setup its DMAs
		2197	* inside probe while being single threaded to avoid racing.
		2198	*/
		2199	if (!dev->iommu \|\| !dev->iommu->attach_deferred)
		2200	return 0;
2190		2201
2191	return 0;	2202	guard(mutex)(&dev->iommu_group->mutex);
		2203
		2204	/*
		2205	* This is a concurrent attach during a device reset. Reject it until
		2206	* pci_dev_reset_iommu_done() attaches the device to group->domain.
		2207	*
		2208	* Note that this might fail the iommu_dma_map(). But there's nothing
		2209	* more we can do here.
		2210	*/
		2211	if (dev->iommu_group->resetting_domain)
		2212	return -EBUSY;
		2213	return __iommu_attach_device(domain, dev, NULL);
2192	}	2214	}
2193		2215
2194	void iommu_detach_device(struct iommu_domain domain, struct device dev)	2216	void iommu_detach_device(struct iommu_domain domain, struct device dev)
@@ -2210,6 +2232,15 @@ out_unlock: · 15 unmodified lines
2210	}	2232	}
2211	EXPORT_SYMBOL_GPL(iommu_detach_device);	2233	EXPORT_SYMBOL_GPL(iommu_detach_device);
2212		2234
		2235	/**
		2236	* iommu_get_domain_for_dev() - Return the DMA API domain pointer
		2237	* @dev: Device to query
		2238	*
		2239	* This function can be called within a driver bound to dev. The returned
		2240	* pointer is valid for the lifetime of the bound driver.
		2241	*
		2242	* It should not be called by drivers with driver_managed_dma = true.
		2243	*/
2213	struct iommu_domain iommu_get_domain_for_dev(struct device dev)	2244	struct iommu_domain iommu_get_domain_for_dev(struct device dev)
2214	{	2245	{
2215	/* Caller must be a probed driver on dev */	2246	/* Caller must be a probed driver on dev */
@@ -2218,10 +2249,40 @@ struct iommu_domain iommu_get_domain_for_dev(struct device dev) · 2 unmodified lines
2218	if (!group)	2249	if (!group)
2219	return NULL;	2250	return NULL;
2220		2251
		2252	lockdep_assert_not_held(&group->mutex);
		2253
2221	return group->domain;	2254	return group->domain;
2222	}	2255	}
2223	EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);	2256	EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
2224		2257
		2258	/**
		2259	* iommu_driver_get_domain_for_dev() - Return the driver-level domain pointer
		2260	* @dev: Device to query
		2261	*
		2262	* This function can be called by an iommu driver that wants to get the physical
		2263	* domain within an iommu callback function where group->mutex is held.
		2264	*/
		2265	struct iommu_domain iommu_driver_get_domain_for_dev(struct device dev)
		2266	{
		2267	struct iommu_group *group = dev->iommu_group;
		2268
		2269	lockdep_assert_held(&group->mutex);
		2270
		2271	/*
		2272	* Driver handles the low-level __iommu_attach_device(), including the
		2273	* one invoked by pci_dev_reset_iommu_done() re-attaching the device to
		2274	* the cached group->domain. In this case, the driver must get the old
		2275	* domain from group->resetting_domain rather than group->domain. This
		2276	* prevents it from re-attaching the device from group->domain (old) to
		2277	* group->domain (new).
		2278	*/
		2279	if (group->resetting_domain)
		2280	return group->resetting_domain;
		2281
		2282	return group->domain;
		2283	}
		2284	EXPORT_SYMBOL_GPL(iommu_driver_get_domain_for_dev);
		2285
2225	/*	2286	/*
2226	* For IOMMU_DOMAIN_DMA implementations which already provide their own	2287	* For IOMMU_DOMAIN_DMA implementations which already provide their own
2227	* guarantees that the group and its default domain are valid and correct.	2288	* guarantees that the group and its default domain are valid and correct.
@@ -2374,6 +2435,13 @@ static int __iommu_group_set_domain_internal(struct iommu_group *group, · 146 unmodified lines
2374	if (WARN_ON(!new_domain))	2435	if (WARN_ON(!new_domain))
2375	return -EINVAL;	2436	return -EINVAL;
2376		2437
		2438	/*
		2439	* This is a concurrent attach during a device reset. Reject it until
		2440	* pci_dev_reset_iommu_done() attaches the device to group->domain.
		2441	*/
		2442	if (group->resetting_domain)
		2443	return -EBUSY;
		2444
2377	/*	2445	/*
2378	* Changing the domain is done by calling attach_dev() on the new	2446	* Changing the domain is done by calling attach_dev() on the new
2379	* domain. This switch does not have to be atomic and DMA can be	2447	* domain. This switch does not have to be atomic and DMA can be
@@ -2562,10 +2630,12 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, · 182 unmodified lines
2562	}	2630	}
2563		2631
2564	/* unroll mapping in case something went wrong */	2632	/* unroll mapping in case something went wrong */
2565	if (ret)	2633	if (ret) {
2566	iommu_unmap(domain, orig_iova, orig_size - size);	2634	iommu_unmap(domain, orig_iova, orig_size - size);
2567	else	2635	} else {
2568	trace_map(orig_iova, orig_paddr, orig_size);	2636	trace_map(orig_iova, orig_paddr, orig_size);
		2637	iommu_debug_map(domain, orig_paddr, orig_size);
		2638	}
2569		2639
2570	return ret;	2640	return ret;
2571	}	2641	}
@@ -2627,6 +2697,8 @@ static size_t __iommu_unmap(struct iommu_domain *domain, · 55 unmodified lines
2627		2697
2628	pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);	2698	pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
2629		2699
		2700	iommu_debug_unmap_begin(domain, iova, size);
		2701
2630	/*	2702	/*
2631	* Keep iterating until we either unmap 'size' bytes (or more)	2703	* Keep iterating until we either unmap 'size' bytes (or more)
2632	* or we hit an area that isn't mapped.	2704	* or we hit an area that isn't mapped.
@@ -2647,6 +2719,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain, · 14 unmodified lines
2647	}	2719	}
2648		2720
2649	trace_unmap(orig_iova, size, unmapped);	2721	trace_unmap(orig_iova, size, unmapped);
		2722	iommu_debug_unmap_end(domain, orig_iova, size, unmapped);
2650	return unmapped;	2723	return unmapped;
2651	}	2724	}
2652		2725
@@ -3148,7 +3221,7 @@ static ssize_t iommu_group_store_type(struct iommu_group *group, · 495 unmodified lines
3148		3221
3149	/* Make sure dma_ops is appropriatley set */	3222	/* Make sure dma_ops is appropriatley set */
3150	for_each_group_device(group, gdev)	3223	for_each_group_device(group, gdev)
3151	iommu_setup_dma_ops(gdev->dev);	3224	iommu_setup_dma_ops(gdev->dev, group->default_domain);
3152		3225
3153	out_unlock:	3226	out_unlock:
3154	mutex_unlock(&group->mutex);	3227	mutex_unlock(&group->mutex);
@@ -3492,6 +3565,16 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, · 337 unmodified lines
3492	return -EINVAL;	3565	return -EINVAL;
3493		3566
3494	mutex_lock(&group->mutex);	3567	mutex_lock(&group->mutex);
		3568
		3569	/*
		3570	* This is a concurrent attach during a device reset. Reject it until
		3571	* pci_dev_reset_iommu_done() attaches the device to group->domain.
		3572	*/
		3573	if (group->resetting_domain) {
		3574	ret = -EBUSY;
		3575	goto out_unlock;
		3576	}
		3577
3495	for_each_group_device(group, device) {	3578	for_each_group_device(group, device) {
3496	/*	3579	/*
3497	* Skip PASID validation for devices without PASID support	3580	* Skip PASID validation for devices without PASID support
@@ -3575,6 +3658,16 @@ int iommu_replace_device_pasid(struct iommu_domain *domain, · 77 unmodified lines
3575	return -EINVAL;	3658	return -EINVAL;
3576		3659
3577	mutex_lock(&group->mutex);	3660	mutex_lock(&group->mutex);
		3661
		3662	/*
		3663	* This is a concurrent attach during a device reset. Reject it until
		3664	* pci_dev_reset_iommu_done() attaches the device to group->domain.
		3665	*/
		3666	if (group->resetting_domain) {
		3667	ret = -EBUSY;
		3668	goto out_unlock;
		3669	}
		3670
3578	entry = iommu_make_pasid_array_entry(domain, handle);	3671	entry = iommu_make_pasid_array_entry(domain, handle);
3579	curr = xa_cmpxchg(&group->pasid_array, pasid, NULL,	3672	curr = xa_cmpxchg(&group->pasid_array, pasid, NULL,
3580	XA_ZERO_ENTRY, GFP_KERNEL);	3673	XA_ZERO_ENTRY, GFP_KERNEL);
@@ -3832,6 +3925,127 @@ err_unlock: · 251 unmodified lines
3832	}	3925	}
3833	EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");	3926	EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");
3834		3927
		3928	/**
		3929	* pci_dev_reset_iommu_prepare() - Block IOMMU to prepare for a PCI device reset
		3930	* @pdev: PCI device that is going to enter a reset routine
		3931	*
		3932	* The PCIe r6.0, sec 10.3.1 IMPLEMENTATION NOTE recommends to disable and block
		3933	* ATS before initiating a reset. This means that a PCIe device during the reset
		3934	* routine wants to block any IOMMU activity: translation and ATS invalidation.
		3935	*
		3936	* This function attaches the device's RID/PASID(s) the group->blocking_domain,
		3937	* setting the group->resetting_domain. This allows the IOMMU driver pausing any
		3938	* IOMMU activity while leaving the group->domain pointer intact. Later when the
		3939	* reset is finished, pci_dev_reset_iommu_done() can restore everything.
		3940	*
		3941	* Caller must use pci_dev_reset_iommu_prepare() with pci_dev_reset_iommu_done()
		3942	* before/after the core-level reset routine, to unset the resetting_domain.
		3943	*
		3944	* Return: 0 on success or negative error code if the preparation failed.
		3945	*
		3946	* These two functions are designed to be used by PCI reset functions that would
		3947	* not invoke any racy iommu_release_device(), since PCI sysfs node gets removed
		3948	* before it notifies with a BUS_NOTIFY_REMOVED_DEVICE. When using them in other
		3949	* case, callers must ensure there will be no racy iommu_release_device() call,
		3950	* which otherwise would UAF the dev->iommu_group pointer.
		3951	*/
		3952	int pci_dev_reset_iommu_prepare(struct pci_dev *pdev)
		3953	{
		3954	struct iommu_group *group = pdev->dev.iommu_group;
		3955	unsigned long pasid;
		3956	void *entry;
		3957	int ret;
		3958
		3959	if (!pci_ats_supported(pdev) \|\| !dev_has_iommu(&pdev->dev))
		3960	return 0;
		3961
		3962	guard(mutex)(&group->mutex);
		3963
		3964	/* Re-entry is not allowed */
		3965	if (WARN_ON(group->resetting_domain))
		3966	return -EBUSY;
		3967
		3968	ret = __iommu_group_alloc_blocking_domain(group);
		3969	if (ret)
		3970	return ret;
		3971
		3972	/* Stage RID domain at blocking_domain while retaining group->domain */
		3973	if (group->domain != group->blocking_domain) {
		3974	ret = __iommu_attach_device(group->blocking_domain, &pdev->dev,
		3975	group->domain);
		3976	if (ret)
		3977	return ret;
		3978	}
		3979
		3980	/*
		3981	* Stage PASID domains at blocking_domain while retaining pasid_array.
		3982	*
		3983	* The pasid_array is mostly fenced by group->mutex, except one reader
		3984	* in iommu_attach_handle_get(), so it's safe to read without xa_lock.
		3985	*/
		3986	xa_for_each_start(&group->pasid_array, pasid, entry, 1)
		3987	iommu_remove_dev_pasid(&pdev->dev, pasid,
		3988	pasid_array_entry_to_domain(entry));
		3989
		3990	group->resetting_domain = group->blocking_domain;
		3991	return ret;
		3992	}
		3993	EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_prepare);
		3994
		3995	/**
		3996	* pci_dev_reset_iommu_done() - Restore IOMMU after a PCI device reset is done
		3997	* @pdev: PCI device that has finished a reset routine
		3998	*
		3999	* After a PCIe device finishes a reset routine, it wants to restore its IOMMU
		4000	* IOMMU activity, including new translation as well as cache invalidation, by
		4001	* re-attaching all RID/PASID of the device's back to the domains retained in
		4002	* the core-level structure.
		4003	*
		4004	* Caller must pair it with a successful pci_dev_reset_iommu_prepare().
		4005	*
		4006	* Note that, although unlikely, there is a risk that re-attaching domains might
		4007	* fail due to some unexpected happening like OOM.
		4008	*/
		4009	void pci_dev_reset_iommu_done(struct pci_dev *pdev)
		4010	{
		4011	struct iommu_group *group = pdev->dev.iommu_group;
		4012	unsigned long pasid;
		4013	void *entry;
		4014
		4015	if (!pci_ats_supported(pdev) \|\| !dev_has_iommu(&pdev->dev))
		4016	return;
		4017
		4018	guard(mutex)(&group->mutex);
		4019
		4020	/* pci_dev_reset_iommu_prepare() was bypassed for the device */
		4021	if (!group->resetting_domain)
		4022	return;
		4023
		4024	/* pci_dev_reset_iommu_prepare() was not successfully called */
		4025	if (WARN_ON(!group->blocking_domain))
		4026	return;
		4027
		4028	/* Re-attach RID domain back to group->domain */
		4029	if (group->domain != group->blocking_domain) {
		4030	WARN_ON(__iommu_attach_device(group->domain, &pdev->dev,
		4031	group->blocking_domain));
		4032	}
		4033
		4034	/*
		4035	* Re-attach PASID domains back to the domains retained in pasid_array.
		4036	*
		4037	* The pasid_array is mostly fenced by group->mutex, except one reader
		4038	* in iommu_attach_handle_get(), so it's safe to read without xa_lock.
		4039	*/
		4040	xa_for_each_start(&group->pasid_array, pasid, entry, 1)
		4041	WARN_ON(__iommu_set_group_pasid(
		4042	pasid_array_entry_to_domain(entry), group, pasid,
		4043	group->blocking_domain));
		4044
		4045	group->resetting_domain = NULL;
		4046	}
		4047	EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_done);
		4048
3835	#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)	4049	#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
3836	/**	4050	/**
3837	* iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain	4051	* iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain

drivers/pci/pci-acpi.c+11−2

 #include <linux/delay.h>
 #include <linux/init.h>
+#include <linux/iommu.h>
 #include <linux/irqdomain.h>
 #include <linux/pci.h>
 #include <linux/msi.h>
 int pci_dev_acpi_reset(struct pci_dev *dev, bool probe)
 {
 	acpi_handle handle = ACPI_HANDLE(&dev->dev);
+	int ret;
 	if (!handle || !acpi_has_method(handle, "_RST"))
 		return -ENOTTY;
 	if (probe)
 		return 0;
+	ret = pci_dev_reset_iommu_prepare(dev);
+	if (ret) {
+		pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
+		return ret;
+	}
 	if (ACPI_FAILURE(acpi_evaluate_object(handle, "_RST", NULL, NULL))) {
 		pci_warn(dev, "ACPI _RST failed\n");
-		return -ENOTTY;
+		ret = -ENOTTY;
 	}
-	return 0;
+	pci_dev_reset_iommu_done(dev);
+	return ret;
 }
 bool acpi_pci_power_manageable(struct pci_dev *dev)

drivers/pci/pci.c+58−7

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	#include <linux/delay.h>	13	#include <linux/delay.h>
14	#include <linux/dmi.h>	14	#include <linux/dmi.h>
15	#include <linux/init.h>	15	#include <linux/init.h>
		16	#include <linux/iommu.h>
16	#include <linux/msi.h>	17	#include <linux/msi.h>
17	#include <linux/of.h>	18	#include <linux/of.h>
18	#include <linux/pci.h>	19	#include <linux/pci.h>
@@ -25,6 +26,7 @@ · 6 unmodified lines
25	#include <linux/logic_pio.h>	26	#include <linux/logic_pio.h>
26	#include <linux/device.h>	27	#include <linux/device.h>
27	#include <linux/pm_runtime.h>	28	#include <linux/pm_runtime.h>
		29	#include <linux/pci-ats.h>
28	#include <linux/pci_hotplug.h>	30	#include <linux/pci_hotplug.h>
29	#include <linux/vmalloc.h>	31	#include <linux/vmalloc.h>
30	#include <asm/dma.h>	32	#include <asm/dma.h>
@@ -4330,13 +4332,22 @@ EXPORT_SYMBOL(pci_wait_for_pending_transaction); · 4299 unmodified lines
4330	*/	4332	*/
4331	int pcie_flr(struct pci_dev *dev)	4333	int pcie_flr(struct pci_dev *dev)
4332	{	4334	{
		4335	int ret;
		4336
4333	if (!pci_wait_for_pending_transaction(dev))	4337	if (!pci_wait_for_pending_transaction(dev))
4334	pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");	4338	pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
4335		4339
		4340	/* Have to call it after waiting for pending DMA transaction */
		4341	ret = pci_dev_reset_iommu_prepare(dev);
		4342	if (ret) {
		4343	pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
		4344	return ret;
		4345	}
		4346
4336	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);	4347	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
4337		4348
4338	if (dev->imm_ready)	4349	if (dev->imm_ready)
4339	return 0;	4350	goto done;
4340		4351
4341	/*	4352	/*
4342	* Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within	4353	* Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
@@ -4345,7 +4356,10 @@ int pcie_flr(struct pci_dev *dev) · 2 unmodified lines
4345	*/	4356	*/
4346	msleep(100);	4357	msleep(100);
4347		4358
4348	return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);	4359	ret = pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);
		4360	done:
		4361	pci_dev_reset_iommu_done(dev);
		4362	return ret;
4349	}	4363	}
4350	EXPORT_SYMBOL_GPL(pcie_flr);	4364	EXPORT_SYMBOL_GPL(pcie_flr);
4351		4365
@@ -4373,6 +4387,7 @@ EXPORT_SYMBOL_GPL(pcie_reset_flr); · 21 unmodified lines
4373		4387
4374	static int pci_af_flr(struct pci_dev *dev, bool probe)	4388	static int pci_af_flr(struct pci_dev *dev, bool probe)
4375	{	4389	{
		4390	int ret;
4376	int pos;	4391	int pos;
4377	u8 cap;	4392	u8 cap;
4378		4393
@@ -4399,10 +4414,17 @@ static int pci_af_flr(struct pci_dev *dev, bool probe) · 20 unmodified lines
4399	PCI_AF_STATUS_TP << 8))	4414	PCI_AF_STATUS_TP << 8))
4400	pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");	4415	pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
4401		4416
		4417	/* Have to call it after waiting for pending DMA transaction */
		4418	ret = pci_dev_reset_iommu_prepare(dev);
		4419	if (ret) {
		4420	pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
		4421	return ret;
		4422	}
		4423
4402	pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);	4424	pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
4403		4425
4404	if (dev->imm_ready)	4426	if (dev->imm_ready)
4405	return 0;	4427	goto done;
4406		4428
4407	/*	4429	/*
4408	* Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,	4430	* Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
@@ -4412,7 +4434,10 @@ static int pci_af_flr(struct pci_dev *dev, bool probe) · 3 unmodified lines
4412	*/	4434	*/
4413	msleep(100);	4435	msleep(100);
4414		4436
4415	return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);	4437	ret = pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);
		4438	done:
		4439	pci_dev_reset_iommu_done(dev);
		4440	return ret;
4416	}	4441	}
4417		4442
4418	/**	4443	/**
@@ -4433,6 +4458,7 @@ static int pci_af_flr(struct pci_dev *dev, bool probe) · 14 unmodified lines
4433	static int pci_pm_reset(struct pci_dev *dev, bool probe)	4458	static int pci_pm_reset(struct pci_dev *dev, bool probe)
4434	{	4459	{
4435	u16 csr;	4460	u16 csr;
		4461	int ret;
4436		4462
4437	if (!dev->pm_cap \|\| dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)	4463	if (!dev->pm_cap \|\| dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
4438	return -ENOTTY;	4464	return -ENOTTY;
@@ -4447,6 +4473,12 @@ static int pci_pm_reset(struct pci_dev *dev, bool probe) · 8 unmodified lines
4447	if (dev->current_state != PCI_D0)	4473	if (dev->current_state != PCI_D0)
4448	return -EINVAL;	4474	return -EINVAL;
4449		4475
		4476	ret = pci_dev_reset_iommu_prepare(dev);
		4477	if (ret) {
		4478	pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
		4479	return ret;
		4480	}
		4481
4450	csr &= ~PCI_PM_CTRL_STATE_MASK;	4482	csr &= ~PCI_PM_CTRL_STATE_MASK;
4451	csr \|= PCI_D3hot;	4483	csr \|= PCI_D3hot;
4452	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);	4484	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
@@ -4457,7 +4489,9 @@ static int pci_pm_reset(struct pci_dev *dev, bool probe) · 4 unmodified lines
4457	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);	4489	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4458	pci_dev_d3_sleep(dev);	4490	pci_dev_d3_sleep(dev);
4459		4491
4460	return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);	4492	ret = pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
		4493	pci_dev_reset_iommu_done(dev);
		4494	return ret;
4461	}	4495	}
4462		4496
4463	/**	4497	/**
@@ -4885,10 +4919,20 @@ static int pci_reset_bus_function(struct pci_dev *dev, bool probe) · 421 unmodified lines
4885	return -ENOTTY;	4919	return -ENOTTY;
4886	}	4920	}
4887		4921
		4922	rc = pci_dev_reset_iommu_prepare(dev);
		4923	if (rc) {
		4924	pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", rc);
		4925	return rc;
		4926	}
		4927
4888	rc = pci_dev_reset_slot_function(dev, probe);	4928	rc = pci_dev_reset_slot_function(dev, probe);
4889	if (rc != -ENOTTY)	4929	if (rc != -ENOTTY)
4890	return rc;	4930	goto done;
4891	return pci_parent_bus_reset(dev, probe);	4931
		4932	rc = pci_parent_bus_reset(dev, probe);
		4933	done:
		4934	pci_dev_reset_iommu_done(dev);
		4935	return rc;
4892	}	4936	}
4893		4937
4894	static int cxl_reset_bus_function(struct pci_dev *dev, bool probe)	4938	static int cxl_reset_bus_function(struct pci_dev *dev, bool probe)
@@ -4912,6 +4956,12 @@ static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) · 17 unmodified lines
4912	if (rc)	4956	if (rc)
4913	return -ENOTTY;	4957	return -ENOTTY;
4914		4958
		4959	rc = pci_dev_reset_iommu_prepare(dev);
		4960	if (rc) {
		4961	pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", rc);
		4962	return rc;
		4963	}
		4964
4915	if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) {	4965	if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) {
4916	val = reg;	4966	val = reg;
4917	} else {	4967	} else {
@@ -4926,6 +4976,7 @@ static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) · 8 unmodified lines
4926	pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL,	4976	pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL,
4927	reg);	4977	reg);
4928		4978
		4979	pci_dev_reset_iommu_done(dev);
4929	return rc;	4980	return rc;
4930	}	4981	}
4931		4982

drivers/pci/quirks.c+18−1

 #include <linux/pci.h>
 #include <linux/isa-dma.h> /* isa_dma_bridge_buggy */
 #include <linux/init.h>
+#include <linux/iommu.h>
 #include <linux/delay.h>
 #include <linux/acpi.h>
 #include <linux/dmi.h>
 	{ 0 }
 };
+static int __pci_dev_specific_reset(struct pci_dev *dev, bool probe,
+				    const struct pci_dev_reset_methods *i)
+{
+	int ret;
+	ret = pci_dev_reset_iommu_prepare(dev);
+	if (ret) {
+		pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
+		return ret;
+	}
+	ret = i->reset(dev, probe);
+	pci_dev_reset_iommu_done(dev);
+	return ret;
+}
 /*
  * These device-specific reset methods are here rather than in a driver
  * because when a host assigns a device to a guest VM, the host may need
 		     i->vendor == (u16)PCI_ANY_ID) &&
 		    (i->device == dev->device ||
 		     i->device == (u16)PCI_ANY_ID))
-			return i->reset(dev, probe);
+			return __pci_dev_specific_reset(dev, probe, i);
 	}
 	return -ENOTTY;

include/linux/iommu-debug-pagealloc.h+32−0

@@ -0,0 +1,32 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2025 - Google Inc
	4	* Author: Mostafa Saleh <smostafa@google.com>
	5	* IOMMU API debug page alloc sanitizer
	6	*/
	7
	8	#ifndef __LINUX_IOMMU_DEBUG_PAGEALLOC_H
	9	#define __LINUX_IOMMU_DEBUG_PAGEALLOC_H
	10
	11	#ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC
	12	DECLARE_STATIC_KEY_FALSE(iommu_debug_initialized);
	13
	14	extern struct page_ext_operations page_iommu_debug_ops;
	15
	16	void __iommu_debug_check_unmapped(const struct page *page, int numpages);
	17
	18	static inline void iommu_debug_check_unmapped(const struct page *page, int numpages)
	19	{
	20	if (static_branch_unlikely(&iommu_debug_initialized))
	21	__iommu_debug_check_unmapped(page, numpages);
	22	}
	23
	24	#else
	25	static inline void iommu_debug_check_unmapped(const struct page *page,
	26	int numpages)
	27	{
	28	}
	29
	30	#endif /* CONFIG_IOMMU_DEBUG_PAGEALLOC */
	31
	32	#endif /* __LINUX_IOMMU_DEBUG_PAGEALLOC_H */

include/linux/iommu.h+14−0

 extern void iommu_detach_device(struct iommu_domain *domain,
 				struct device *dev);
 extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
+struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev);
 extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
 extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
 		     phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
 			       struct device *dev, ioasid_t pasid);
 ioasid_t iommu_alloc_global_pasid(struct device *dev);
 void iommu_free_global_pasid(ioasid_t pasid);
+/* PCI device reset functions */
+int pci_dev_reset_iommu_prepare(struct pci_dev *pdev);
+void pci_dev_reset_iommu_done(struct pci_dev *pdev);
 #else /* CONFIG_IOMMU_API */
 struct iommu_ops {};
 }
 static inline void iommu_free_global_pasid(ioasid_t pasid) {}
+static inline int pci_dev_reset_iommu_prepare(struct pci_dev *pdev)
+{
+	return 0;
+}
+static inline void pci_dev_reset_iommu_done(struct pci_dev *pdev)
+{
+}
 #endif /* CONFIG_IOMMU_API */
 #ifdef CONFIG_IRQ_MSI_IOMMU

include/linux/mm.h+5−0

 #include <linux/rcuwait.h>
 #include <linux/bitmap.h>
 #include <linux/bitops.h>
+#include <linux/iommu-debug-pagealloc.h>
 struct mempolicy;
 struct anon_vma;
 #ifdef CONFIG_DEBUG_PAGEALLOC
 static inline void debug_pagealloc_map_pages(struct page *page, int numpages)
 {
+	iommu_debug_check_unmapped(page, numpages);
 	if (debug_pagealloc_enabled_static())
 		__kernel_map_pages(page, numpages, 1);
 }
 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages)
 {
+	iommu_debug_check_unmapped(page, numpages);
 	if (debug_pagealloc_enabled_static())
 		__kernel_map_pages(page, numpages, 0);
 }

include/linux/page_ext.h+6−0

 #endif
 extern struct page_ext *page_ext_get(const struct page *page);
+extern struct page_ext *page_ext_from_phys(phys_addr_t phys);
 extern void page_ext_put(struct page_ext *page_ext);
 extern struct page_ext *page_ext_lookup(unsigned long pfn);
 	return NULL;
 }
+static inline struct page_ext *page_ext_from_phys(phys_addr_t phys)
+{
+	return NULL;
+}
 static inline void page_ext_put(struct page_ext *page_ext)
 {
 }

include/uapi/linux/iommufd.h+39−0

@@ -465,16 +465,27 @@ struct iommu_hwpt_arm_smmuv3 { · 464 unmodified lines
465	__aligned_le64 ste[2];	465	__aligned_le64 ste[2];
466	};	466	};
467		467
		468	/**
		469	* struct iommu_hwpt_amd_guest - AMD IOMMU guest I/O page table data
		470	* (IOMMU_HWPT_DATA_AMD_GUEST)
		471	* @dte: Guest Device Table Entry (DTE)
		472	*/
		473	struct iommu_hwpt_amd_guest {
		474	__aligned_u64 dte[4];
		475	};
		476
468	/**	477	/**
469	* enum iommu_hwpt_data_type - IOMMU HWPT Data Type	478	* enum iommu_hwpt_data_type - IOMMU HWPT Data Type
470	* @IOMMU_HWPT_DATA_NONE: no data	479	* @IOMMU_HWPT_DATA_NONE: no data
471	* @IOMMU_HWPT_DATA_VTD_S1: Intel VT-d stage-1 page table	480	* @IOMMU_HWPT_DATA_VTD_S1: Intel VT-d stage-1 page table
472	* @IOMMU_HWPT_DATA_ARM_SMMUV3: ARM SMMUv3 Context Descriptor Table	481	* @IOMMU_HWPT_DATA_ARM_SMMUV3: ARM SMMUv3 Context Descriptor Table
		482	* @IOMMU_HWPT_DATA_AMD_GUEST: AMD IOMMU guest page table
473	*/	483	*/
474	enum iommu_hwpt_data_type {	484	enum iommu_hwpt_data_type {
475	IOMMU_HWPT_DATA_NONE = 0,	485	IOMMU_HWPT_DATA_NONE = 0,
476	IOMMU_HWPT_DATA_VTD_S1 = 1,	486	IOMMU_HWPT_DATA_VTD_S1 = 1,
477	IOMMU_HWPT_DATA_ARM_SMMUV3 = 2,	487	IOMMU_HWPT_DATA_ARM_SMMUV3 = 2,
		488	IOMMU_HWPT_DATA_AMD_GUEST = 3,
478	};	489	};
479		490
480	/**	491	/**
@@ -623,6 +634,32 @@ struct iommu_hw_info_tegra241_cmdqv { · 142 unmodified lines
623	__u8 __reserved;	634	__u8 __reserved;
624	};	635	};
625		636
		637	/**
		638	* struct iommu_hw_info_amd - AMD IOMMU device info
		639	*
		640	* @efr : Value of AMD IOMMU Extended Feature Register (EFR)
		641	* @efr2: Value of AMD IOMMU Extended Feature 2 Register (EFR2)
		642	*
		643	* Please See description of these registers in the following sections of
		644	* the AMD I/O Virtualization Technology (IOMMU) Specification.
		645	* (https://docs.amd.com/v/u/en-US/48882_3.10_PUB)
		646	*
		647	* - MMIO Offset 0030h IOMMU Extended Feature Register
		648	* - MMIO Offset 01A0h IOMMU Extended Feature 2 Register
		649	*
		650	* Note: The EFR and EFR2 are raw values reported by hardware.
		651	* VMM is responsible to determine the appropriate flags to be exposed to
		652	* the VM since cetertain features are not currently supported by the kernel
		653	* for HW-vIOMMU.
		654	*
		655	* Current VMM-allowed list of feature flags are:
		656	* - EFR[GTSup, GASup, GioSup, PPRSup, EPHSup, GATS, GLX, PASmax]
		657	*/
		658	struct iommu_hw_info_amd {
		659	__aligned_u64 efr;
		660	__aligned_u64 efr2;
		661	};
		662
626	/**	663	/**
627	* enum iommu_hw_info_type - IOMMU Hardware Info Types	664	* enum iommu_hw_info_type - IOMMU Hardware Info Types
628	* @IOMMU_HW_INFO_TYPE_NONE: Output by the drivers that do not report hardware	665	* @IOMMU_HW_INFO_TYPE_NONE: Output by the drivers that do not report hardware
@@ -632,6 +669,7 @@ struct iommu_hw_info_tegra241_cmdqv { · 3 unmodified lines
632	* @IOMMU_HW_INFO_TYPE_ARM_SMMUV3: ARM SMMUv3 iommu info type	669	* @IOMMU_HW_INFO_TYPE_ARM_SMMUV3: ARM SMMUv3 iommu info type
633	* @IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV: NVIDIA Tegra241 CMDQV (extension for ARM	670	* @IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV: NVIDIA Tegra241 CMDQV (extension for ARM
634	* SMMUv3) info type	671	* SMMUv3) info type
		672	* @IOMMU_HW_INFO_TYPE_AMD: AMD IOMMU info type
635	*/	673	*/
636	enum iommu_hw_info_type {	674	enum iommu_hw_info_type {
637	IOMMU_HW_INFO_TYPE_NONE = 0,	675	IOMMU_HW_INFO_TYPE_NONE = 0,
@@ -639,6 +677,7 @@ enum iommu_hw_info_type { · 1 unmodified line
639	IOMMU_HW_INFO_TYPE_INTEL_VTD = 1,	677	IOMMU_HW_INFO_TYPE_INTEL_VTD = 1,
640	IOMMU_HW_INFO_TYPE_ARM_SMMUV3 = 2,	678	IOMMU_HW_INFO_TYPE_ARM_SMMUV3 = 2,
641	IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV = 3,	679	IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV = 3,
		680	IOMMU_HW_INFO_TYPE_AMD = 4,
642	};	681	};
643		682
644	/**	683	/**

include/uapi/linux/vfio.h+4−0

  * hwpt corresponding to the given pt_id.
  *
  * Return: 0 on success, -errno on failure.
+ *
+ * When a device is resetting, -EBUSY will be returned to reject any concurrent
+ * attachment to the resetting device itself or any sibling device in the IOMMU
+ * group having the resetting device.
  */
 struct vfio_device_attach_iommufd_pt {
 	__u32	argsz;

mm/page_ext.c+27−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	#include <linux/page_table_check.h>	11	#include <linux/page_table_check.h>
12	#include <linux/rcupdate.h>	12	#include <linux/rcupdate.h>
13	#include <linux/pgalloc_tag.h>	13	#include <linux/pgalloc_tag.h>
		14	#include <linux/iommu-debug-pagealloc.h>
14		15
15	/*	16	/*
16	* struct page extension	17	* struct page extension
@@ -89,6 +90,9 @@ static struct page_ext_operations *page_ext_ops[] __initdata = { · 72 unmodified lines
89	#ifdef CONFIG_PAGE_TABLE_CHECK	90	#ifdef CONFIG_PAGE_TABLE_CHECK
90	&page_table_check_ops,	91	&page_table_check_ops,
91	#endif	92	#endif
		93	#ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC
		94	&page_iommu_debug_ops,
		95	#endif
92	};	96	};
93		97
94	unsigned long page_ext_size;	98	unsigned long page_ext_size;
@@ -534,6 +538,29 @@ struct page_ext page_ext_get(const struct page page) · 439 unmodified lines
534	return page_ext;	538	return page_ext;
535	}	539	}
536		540
		541	/**
		542	* page_ext_from_phys() - Get the page_ext structure for a physical address.
		543	* @phys: The physical address to query.
		544	*
		545	* This function safely gets the `struct page_ext` associated with a given
		546	* physical address. It performs validation to ensure the address corresponds
		547	* to a valid, online struct page before attempting to access it.
		548	* It returns NULL for MMIO, ZONE_DEVICE, holes and offline memory.
		549	*
		550	* Return: NULL if no page_ext exists for this physical address.
		551	* Context: Any context. Caller may not sleep until they have called
		552	* page_ext_put().
		553	*/
		554	struct page_ext *page_ext_from_phys(phys_addr_t phys)
		555	{
		556	struct page *page = pfn_to_online_page(__phys_to_pfn(phys));
		557
		558	if (!page)
		559	return NULL;
		560
		561	return page_ext_get(page);
		562	}
		563
537	/**	564	/**
538	* page_ext_put() - Working with page extended information is done.	565	* page_ext_put() - Working with page extended information is done.
539	* @page_ext: Page extended information received from page_ext_get().	566	* @page_ext: Page extended information received from page_ext_get().

rust/bindings/bindings_helper.h+2−1

 #include <linux/fdtable.h>
 #include <linux/file.h>
 #include <linux/firmware.h>
-#include <linux/interrupt.h>
 #include <linux/fs.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/io-pgtable.h>
 #include <linux/ioport.h>
 #include <linux/jiffies.h>
 #include <linux/jump_label.h>

rust/kernel/iommu/mod.rs+5−0

@@ -0,0 +1,5 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	//! Rust support related to IOMMU.
	4
	5	pub mod pgtable;

rust/kernel/iommu/pgtable.rs+279−0

@@ -0,0 +1,279 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	//! IOMMU page table management.
	4	//!
	5	//! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h)
	6
	7	use core::{
	8	marker::PhantomData,
	9	ptr::NonNull, //
	10	};
	11
	12	use crate::{
	13	alloc,
	14	bindings,
	15	device::{
	16	Bound,
	17	Device, //
	18	},
	19	devres::Devres,
	20	error::to_result,
	21	io::PhysAddr,
	22	prelude::*, //
	23	};
	24
	25	use bindings::io_pgtable_fmt;
	26
	27	/// Protection flags used with IOMMU mappings.
	28	pub mod prot {
	29	/// Read access.
	30	pub const READ: u32 = bindings::IOMMU_READ;
	31	/// Write access.
	32	pub const WRITE: u32 = bindings::IOMMU_WRITE;
	33	/// Request cache coherency.
	34	pub const CACHE: u32 = bindings::IOMMU_CACHE;
	35	/// Request no-execute permission.
	36	pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC;
	37	/// MMIO peripheral mapping.
	38	pub const MMIO: u32 = bindings::IOMMU_MMIO;
	39	/// Privileged mapping.
	40	pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV;
	41	}
	42
	43	/// Represents a requested `io_pgtable` configuration.
	44	pub struct Config {
	45	/// Quirk bitmask (type-specific).
	46	pub quirks: usize,
	47	/// Valid page sizes, as a bitmask of powers of two.
	48	pub pgsize_bitmap: usize,
	49	/// Input address space size in bits.
	50	pub ias: u32,
	51	/// Output address space size in bits.
	52	pub oas: u32,
	53	/// IOMMU uses coherent accesses for page table walks.
	54	pub coherent_walk: bool,
	55	}
	56
	57	/// An io page table using a specific format.
	58	///
	59	/// # Invariants
	60	///
	61	/// The pointer references a valid io page table.
	62	pub struct IoPageTable<F: IoPageTableFmt> {
	63	ptr: NonNull<bindings::io_pgtable_ops>,
	64	_marker: PhantomData<F>,
	65	}
	66
	67	// SAFETY: `struct io_pgtable_ops` is not restricted to a single thread.
	68	unsafe impl<F: IoPageTableFmt> Send for IoPageTable<F> {}
	69	// SAFETY: `struct io_pgtable_ops` may be accessed concurrently.
	70	unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<F> {}
	71
	72	/// The format used by this page table.
	73	pub trait IoPageTableFmt: 'static {
	74	/// The value representing this format.
	75	const FORMAT: io_pgtable_fmt;
	76	}
	77
	78	impl<F: IoPageTableFmt> IoPageTable<F> {
	79	/// Create a new `IoPageTable` as a device resource.
	80	#[inline]
	81	pub fn new(
	82	dev: &Device<Bound>,
	83	config: Config,
	84	) -> impl PinInit<Devres<IoPageTable<F>>, Error> + '_ {
	85	// SAFETY: Devres ensures that the value is dropped during device unbind.
	86	Devres::new(dev, unsafe { Self::new_raw(dev, config) })
	87	}
	88
	89	/// Create a new `IoPageTable`.
	90	///
	91	/// # Safety
	92	///
	93	/// If successful, then the returned `IoPageTable` must be dropped before the device is
	94	/// unbound.
	95	#[inline]
	96	pub unsafe fn new_raw(dev: &Device<Bound>, config: Config) -> Result<IoPageTable<F>> {
	97	let mut raw_cfg = bindings::io_pgtable_cfg {
	98	quirks: config.quirks,
	99	pgsize_bitmap: config.pgsize_bitmap,
	100	ias: config.ias,
	101	oas: config.oas,
	102	coherent_walk: config.coherent_walk,
	103	tlb: &raw const NOOP_FLUSH_OPS,
	104	iommu_dev: dev.as_raw(),
	105	// SAFETY: All zeroes is a valid value for `struct io_pgtable_cfg`.
	106	..unsafe { core::mem::zeroed() }
	107	};
	108
	109	// SAFETY:
	110	// * The raw_cfg pointer is valid for the duration of this call.
	111	// * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer
	112	// as cookie.
	113	// * The caller ensures that the io pgtable does not outlive the device.
	114	let ops = unsafe {
	115	bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut())
	116	};
	117
	118	// INVARIANT: We successfully created a valid page table.
	119	Ok(IoPageTable {
	120	ptr: NonNull::new(ops).ok_or(ENOMEM)?,
	121	_marker: PhantomData,
	122	})
	123	}
	124
	125	/// Obtain a raw pointer to the underlying `struct io_pgtable_ops`.
	126	#[inline]
	127	pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops {
	128	self.ptr.as_ptr()
	129	}
	130
	131	/// Obtain a raw pointer to the underlying `struct io_pgtable`.
	132	#[inline]
	133	pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable {
	134	// SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable.
	135	unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) }
	136	}
	137
	138	/// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`.
	139	#[inline]
	140	pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg {
	141	// SAFETY: The `raw_pgtable()` method returns a valid pointer.
	142	unsafe { &raw mut (*self.raw_pgtable()).cfg }
	143	}
	144
	145	/// Map a physically contiguous range of pages of the same size.
	146	///
	147	/// Even if successful, this operation may not map the entire range. In that case, only a
	148	/// prefix of the range is mapped, and the returned integer indicates its length in bytes. In
	149	/// this case, the caller will usually call `map_pages` again for the remaining range.
	150	///
	151	/// The returned [`Result`] indicates whether an error was encountered while mapping pages.
	152	/// Note that this may return a non-zero length even if an error was encountered. The caller
	153	/// will usually [unmap the relevant pages](Self::unmap_pages) on error.
	154	///
	155	/// The caller must flush the TLB before using the pgtable to access the newly created mapping.
	156	///
	157	/// # Safety
	158	///
	159	/// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
	160	/// this `map_pages` operation executes.
	161	/// * This page table must not contain any mapping that overlaps with the mapping created by
	162	/// this call.
	163	/// * If this page table is live, then the caller must ensure that it's okay to access the
	164	/// physical address being mapped for the duration in which it is mapped.
	165	#[inline]
	166	pub unsafe fn map_pages(
	167	&self,
	168	iova: usize,
	169	paddr: PhysAddr,
	170	pgsize: usize,
	171	pgcount: usize,
	172	prot: u32,
	173	flags: alloc::Flags,
	174	) -> (usize, Result) {
	175	let mut mapped: usize = 0;
	176
	177	// SAFETY: The `map_pages` function in `io_pgtable_ops` is never null.
	178	let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() };
	179
	180	// SAFETY: The safety requirements of this method are sufficient to call `map_pages`.
	181	let ret = to_result(unsafe {
	182	(map_pages)(
	183	self.raw_ops(),
	184	iova,
	185	paddr,
	186	pgsize,
	187	pgcount,
	188	prot as i32,
	189	flags.as_raw(),
	190	&mut mapped,
	191	)
	192	});
	193
	194	(mapped, ret)
	195	}
	196
	197	/// Unmap a range of virtually contiguous pages of the same size.
	198	///
	199	/// This may not unmap the entire range, and returns the length of the unmapped prefix in
	200	/// bytes.
	201	///
	202	/// # Safety
	203	///
	204	/// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
	205	/// this `unmap_pages` operation executes.
	206	/// * This page table must contain one or more consecutive mappings starting at `iova` whose
	207	/// total size is `pgcount * pgsize`.
	208	#[inline]
	209	#[must_use]
	210	pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize {
	211	// SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null.
	212	let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() };
	213
	214	// SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`.
	215	unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) }
	216	}
	217	}
	218
	219	// For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all
	220	// required invalidations using a range. There is no need for it get ARM style invalidation
	221	// instructions from the page table code.
	222	//
	223	// Support for flushing the TLB with ARM style invalidation instructions may be added in the
	224	// future.
	225	static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops {
	226	tlb_flush_all: Some(rust_tlb_flush_all_noop),
	227	tlb_flush_walk: Some(rust_tlb_flush_walk_noop),
	228	tlb_add_page: None,
	229	};
	230
	231	#[no_mangle]
	232	extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {}
	233
	234	#[no_mangle]
	235	extern "C" fn rust_tlb_flush_walk_noop(
	236	_iova: usize,
	237	_size: usize,
	238	_granule: usize,
	239	_cookie: *mut core::ffi::c_void,
	240	) {
	241	}
	242
	243	impl<F: IoPageTableFmt> Drop for IoPageTable<F> {
	244	fn drop(&mut self) {
	245	// SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this
	246	// destructor runs.
	247	unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) };
	248	}
	249	}
	250
	251	/// The `ARM_64_LPAE_S1` page table format.
	252	pub enum ARM64LPAES1 {}
	253
	254	impl IoPageTableFmt for ARM64LPAES1 {
	255	const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt;
	256	}
	257
	258	impl IoPageTable<ARM64LPAES1> {
	259	/// Access the `ttbr` field of the configuration.
	260	///
	261	/// This is the physical address of the page table, which may be passed to the device that
	262	/// needs to use it.
	263	///
	264	/// # Safety
	265	///
	266	/// The caller must ensure that the device stops using the page table before dropping it.
	267	#[inline]
	268	pub unsafe fn ttbr(&self) -> u64 {
	269	// SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
	270	unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr }
	271	}
	272
	273	/// Access the `mair` field of the configuration.
	274	#[inline]
	275	pub fn mair(&self) -> u64 {
	276	// SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
	277	unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair }
	278	}
	279	}

rust/kernel/lib.rs+1−0

@@ -103,6 +103,7 @@ pub mod id_pool; · 102 unmodified lines
103	pub mod init;	103	pub mod init;
104	pub mod io;	104	pub mod io;
105	pub mod ioctl;	105	pub mod ioctl;
		106	pub mod iommu;
106	pub mod iov;	107	pub mod iov;
107	pub mod irq;	108	pub mod irq;
108	pub mod jump_label;	109	pub mod jump_label;