History

Merge branches 'fixes', 'arm/smmu/updates', 'intel/vt-d', 'amd/amd-vi' and 'core' into next

Joerg Roedel committed 9 months ago

Commit ad095635 parents 02f9d76 a45dd34 c3b1ede 9e249c4 b48ca9249 files · +1996 −498Browse files

Unified · Split

49 files
  1. Documentation/admin-guide/kernel-parameters.txt +9 −0
  2. MAINTAINERS +1 −0
  3. drivers/iommu/Kconfig +19 −0
  4. drivers/iommu/Makefile +1 −0
  5. drivers/iommu/amd/Kconfig +10 −0
  6. drivers/iommu/amd/Makefile +1 −0
  7. drivers/iommu/amd/amd_iommu.h +33 −0
  8. drivers/iommu/amd/amd_iommu_types.h +48 −2
  9. drivers/iommu/amd/init.c +10 −2
  10. drivers/iommu/amd/iommu.c +190 −117
  11. drivers/iommu/amd/iommufd.c +77 −0
  12. drivers/iommu/amd/iommufd.h +20 −0
  13. drivers/iommu/amd/nested.c +294 −0
  14. drivers/iommu/arm/Kconfig +0 −1
  15. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +3 −1
  16. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +75 −3
  17. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +120 −33
  18. drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +4 −2
  19. drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +12 −72
  20. drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +28 −0
  21. drivers/iommu/arm/arm-smmu/qcom_iommu.c +2 −6
  22. drivers/iommu/dma-iommu.c +1 −3
  23. drivers/iommu/dma-iommu.h +3 −2
  24. drivers/iommu/generic_pt/fmt/amdv1.h +2 −1
  25. drivers/iommu/generic_pt/fmt/x86_64.h +2 −1
  26. drivers/iommu/generic_pt/iommu_pt.h +1 −2
  27. drivers/iommu/intel/cache.c +8 −1
  28. drivers/iommu/intel/iommu.c +15 −18
  29. drivers/iommu/intel/iommu.h +20 −1
  30. drivers/iommu/intel/nested.c +4 −5
  31. drivers/iommu/intel/pasid.c +22 −190
  32. drivers/iommu/intel/pasid.h +14 −14
  33. drivers/iommu/iommu-debug-pagealloc.c +164 −0
  34. drivers/iommu/iommu-priv.h +58 −0
  35. drivers/iommu/iommu.c +224 −10
  36. drivers/pci/pci-acpi.c +11 −2
  37. drivers/pci/pci.c +58 −7
  38. drivers/pci/quirks.c +18 −1
  39. include/linux/iommu-debug-pagealloc.h +32 −0
  40. include/linux/iommu.h +14 −0
  41. include/linux/mm.h +5 −0
  42. include/linux/page_ext.h +6 −0
  43. include/uapi/linux/iommufd.h +39 −0
  44. include/uapi/linux/vfio.h +4 −0
  45. mm/page_ext.c +27 −0
  46. rust/bindings/bindings_helper.h +2 −1
  47. rust/kernel/iommu/mod.rs +5 −0
  48. rust/kernel/iommu/pgtable.rs +279 −0
  49. rust/kernel/lib.rs +1 −0
Documentation/admin-guide/kernel-parameters.txt+9−0
@@ -2675,6 +2675,15 @@ Kernel parameters · 2674 unmodified lines
26752675 1 - Bypass the IOMMU for DMA.
26762676 unset - Use value of CONFIG_IOMMU_DEFAULT_PASSTHROUGH.
26772677
2678+ iommu.debug_pagealloc=
2679+ [KNL,EARLY] When CONFIG_IOMMU_DEBUG_PAGEALLOC is set, this
2680+ parameter enables the feature at boot time. By default, it
2681+ is disabled and the system behaves the same way as a kernel
2682+ built without CONFIG_IOMMU_DEBUG_PAGEALLOC.
2683+ Format: { "0" | "1" }
2684+ 0 - Sanitizer disabled.
2685+ 1 - Sanitizer enabled, expect runtime overhead.
2686+
26782687 io7= [HW] IO7 for Marvel-based Alpha systems
26792688 See comment before marvel_specify_io7 in
26802689 arch/alpha/kernel/core_marvel.c.
MAINTAINERS+1−0
@@ -13251,6 +13251,7 @@ F: drivers/iommu/ · 13250 unmodified lines
1325113251 F: include/linux/iommu.h
1325213252 F: include/linux/iova.h
1325313253 F: include/linux/of_iommu.h
13254+F: rust/kernel/iommu/
1325413255
1325513256 IOMMUFD
1325613257 M: Jason Gunthorpe <jgg@nvidia.com>
drivers/iommu/Kconfig+19−0
@@ -384,6 +384,25 @@ config SPRD_IOMMU · 383 unmodified lines
384384
385385 Say Y here if you want to use the multimedia devices listed above.
386386
387+config IOMMU_DEBUG_PAGEALLOC
388+ bool "Debug IOMMU mappings against page allocations"
389+ depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION
390+ help
391+ This enables a consistency check between the kernel page allocator and
392+ the IOMMU subsystem. It verifies that pages being allocated or freed
393+ are not currently mapped in any IOMMU domain.
394+
395+ This helps detect DMA use-after-free bugs where a driver frees a page
396+ but forgets to unmap it from the IOMMU, potentially allowing a device
397+ to overwrite memory that the kernel has repurposed.
398+
399+ These checks are best-effort and may not detect all problems.
400+
401+ Due to performance overhead, this feature is disabled by default.
402+ You must enable "iommu.debug_pagealloc" from the kernel command
403+ line to activate the runtime checks.
404+
405+ If unsure, say N.
387406 endif # IOMMU_SUPPORT
388407
389408 source "drivers/iommu/generic_pt/Kconfig"
drivers/iommu/Makefile+1−0
@@ -36,3 +36,4 @@ obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o · 35 unmodified lines
3636 obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o
3737 obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
3838 obj-$(CONFIG_APPLE_DART) += apple-dart.o
39+obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o
drivers/iommu/amd/Kconfig+10−0
@@ -30,6 +30,16 @@ config AMD_IOMMU · 29 unmodified lines
3030 your BIOS for an option to enable it or if you have an IVRS ACPI
3131 table.
3232
33+config AMD_IOMMU_IOMMUFD
34+ bool "Enable IOMMUFD features for AMD IOMMU (EXPERIMENTAL)"
35+ depends on IOMMUFD
36+ depends on AMD_IOMMU
37+ help
38+ Support for IOMMUFD features intended to support virtual machines
39+ with accelerated virtual IOMMUs.
40+
41+ Say Y here if you are doing development and testing on this feature.
42+
3343 config AMD_IOMMU_DEBUGFS
3444 bool "Enable AMD IOMMU internals in DebugFS"
3545 depends on AMD_IOMMU && IOMMU_DEBUGFS
drivers/iommu/amd/Makefile+1−0
@@ -1,3 +1,4 @@
11 # SPDX-License-Identifier: GPL-2.0-only
22 obj-y += iommu.o init.o quirks.o ppr.o pasid.o
3+obj-$(CONFIG_AMD_IOMMU_IOMMUFD) += iommufd.o nested.o
34 obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o
drivers/iommu/amd/amd_iommu.h+33−0
@@ -190,4 +190,37 @@ void amd_iommu_domain_set_pgtable(struct protection_domain *domain, · 189 unmodified lines
190190 struct dev_table_entry *get_dev_table(struct amd_iommu *iommu);
191191 struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid);
192192
193+void amd_iommu_set_dte_v1(struct iommu_dev_data *dev_data,
194+ struct protection_domain *domain, u16 domid,
195+ struct pt_iommu_amdv1_hw_info *pt_info,
196+ struct dev_table_entry *new);
197+void amd_iommu_update_dte(struct amd_iommu *iommu,
198+ struct iommu_dev_data *dev_data,
199+ struct dev_table_entry *new);
200+
201+static inline void
202+amd_iommu_make_clear_dte(struct iommu_dev_data *dev_data, struct dev_table_entry *new)
203+{
204+ struct dev_table_entry *initial_dte;
205+ struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
206+
207+ /* All existing DTE must have V bit set */
208+ new->data128[0] = DTE_FLAG_V;
209+ new->data128[1] = 0;
210+
211+ /*
212+ * Restore cached persistent DTE bits, which can be set by information
213+ * in IVRS table. See set_dev_entry_from_acpi().
214+ */
215+ initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid);
216+ if (initial_dte) {
217+ new->data128[0] |= initial_dte->data128[0];
218+ new->data128[1] |= initial_dte->data128[1];
219+ }
220+}
221+
222+/* NESTED */
223+struct iommu_domain *
224+amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags,
225+ const struct iommu_user_data *user_data);
193226 #endif /* AMD_IOMMU_H */
drivers/iommu/amd/amd_iommu_types.h+48−2
@@ -17,9 +17,12 @@ · 16 unmodified lines
1717 #include <linux/list.h>
1818 #include <linux/spinlock.h>
1919 #include <linux/pci.h>
20+#include <linux/iommufd.h>
2021 #include <linux/irqreturn.h>
2122 #include <linux/generic_pt/iommu.h>
2223
24+#include <uapi/linux/iommufd.h>
25+
2326 /*
2427 * Maximum number of IOMMUs supported
2528 */
@@ -108,6 +111,7 @@ · 82 unmodified lines
108111
109112 /* Extended Feature 2 Bits */
110113 #define FEATURE_SEVSNPIO_SUP BIT_ULL(1)
114+#define FEATURE_GCR3TRPMODE BIT_ULL(3)
111115 #define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5)
112116 #define FEATURE_SNPAVICSUP_GAM(x) \
113117 (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
@@ -186,6 +190,7 @@ · 72 unmodified lines
186190 #define CONTROL_EPH_EN 45
187191 #define CONTROL_XT_EN 50
188192 #define CONTROL_INTCAPXT_EN 51
193+#define CONTROL_GCR3TRPMODE 58
189194 #define CONTROL_IRTCACHEDIS 59
190195 #define CONTROL_SNPAVIC_EN 61
191196
@@ -350,6 +355,9 @@ · 158 unmodified lines
350355 #define DTE_FLAG_V BIT_ULL(0)
351356 #define DTE_FLAG_TV BIT_ULL(1)
352357 #define DTE_FLAG_HAD (3ULL << 7)
358+#define DTE_MODE_MASK GENMASK_ULL(11, 9)
359+#define DTE_HOST_TRP GENMASK_ULL(51, 12)
360+#define DTE_FLAG_PPR BIT_ULL(52)
353361 #define DTE_FLAG_GIOV BIT_ULL(54)
354362 #define DTE_FLAG_GV BIT_ULL(55)
355363 #define DTE_GLX GENMASK_ULL(57, 56)
@@ -358,7 +366,7 @@ · 2 unmodified lines
358366
359367 #define DTE_FLAG_IOTLB BIT_ULL(32)
360368 #define DTE_FLAG_MASK (0x3ffULL << 32)
361−#define DEV_DOMID_MASK 0xffffULL
369+#define DTE_DOMID_MASK GENMASK_ULL(15, 0)
362370
363371 #define DTE_GCR3_14_12 GENMASK_ULL(60, 58)
364372 #define DTE_GCR3_30_15 GENMASK_ULL(31, 16)
@@ -493,6 +501,38 @@ struct pdom_iommu_info { · 128 unmodified lines
493501 u32 refcnt; /* Count of attached dev/pasid per domain/IOMMU */
494502 };
495503
504+struct amd_iommu_viommu {
505+ struct iommufd_viommu core;
506+ struct protection_domain *parent; /* nest parent domain for this viommu */
507+ struct list_head pdom_list; /* For protection_domain->viommu_list */
508+
509+ /*
510+ * Per-vIOMMU guest domain ID to host domain ID mapping.
511+ * Indexed by guest domain ID.
512+ */
513+ struct xarray gdomid_array;
514+};
515+
516+/*
517+ * Contains guest domain ID mapping info,
518+ * which is stored in the struct xarray gdomid_array.
519+ */
520+struct guest_domain_mapping_info {
521+ refcount_t users;
522+ u32 hdom_id; /* Host domain ID */
523+};
524+
525+/*
526+ * Nested domain is specifically used for nested translation
527+ */
528+struct nested_domain {
529+ struct iommu_domain domain; /* generic domain handle used by iommu core code */
530+ u16 gdom_id; /* domain ID from gDTE */
531+ struct guest_domain_mapping_info *gdom_info;
532+ struct iommu_hwpt_amd_guest gdte; /* Guest vIOMMU DTE */
533+ struct amd_iommu_viommu *viommu; /* AMD hw-viommu this nested domain belong to */
534+};
535+
496536 /*
497537 * This structure contains generic data for IOMMU protection domains
498538 * independent of their use.
@@ -513,6 +553,12 @@ struct protection_domain { · 14 unmodified lines
513553
514554 struct mmu_notifier mn; /* mmu notifier for the SVA domain */
515555 struct list_head dev_data_list; /* List of pdom_dev_data */
556+
557+ /*
558+ * Store reference to list of vIOMMUs, which use this protection domain.
559+ * This will be used to look up host domain ID when flushing this domain.
560+ */
561+ struct list_head viommu_list;
516562 };
517563 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain);
518564 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain);
@@ -706,7 +752,7 @@ struct amd_iommu { · 187 unmodified lines
706752
707753 u32 flags;
708754 volatile u64 *cmd_sem;
709− atomic64_t cmd_sem_val;
755+ u64 cmd_sem_val;
710756 /*
711757 * Track physical address to directly use it in build_completion_wait()
712758 * and avoid adding any special checks and handling for kdump.
drivers/iommu/amd/init.c+10−2
@@ -1122,6 +1122,14 @@ static void iommu_enable_gt(struct amd_iommu *iommu) · 1121 unmodified lines
11221122 return;
11231123
11241124 iommu_feature_enable(iommu, CONTROL_GT_EN);
1125+
1126+ /*
1127+ * This feature needs to be enabled prior to a call
1128+ * to iommu_snp_enable(). Since this function is called
1129+ * in early_enable_iommu(), it is safe to enable here.
1130+ */
1131+ if (check_feature2(FEATURE_GCR3TRPMODE))
1132+ iommu_feature_enable(iommu, CONTROL_GCR3TRPMODE);
11251133 }
11261134
11271135 /* sets a specific bit in the device table entry. */
@@ -1179,7 +1187,7 @@ static bool __reuse_device_table(struct amd_iommu *iommu) · 51 unmodified lines
11791187 for (devid = 0; devid <= pci_seg->last_bdf; devid++) {
11801188 old_dev_tbl_entry = &pci_seg->old_dev_tbl_cpy[devid];
11811189 dte_v = FIELD_GET(DTE_FLAG_V, old_dev_tbl_entry->data[0]);
1182− dom_id = FIELD_GET(DEV_DOMID_MASK, old_dev_tbl_entry->data[1]);
1190+ dom_id = FIELD_GET(DTE_DOMID_MASK, old_dev_tbl_entry->data[1]);
11831191
11841192 if (!dte_v || !dom_id)
11851193 continue;
@@ -1877,7 +1885,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, · 691 unmodified lines
18771885 iommu->pci_seg = pci_seg;
18781886
18791887 raw_spin_lock_init(&iommu->lock);
1880− atomic64_set(&iommu->cmd_sem_val, 0);
1888+ iommu->cmd_sem_val = 0;
18811889
18821890 /* Add IOMMU to internal data structures */
18831891 list_add_tail(&iommu->list, &amd_iommu_list);
drivers/iommu/amd/iommu.c+190−117
@@ -43,6 +43,7 @@ · 42 unmodified lines
4343 #include <linux/generic_pt/iommu.h>
4444
4545 #include "amd_iommu.h"
46+#include "iommufd.h"
4647 #include "../irq_remapping.h"
4748 #include "../iommu-pages.h"
4849
@@ -75,6 +76,8 @@ static void set_dte_entry(struct amd_iommu *iommu, · 26 unmodified lines
7576 struct iommu_dev_data *dev_data,
7677 phys_addr_t top_paddr, unsigned int top_level);
7778
79+static int device_flush_dte(struct iommu_dev_data *dev_data);
80+
7881 static void amd_iommu_change_top(struct pt_iommu *iommu_table,
7982 phys_addr_t top_paddr, unsigned int top_level);
8083
@@ -85,6 +88,10 @@ static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain); · 4 unmodified lines
8588 static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain,
8689 bool enable);
8790
91+static void clone_aliases(struct amd_iommu *iommu, struct device *dev);
92+
93+static int iommu_completion_wait(struct amd_iommu *iommu);
94+
8895 /****************************************************************************
8996 *
9097 * Helper functions
@@ -202,6 +209,16 @@ static void update_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_da · 111 unmodified lines
202209 spin_unlock_irqrestore(&dev_data->dte_lock, flags);
203210 }
204211
212+void amd_iommu_update_dte(struct amd_iommu *iommu,
213+ struct iommu_dev_data *dev_data,
214+ struct dev_table_entry *new)
215+{
216+ update_dte256(iommu, dev_data, new);
217+ clone_aliases(iommu, dev_data->dev);
218+ device_flush_dte(dev_data);
219+ iommu_completion_wait(iommu);
220+}
221+
205222 static void get_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data,
206223 struct dev_table_entry *dte)
207224 {
@@ -1185,7 +1202,12 @@ static int wait_on_sem(struct amd_iommu *iommu, u64 data) · 977 unmodified lines
11851202 {
11861203 int i = 0;
11871204
1188− while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) {
1205+ /*
1206+ * cmd_sem holds a monotonically non-decreasing completion sequence
1207+ * number.
1208+ */
1209+ while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 &&
1210+ i < LOOP_TIMEOUT) {
11891211 udelay(1);
11901212 i += 1;
11911213 }
@@ -1417,6 +1439,12 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) · 225 unmodified lines
14171439 return iommu_queue_command_sync(iommu, cmd, true);
14181440 }
14191441
1442+static u64 get_cmdsem_val(struct amd_iommu *iommu)
1443+{
1444+ lockdep_assert_held(&iommu->lock);
1445+ return ++iommu->cmd_sem_val;
1446+}
1447+
14201448 /*
14211449 * This function queues a completion wait command into the command
14221450 * buffer of an IOMMU
@@ -1431,20 +1459,19 @@ static int iommu_completion_wait(struct amd_iommu *iommu) · 8 unmodified lines
14311459 if (!iommu->need_sync)
14321460 return 0;
14331461
1434− data = atomic64_inc_return(&iommu->cmd_sem_val);
1435− build_completion_wait(&cmd, iommu, data);
1436−
14371462 raw_spin_lock_irqsave(&iommu->lock, flags);
14381463
1464+ data = get_cmdsem_val(iommu);
1465+ build_completion_wait(&cmd, iommu, data);
1466+
14391467 ret = __iommu_queue_command_sync(iommu, &cmd, false);
1468+ raw_spin_unlock_irqrestore(&iommu->lock, flags);
1469+
14401470 if (ret)
1441− goto out_unlock;
1471+ return ret;
14421472
14431473 ret = wait_on_sem(iommu, data);
14441474
1445−out_unlock:
1446− raw_spin_unlock_irqrestore(&iommu->lock, flags);
1447−
14481475 return ret;
14491476 }
14501477
@@ -1522,6 +1549,32 @@ static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id) · 71 unmodified lines
15221549 iommu_completion_wait(iommu);
15231550 }
15241551
1552+static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 address, size_t size)
1553+{
1554+ int ret = 0;
1555+ struct amd_iommu_viommu *aviommu;
1556+
1557+ list_for_each_entry(aviommu, &pdom->viommu_list, pdom_list) {
1558+ unsigned long i;
1559+ struct guest_domain_mapping_info *gdom_info;
1560+ struct amd_iommu *iommu = container_of(aviommu->core.iommu_dev,
1561+ struct amd_iommu, iommu);
1562+
1563+ xa_lock(&aviommu->gdomid_array);
1564+ xa_for_each(&aviommu->gdomid_array, i, gdom_info) {
1565+ struct iommu_cmd cmd;
1566+
1567+ pr_debug("%s: iommu=%#x, hdom_id=%#x\n", __func__,
1568+ iommu->devid, gdom_info->hdom_id);
1569+ build_inv_iommu_pages(&cmd, address, size, gdom_info->hdom_id,
1570+ IOMMU_NO_PASID, false);
1571+ ret |= iommu_queue_command(iommu, &cmd);
1572+ }
1573+ xa_unlock(&aviommu->gdomid_array);
1574+ }
1575+ return ret;
1576+}
1577+
15251578 static void amd_iommu_flush_all(struct amd_iommu *iommu)
15261579 {
15271580 struct iommu_cmd cmd;
@@ -1670,6 +1723,17 @@ static int domain_flush_pages_v1(struct protection_domain *pdom, · 142 unmodified lines
16701723 ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd);
16711724 }
16721725
1726+ /*
1727+ * A domain w/ v1 table can be a nest parent, which can have
1728+ * multiple nested domains. Each nested domain has 1:1 mapping
1729+ * between gDomID and hDomID. Therefore, flush every hDomID
1730+ * associated to this nest parent domain.
1731+ *
1732+ * See drivers/iommu/amd/nested.c: amd_iommu_alloc_domain_nested()
1733+ */
1734+ if (!list_empty(&pdom->viommu_list))
1735+ ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, size);
1736+
16731737 return ret;
16741738 }
16751739
@@ -2010,127 +2074,112 @@ int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid) · 334 unmodified lines
20102074 return ret;
20112075 }
20122076
2013−static void make_clear_dte(struct iommu_dev_data *dev_data, struct dev_table_entry *ptr,
2014− struct dev_table_entry *new)
2015−{
2016− /* All existing DTE must have V bit set */
2017− new->data128[0] = DTE_FLAG_V;
2018− new->data128[1] = 0;
2019−}
2020−
20212077 /*
20222078 * Note:
20232079 * The old value for GCR3 table and GPT have been cleared from caller.
20242080 */
2025−static void set_dte_gcr3_table(struct amd_iommu *iommu,
2026− struct iommu_dev_data *dev_data,
2027− struct dev_table_entry *target)
2081+static void set_dte_gcr3_table(struct iommu_dev_data *dev_data,
2082+ struct dev_table_entry *new)
20282083 {
20292084 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
2030− u64 gcr3;
2031−
2032− if (!gcr3_info->gcr3_tbl)
2033− return;
2034−
2035− pr_debug("%s: devid=%#x, glx=%#x, gcr3_tbl=%#llx\n",
2036− __func__, dev_data->devid, gcr3_info->glx,
2037− (unsigned long long)gcr3_info->gcr3_tbl);
2085+ u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl);
20382086
2039− gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl);
2087+ new->data[0] |= DTE_FLAG_TV |
2088+ (dev_data->ppr ? DTE_FLAG_PPR : 0) |
2089+ (pdom_is_v2_pgtbl_mode(dev_data->domain) ? DTE_FLAG_GIOV : 0) |
2090+ DTE_FLAG_GV |
2091+ FIELD_PREP(DTE_GLX, gcr3_info->glx) |
2092+ FIELD_PREP(DTE_GCR3_14_12, gcr3 >> 12) |
2093+ DTE_FLAG_IR | DTE_FLAG_IW;
20402094
2041− target->data[0] |= DTE_FLAG_GV |
2042− FIELD_PREP(DTE_GLX, gcr3_info->glx) |
2043− FIELD_PREP(DTE_GCR3_14_12, gcr3 >> 12);
2044− if (pdom_is_v2_pgtbl_mode(dev_data->domain))
2045− target->data[0] |= DTE_FLAG_GIOV;
2046−
2047− target->data[1] |= FIELD_PREP(DTE_GCR3_30_15, gcr3 >> 15) |
2048− FIELD_PREP(DTE_GCR3_51_31, gcr3 >> 31);
2095+ new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, dev_data->gcr3_info.domid) |
2096+ FIELD_PREP(DTE_GCR3_30_15, gcr3 >> 15) |
2097+ (dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0) |
2098+ FIELD_PREP(DTE_GCR3_51_31, gcr3 >> 31);
20492099
20502100 /* Guest page table can only support 4 and 5 levels */
20512101 if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL)
2052− target->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_5_LEVEL);
2102+ new->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_5_LEVEL);
20532103 else
2054− target->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_4_LEVEL);
2104+ new->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_4_LEVEL);
20552105 }
20562106
2057−static void set_dte_entry(struct amd_iommu *iommu,
2058− struct iommu_dev_data *dev_data,
2059− phys_addr_t top_paddr, unsigned int top_level)
2107+void amd_iommu_set_dte_v1(struct iommu_dev_data *dev_data,
2108+ struct protection_domain *domain, u16 domid,
2109+ struct pt_iommu_amdv1_hw_info *pt_info,
2110+ struct dev_table_entry *new)
20602111 {
2061− u16 domid;
2062− u32 old_domid;
2063− struct dev_table_entry *initial_dte;
2064− struct dev_table_entry new = {};
2065− struct protection_domain *domain = dev_data->domain;
2066− struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
2067− struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];
2068− struct pt_iommu_amdv1_hw_info pt_info;
2112+ u64 host_pt_root = __sme_set(pt_info->host_pt_root);
20692113
2070− make_clear_dte(dev_data, dte, &new);
2071−
2072− if (gcr3_info && gcr3_info->gcr3_tbl)
2073− domid = dev_data->gcr3_info.domid;
2074− else {
2075− domid = domain->id;
2076−
2077− if (domain->domain.type & __IOMMU_DOMAIN_PAGING) {
2078− /*
2079− * When updating the IO pagetable, the new top and level
2080− * are provided as parameters. For other operations i.e.
2081− * device attach, retrieve the current pagetable info
2082− * via the IOMMU PT API.
2083− */
2084− if (top_paddr) {
2085− pt_info.host_pt_root = top_paddr;
2086− pt_info.mode = top_level + 1;
2087− } else {
2088− WARN_ON(top_paddr || top_level);
2089− pt_iommu_amdv1_hw_info(&domain->amdv1,
2090− &pt_info);
2091− }
2114+ /* Note Dirty tracking is used for v1 table only for now */
2115+ new->data[0] |= DTE_FLAG_TV |
2116+ FIELD_PREP(DTE_MODE_MASK, pt_info->mode) |
2117+ (domain->dirty_tracking ? DTE_FLAG_HAD : 0) |
2118+ FIELD_PREP(DTE_HOST_TRP, host_pt_root >> 12) |
2119+ DTE_FLAG_IR | DTE_FLAG_IW;
20922120
2093− new.data[0] |= __sme_set(pt_info.host_pt_root) |
2094− (pt_info.mode & DEV_ENTRY_MODE_MASK)
2095− << DEV_ENTRY_MODE_SHIFT;
2096− }
2097− }
2121+ new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, domid) |
2122+ (dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0);
2123+}
20982124
2099− new.data[0] |= DTE_FLAG_IR | DTE_FLAG_IW;
2125+static void set_dte_v1(struct iommu_dev_data *dev_data,
2126+ struct protection_domain *domain, u16 domid,
2127+ phys_addr_t top_paddr, unsigned int top_level,
2128+ struct dev_table_entry *new)
2129+{
2130+ struct pt_iommu_amdv1_hw_info pt_info;
21002131
21012132 /*
2102− * When SNP is enabled, we can only support TV=1 with non-zero domain ID.
2103− * This is prevented by the SNP-enable and IOMMU_DOMAIN_IDENTITY check in
2104− * do_iommu_domain_alloc().
2133+ * When updating the IO pagetable, the new top and level
2134+ * are provided as parameters. For other operations i.e.
2135+ * device attach, retrieve the current pagetable info
2136+ * via the IOMMU PT API.
21052137 */
2106− WARN_ON(amd_iommu_snp_en && (domid == 0));
2107− new.data[0] |= DTE_FLAG_TV;
2138+ if (top_paddr) {
2139+ pt_info.host_pt_root = top_paddr;
2140+ pt_info.mode = top_level + 1;
2141+ } else {
2142+ WARN_ON(top_paddr || top_level);
2143+ pt_iommu_amdv1_hw_info(&domain->amdv1, &pt_info);
2144+ }
21082145
2109− if (dev_data->ppr)
2110− new.data[0] |= 1ULL << DEV_ENTRY_PPR;
2146+ amd_iommu_set_dte_v1(dev_data, domain, domid, &pt_info, new);
2147+}
21112148
2112− if (domain->dirty_tracking)
2113− new.data[0] |= DTE_FLAG_HAD;
2149+static void set_dte_passthrough(struct iommu_dev_data *dev_data,
2150+ struct protection_domain *domain,
2151+ struct dev_table_entry *new)
2152+{
2153+ new->data[0] |= DTE_FLAG_TV | DTE_FLAG_IR | DTE_FLAG_IW;
21142154
2115− if (dev_data->ats_enabled)
2116− new.data[1] |= DTE_FLAG_IOTLB;
2155+ new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, domain->id) |
2156+ (dev_data->ats_enabled) ? DTE_FLAG_IOTLB : 0;
2157+}
21172158
2118− old_domid = READ_ONCE(dte->data[1]) & DEV_DOMID_MASK;
2119− new.data[1] |= domid;
2159+static void set_dte_entry(struct amd_iommu *iommu,
2160+ struct iommu_dev_data *dev_data,
2161+ phys_addr_t top_paddr, unsigned int top_level)
2162+{
2163+ u32 old_domid;
2164+ struct dev_table_entry new = {};
2165+ struct protection_domain *domain = dev_data->domain;
2166+ struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
2167+ struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];
21202168
2121− /*
2122− * Restore cached persistent DTE bits, which can be set by information
2123− * in IVRS table. See set_dev_entry_from_acpi().
2124− */
2125− initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid);
2126− if (initial_dte) {
2127− new.data128[0] |= initial_dte->data128[0];
2128− new.data128[1] |= initial_dte->data128[1];
2129− }
2169+ amd_iommu_make_clear_dte(dev_data, &new);
21302170
2131− set_dte_gcr3_table(iommu, dev_data, &new);
2171+ old_domid = READ_ONCE(dte->data[1]) & DTE_DOMID_MASK;
2172+ if (gcr3_info->gcr3_tbl)
2173+ set_dte_gcr3_table(dev_data, &new);
2174+ else if (domain->domain.type == IOMMU_DOMAIN_IDENTITY)
2175+ set_dte_passthrough(dev_data, domain, &new);
2176+ else if ((domain->domain.type & __IOMMU_DOMAIN_PAGING) &&
2177+ domain->pd_mode == PD_MODE_V1)
2178+ set_dte_v1(dev_data, domain, domain->id, top_paddr, top_level, &new);
2179+ else
2180+ WARN_ON(true);
21322181
2133− update_dte256(iommu, dev_data, &new);
2182+ amd_iommu_update_dte(iommu, dev_data, &new);
21342183
21352184 /*
21362185 * A kdump kernel might be replacing a domain ID that was copied from
@@ -2148,10 +2197,9 @@ static void set_dte_entry(struct amd_iommu *iommu, · 11 unmodified lines
21482197 static void clear_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data)
21492198 {
21502199 struct dev_table_entry new = {};
2151− struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];
21522200
2153− make_clear_dte(dev_data, dte, &new);
2154− update_dte256(iommu, dev_data, &new);
2201+ amd_iommu_make_clear_dte(dev_data, &new);
2202+ amd_iommu_update_dte(iommu, dev_data, &new);
21552203 }
21562204
21572205 /* Update and flush DTE for the given device */
@@ -2163,10 +2211,6 @@ static void dev_update_dte(struct iommu_dev_data *dev_data, bool set) · 5 unmodified lines
21632211 set_dte_entry(iommu, dev_data, 0, 0);
21642212 else
21652213 clear_dte_entry(iommu, dev_data);
2166−
2167− clone_aliases(iommu, dev_data->dev);
2168− device_flush_dte(dev_data);
2169− iommu_completion_wait(iommu);
21702214 }
21712215
21722216 /*
@@ -2499,6 +2543,7 @@ static void protection_domain_init(struct protection_domain *domain) · 326 unmodified lines
24992543 spin_lock_init(&domain->lock);
25002544 INIT_LIST_HEAD(&domain->dev_list);
25012545 INIT_LIST_HEAD(&domain->dev_data_list);
2546+ INIT_LIST_HEAD(&domain->viommu_list);
25022547 xa_init(&domain->iommu_array);
25032548 }
25042549
@@ -2760,6 +2805,14 @@ static struct iommu_domain *amd_iommu_domain_alloc_paging_v2(struct device *dev, · 255 unmodified lines
27602805 return &domain->domain;
27612806 }
27622807
2808+static inline bool is_nest_parent_supported(u32 flags)
2809+{
2810+ /* Only allow nest parent when these features are supported */
2811+ return check_feature(FEATURE_GT) &&
2812+ check_feature(FEATURE_GIOSUP) &&
2813+ check_feature2(FEATURE_GCR3TRPMODE);
2814+}
2815+
27632816 static struct iommu_domain *
27642817 amd_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,
27652818 const struct iommu_user_data *user_data)
@@ -2767,16 +2820,28 @@ amd_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags, · 1 unmodified line
27672820 {
27682821 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
27692822 const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
2770− IOMMU_HWPT_ALLOC_PASID;
2823+ IOMMU_HWPT_ALLOC_PASID |
2824+ IOMMU_HWPT_ALLOC_NEST_PARENT;
27712825
27722826 if ((flags & ~supported_flags) || user_data)
27732827 return ERR_PTR(-EOPNOTSUPP);
27742828
27752829 switch (flags & supported_flags) {
27762830 case IOMMU_HWPT_ALLOC_DIRTY_TRACKING:
2777− /* Allocate domain with v1 page table for dirty tracking */
2778− if (!amd_iommu_hd_support(iommu))
2831+ case IOMMU_HWPT_ALLOC_NEST_PARENT:
2832+ case IOMMU_HWPT_ALLOC_DIRTY_TRACKING | IOMMU_HWPT_ALLOC_NEST_PARENT:
2833+ /*
2834+ * Allocate domain with v1 page table for dirty tracking
2835+ * and/or Nest parent.
2836+ */
2837+ if ((flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) &&
2838+ !amd_iommu_hd_support(iommu))
27792839 break;
2840+
2841+ if ((flags & IOMMU_HWPT_ALLOC_NEST_PARENT) &&
2842+ !is_nest_parent_supported(flags))
2843+ break;
2844+
27802845 return amd_iommu_domain_alloc_paging_v1(dev, flags);
27812846 case IOMMU_HWPT_ALLOC_PASID:
27822847 /* Allocate domain with v2 page table if IOMMU supports PASID. */
@@ -3078,6 +3143,7 @@ static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain) · 295 unmodified lines
30783143
30793144 const struct iommu_ops amd_iommu_ops = {
30803145 .capable = amd_iommu_capable,
3146+ .hw_info = amd_iommufd_hw_info,
30813147 .blocked_domain = &blocked_domain,
30823148 .release_domain = &blocked_domain,
30833149 .identity_domain = &identity_domain.domain,
@@ -3090,6 +3156,8 @@ const struct iommu_ops amd_iommu_ops = { · 6 unmodified lines
30903156 .is_attach_deferred = amd_iommu_is_attach_deferred,
30913157 .def_domain_type = amd_iommu_def_domain_type,
30923158 .page_response = amd_iommu_page_response,
3159+ .get_viommu_size = amd_iommufd_get_viommu_size,
3160+ .viommu_init = amd_iommufd_viommu_init,
30933161 };
30943162
30953163 #ifdef CONFIG_IRQ_REMAP
@@ -3114,18 +3182,23 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) · 18 unmodified lines
31143182 return;
31153183
31163184 build_inv_irt(&cmd, devid);
3117− data = atomic64_inc_return(&iommu->cmd_sem_val);
3118− build_completion_wait(&cmd2, iommu, data);
31193185
31203186 raw_spin_lock_irqsave(&iommu->lock, flags);
3187+ data = get_cmdsem_val(iommu);
3188+ build_completion_wait(&cmd2, iommu, data);
3189+
31213190 ret = __iommu_queue_command_sync(iommu, &cmd, true);
31223191 if (ret)
3123− goto out;
3192+ goto out_err;
31243193 ret = __iommu_queue_command_sync(iommu, &cmd2, false);
31253194 if (ret)
3126− goto out;
3195+ goto out_err;
3196+ raw_spin_unlock_irqrestore(&iommu->lock, flags);
3197+
31273198 wait_on_sem(iommu, data);
3128−out:
3199+ return;
3200+
3201+out_err:
31293202 raw_spin_unlock_irqrestore(&iommu->lock, flags);
31303203 }
31313204
@@ -3239,7 +3312,7 @@ static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu, · 107 unmodified lines
32393312 struct irq_remap_table *new_table = NULL;
32403313 struct amd_iommu_pci_seg *pci_seg;
32413314 unsigned long flags;
3242− int nid = iommu && iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
3315+ int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
32433316 u16 alias;
32443317
32453318 spin_lock_irqsave(&iommu_table_lock, flags);
drivers/iommu/amd/iommufd.c+77−0
@@ -0,0 +1,77 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
4+ */
5+
6+#include <linux/iommu.h>
7+
8+#include "iommufd.h"
9+#include "amd_iommu.h"
10+#include "amd_iommu_types.h"
11+
12+static const struct iommufd_viommu_ops amd_viommu_ops;
13+
14+void *amd_iommufd_hw_info(struct device *dev, u32 *length, enum iommu_hw_info_type *type)
15+{
16+ struct iommu_hw_info_amd *hwinfo;
17+
18+ if (*type != IOMMU_HW_INFO_TYPE_DEFAULT &&
19+ *type != IOMMU_HW_INFO_TYPE_AMD)
20+ return ERR_PTR(-EOPNOTSUPP);
21+
22+ hwinfo = kzalloc(sizeof(*hwinfo), GFP_KERNEL);
23+ if (!hwinfo)
24+ return ERR_PTR(-ENOMEM);
25+
26+ *length = sizeof(*hwinfo);
27+ *type = IOMMU_HW_INFO_TYPE_AMD;
28+
29+ hwinfo->efr = amd_iommu_efr;
30+ hwinfo->efr2 = amd_iommu_efr2;
31+
32+ return hwinfo;
33+}
34+
35+size_t amd_iommufd_get_viommu_size(struct device *dev, enum iommu_viommu_type viommu_type)
36+{
37+ return VIOMMU_STRUCT_SIZE(struct amd_iommu_viommu, core);
38+}
39+
40+int amd_iommufd_viommu_init(struct iommufd_viommu *viommu, struct iommu_domain *parent,
41+ const struct iommu_user_data *user_data)
42+{
43+ unsigned long flags;
44+ struct protection_domain *pdom = to_pdomain(parent);
45+ struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
46+
47+ xa_init_flags(&aviommu->gdomid_array, XA_FLAGS_ALLOC1);
48+ aviommu->parent = pdom;
49+
50+ viommu->ops = &amd_viommu_ops;
51+
52+ spin_lock_irqsave(&pdom->lock, flags);
53+ list_add(&aviommu->pdom_list, &pdom->viommu_list);
54+ spin_unlock_irqrestore(&pdom->lock, flags);
55+
56+ return 0;
57+}
58+
59+static void amd_iommufd_viommu_destroy(struct iommufd_viommu *viommu)
60+{
61+ unsigned long flags;
62+ struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
63+ struct protection_domain *pdom = aviommu->parent;
64+
65+ spin_lock_irqsave(&pdom->lock, flags);
66+ list_del(&aviommu->pdom_list);
67+ spin_unlock_irqrestore(&pdom->lock, flags);
68+ xa_destroy(&aviommu->gdomid_array);
69+}
70+
71+/*
72+ * See include/linux/iommufd.h
73+ * struct iommufd_viommu_ops - vIOMMU specific operations
74+ */
75+static const struct iommufd_viommu_ops amd_viommu_ops = {
76+ .destroy = amd_iommufd_viommu_destroy,
77+};
drivers/iommu/amd/iommufd.h+20−0
@@ -0,0 +1,20 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/*
3+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
4+ */
5+
6+#ifndef AMD_IOMMUFD_H
7+#define AMD_IOMMUFD_H
8+
9+#if IS_ENABLED(CONFIG_AMD_IOMMU_IOMMUFD)
10+void *amd_iommufd_hw_info(struct device *dev, u32 *length, enum iommu_hw_info_type *type);
11+size_t amd_iommufd_get_viommu_size(struct device *dev, enum iommu_viommu_type viommu_type);
12+int amd_iommufd_viommu_init(struct iommufd_viommu *viommu, struct iommu_domain *parent,
13+ const struct iommu_user_data *user_data);
14+#else
15+#define amd_iommufd_hw_info NULL
16+#define amd_iommufd_viommu_init NULL
17+#define amd_iommufd_get_viommu_size NULL
18+#endif /* CONFIG_AMD_IOMMU_IOMMUFD */
19+
20+#endif /* AMD_IOMMUFD_H */
drivers/iommu/amd/nested.c+294−0
@@ -0,0 +1,294 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
4+ */
5+
6+#define dev_fmt(fmt) "AMD-Vi: " fmt
7+
8+#include <linux/iommu.h>
9+#include <linux/refcount.h>
10+#include <uapi/linux/iommufd.h>
11+
12+#include "amd_iommu.h"
13+
14+static const struct iommu_domain_ops nested_domain_ops;
15+
16+static inline struct nested_domain *to_ndomain(struct iommu_domain *dom)
17+{
18+ return container_of(dom, struct nested_domain, domain);
19+}
20+
21+/*
22+ * Validate guest DTE to make sure that configuration for host (v1)
23+ * and guest (v2) page tables are valid when allocating nested domain.
24+ */
25+static int validate_gdte_nested(struct iommu_hwpt_amd_guest *gdte)
26+{
27+ u32 gpt_level = FIELD_GET(DTE_GPT_LEVEL_MASK, gdte->dte[2]);
28+
29+ /* Must be zero: Mode, Host-TPR */
30+ if (FIELD_GET(DTE_MODE_MASK, gdte->dte[0]) != 0 ||
31+ FIELD_GET(DTE_HOST_TRP, gdte->dte[0]) != 0)
32+ return -EINVAL;
33+
34+ /* GCR3 TRP must be non-zero if V, GV is set */
35+ if (FIELD_GET(DTE_FLAG_V, gdte->dte[0]) == 1 &&
36+ FIELD_GET(DTE_FLAG_GV, gdte->dte[0]) == 1 &&
37+ FIELD_GET(DTE_GCR3_14_12, gdte->dte[0]) == 0 &&
38+ FIELD_GET(DTE_GCR3_30_15, gdte->dte[1]) == 0 &&
39+ FIELD_GET(DTE_GCR3_51_31, gdte->dte[1]) == 0)
40+ return -EINVAL;
41+
42+ /* Valid Guest Paging Mode values are 0 and 1 */
43+ if (gpt_level != GUEST_PGTABLE_4_LEVEL &&
44+ gpt_level != GUEST_PGTABLE_5_LEVEL)
45+ return -EINVAL;
46+
47+ /* GLX = 3 is reserved */
48+ if (FIELD_GET(DTE_GLX, gdte->dte[0]) == 3)
49+ return -EINVAL;
50+
51+ /*
52+ * We need to check host capability before setting
53+ * the Guest Paging Mode
54+ */
55+ if (gpt_level == GUEST_PGTABLE_5_LEVEL &&
56+ amd_iommu_gpt_level < PAGE_MODE_5_LEVEL)
57+ return -EOPNOTSUPP;
58+
59+ return 0;
60+}
61+
62+static void *gdom_info_load_or_alloc_locked(struct xarray *xa, unsigned long index)
63+{
64+ struct guest_domain_mapping_info *elm, *res;
65+
66+ elm = xa_load(xa, index);
67+ if (elm)
68+ return elm;
69+
70+ xa_unlock(xa);
71+ elm = kzalloc(sizeof(struct guest_domain_mapping_info), GFP_KERNEL);
72+ xa_lock(xa);
73+ if (!elm)
74+ return ERR_PTR(-ENOMEM);
75+
76+ res = __xa_cmpxchg(xa, index, NULL, elm, GFP_KERNEL);
77+ if (xa_is_err(res))
78+ res = ERR_PTR(xa_err(res));
79+
80+ if (res) {
81+ kfree(elm);
82+ return res;
83+ }
84+
85+ refcount_set(&elm->users, 0);
86+ return elm;
87+}
88+
89+/*
90+ * This function is assigned to struct iommufd_viommu_ops.alloc_domain_nested()
91+ * during the call to struct iommu_ops.viommu_init().
92+ */
93+struct iommu_domain *
94+amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags,
95+ const struct iommu_user_data *user_data)
96+{
97+ int ret;
98+ struct nested_domain *ndom;
99+ struct guest_domain_mapping_info *gdom_info;
100+ struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
101+
102+ if (user_data->type != IOMMU_HWPT_DATA_AMD_GUEST)
103+ return ERR_PTR(-EOPNOTSUPP);
104+
105+ ndom = kzalloc(sizeof(*ndom), GFP_KERNEL);
106+ if (!ndom)
107+ return ERR_PTR(-ENOMEM);
108+
109+ ret = iommu_copy_struct_from_user(&ndom->gdte, user_data,
110+ IOMMU_HWPT_DATA_AMD_GUEST,
111+ dte);
112+ if (ret)
113+ goto out_err;
114+
115+ ret = validate_gdte_nested(&ndom->gdte);
116+ if (ret)
117+ goto out_err;
118+
119+ ndom->gdom_id = FIELD_GET(DTE_DOMID_MASK, ndom->gdte.dte[1]);
120+ ndom->domain.ops = &nested_domain_ops;
121+ ndom->domain.type = IOMMU_DOMAIN_NESTED;
122+ ndom->viommu = aviommu;
123+
124+ /*
125+ * Normally, when a guest has multiple pass-through devices,
126+ * the IOMMU driver setup DTEs with the same stage-2 table and
127+ * use the same host domain ID (hDomId). In case of nested translation,
128+ * if the guest setup different stage-1 tables with same PASID,
129+ * IOMMU would use the same TLB tag. This will results in TLB
130+ * aliasing issue.
131+ *
132+ * The guest is assigning gDomIDs based on its own algorithm for managing
133+ * cache tags of (DomID, PASID). Within a single viommu, the nest parent domain
134+ * (w/ S2 table) is used by all DTEs. But we need to consistently map the gDomID
135+ * to a single hDomID. This is done using an xarray in the vIOMMU to
136+ * keep track of the gDomID mapping. When the S2 is changed, the INVALIDATE_IOMMU_PAGES
137+ * command must be issued for each hDomID in the xarray.
138+ */
139+ xa_lock(&aviommu->gdomid_array);
140+
141+ gdom_info = gdom_info_load_or_alloc_locked(&aviommu->gdomid_array, ndom->gdom_id);
142+ if (IS_ERR(gdom_info)) {
143+ xa_unlock(&aviommu->gdomid_array);
144+ ret = PTR_ERR(gdom_info);
145+ goto out_err;
146+ }
147+
148+ /* Check if gDomID exist */
149+ if (refcount_inc_not_zero(&gdom_info->users)) {
150+ ndom->gdom_info = gdom_info;
151+ xa_unlock(&aviommu->gdomid_array);
152+
153+ pr_debug("%s: Found gdom_id=%#x, hdom_id=%#x\n",
154+ __func__, ndom->gdom_id, gdom_info->hdom_id);
155+
156+ return &ndom->domain;
157+ }
158+
159+ /* The gDomID does not exist. We allocate new hdom_id */
160+ gdom_info->hdom_id = amd_iommu_pdom_id_alloc();
161+ if (gdom_info->hdom_id <= 0) {
162+ __xa_cmpxchg(&aviommu->gdomid_array,
163+ ndom->gdom_id, gdom_info, NULL, GFP_ATOMIC);
164+ xa_unlock(&aviommu->gdomid_array);
165+ ret = -ENOSPC;
166+ goto out_err_gdom_info;
167+ }
168+
169+ ndom->gdom_info = gdom_info;
170+ refcount_set(&gdom_info->users, 1);
171+
172+ xa_unlock(&aviommu->gdomid_array);
173+
174+ pr_debug("%s: Allocate gdom_id=%#x, hdom_id=%#x\n",
175+ __func__, ndom->gdom_id, gdom_info->hdom_id);
176+
177+ return &ndom->domain;
178+
179+out_err_gdom_info:
180+ kfree(gdom_info);
181+out_err:
182+ kfree(ndom);
183+ return ERR_PTR(ret);
184+}
185+
186+static void set_dte_nested(struct amd_iommu *iommu, struct iommu_domain *dom,
187+ struct iommu_dev_data *dev_data, struct dev_table_entry *new)
188+{
189+ struct protection_domain *parent;
190+ struct nested_domain *ndom = to_ndomain(dom);
191+ struct iommu_hwpt_amd_guest *gdte = &ndom->gdte;
192+ struct pt_iommu_amdv1_hw_info pt_info;
193+
194+ /*
195+ * The nest parent domain is attached during the call to the
196+ * struct iommu_ops.viommu_init(), which will be stored as part
197+ * of the struct amd_iommu_viommu.parent.
198+ */
199+ if (WARN_ON(!ndom->viommu || !ndom->viommu->parent))
200+ return;
201+
202+ parent = ndom->viommu->parent;
203+ amd_iommu_make_clear_dte(dev_data, new);
204+
205+ /* Retrieve the current pagetable info via the IOMMU PT API. */
206+ pt_iommu_amdv1_hw_info(&parent->amdv1, &pt_info);
207+
208+ /*
209+ * Use domain ID from nested domain to program DTE.
210+ * See amd_iommu_alloc_domain_nested().
211+ */
212+ amd_iommu_set_dte_v1(dev_data, parent, ndom->gdom_info->hdom_id,
213+ &pt_info, new);
214+
215+ /* GV is required for nested page table */
216+ new->data[0] |= DTE_FLAG_GV;
217+
218+ /* Guest PPR */
219+ new->data[0] |= gdte->dte[0] & DTE_FLAG_PPR;
220+
221+ /* Guest translation stuff */
222+ new->data[0] |= gdte->dte[0] & (DTE_GLX | DTE_FLAG_GIOV);
223+
224+ /* GCR3 table */
225+ new->data[0] |= gdte->dte[0] & DTE_GCR3_14_12;
226+ new->data[1] |= gdte->dte[1] & (DTE_GCR3_30_15 | DTE_GCR3_51_31);
227+
228+ /* Guest paging mode */
229+ new->data[2] |= gdte->dte[2] & DTE_GPT_LEVEL_MASK;
230+}
231+
232+static int nested_attach_device(struct iommu_domain *dom, struct device *dev,
233+ struct iommu_domain *old)
234+{
235+ struct dev_table_entry new = {0};
236+ struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
237+ struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
238+ int ret = 0;
239+
240+ /*
241+ * Needs to make sure PASID is not enabled
242+ * for this attach path.
243+ */
244+ if (WARN_ON(dev_data->pasid_enabled))
245+ return -EINVAL;
246+
247+ mutex_lock(&dev_data->mutex);
248+
249+ set_dte_nested(iommu, dom, dev_data, &new);
250+
251+ amd_iommu_update_dte(iommu, dev_data, &new);
252+
253+ mutex_unlock(&dev_data->mutex);
254+
255+ return ret;
256+}
257+
258+static void nested_domain_free(struct iommu_domain *dom)
259+{
260+ struct guest_domain_mapping_info *curr;
261+ struct nested_domain *ndom = to_ndomain(dom);
262+ struct amd_iommu_viommu *aviommu = ndom->viommu;
263+
264+ xa_lock(&aviommu->gdomid_array);
265+
266+ if (!refcount_dec_and_test(&ndom->gdom_info->users)) {
267+ xa_unlock(&aviommu->gdomid_array);
268+ return;
269+ }
270+
271+ /*
272+ * The refcount for the gdom_id to hdom_id mapping is zero.
273+ * It is now safe to remove the mapping.
274+ */
275+ curr = __xa_cmpxchg(&aviommu->gdomid_array, ndom->gdom_id,
276+ ndom->gdom_info, NULL, GFP_ATOMIC);
277+
278+ xa_unlock(&aviommu->gdomid_array);
279+ if (WARN_ON(!curr || xa_err(curr)))
280+ return;
281+
282+ /* success */
283+ pr_debug("%s: Free gdom_id=%#x, hdom_id=%#x\n",
284+ __func__, ndom->gdom_id, curr->hdom_id);
285+
286+ amd_iommu_pdom_id_free(ndom->gdom_info->hdom_id);
287+ kfree(curr);
288+ kfree(ndom);
289+}
290+
291+static const struct iommu_domain_ops nested_domain_ops = {
292+ .attach_dev = nested_attach_device,
293+ .free = nested_domain_free,
294+};
drivers/iommu/arm/Kconfig+0−1
@@ -121,7 +121,6 @@ config ARM_SMMU_V3_KUNIT_TEST · 120 unmodified lines
121121
122122 config TEGRA241_CMDQV
123123 bool "NVIDIA Tegra241 CMDQ-V extension support for ARM SMMUv3"
124− depends on ACPI
125124 help
126125 Support for NVIDIA CMDQ-Virtualization extension for ARM SMMUv3. The
127126 CMDQ-V extension is similar to v3.3 ECMDQ for multi command queues
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c+3−1
@@ -177,7 +177,9 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, · 176 unmodified lines
177177 * config bit here base this off the EATS value in the STE. If the EATS
178178 * is set then the VM must generate ATC flushes.
179179 */
180− state.disable_ats = !nested_domain->enable_ats;
180+ if (FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(nested_domain->ste[0])) ==
181+ STRTAB_STE_0_CFG_S1_TRANS)
182+ state.disable_ats = !nested_domain->enable_ats;
181183 ret = arm_smmu_attach_prepare(&state, domain);
182184 if (ret) {
183185 mutex_unlock(&arm_smmu_asid_lock);
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c+75−3
@@ -33,18 +33,25 @@ static struct mm_struct sva_mm = { · 32 unmodified lines
3333 enum arm_smmu_test_master_feat {
3434 ARM_SMMU_MASTER_TEST_ATS = BIT(0),
3535 ARM_SMMU_MASTER_TEST_STALL = BIT(1),
36+ ARM_SMMU_MASTER_TEST_NESTED = BIT(2),
3637 };
3738
39+static void arm_smmu_test_make_s2_ste(struct arm_smmu_ste *ste,
40+ enum arm_smmu_test_master_feat feat);
41+
3842 static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry,
3943 const __le64 *used_bits,
4044 const __le64 *target,
45+ const __le64 *safe,
4146 unsigned int length)
4247 {
4348 bool differs = false;
4449 unsigned int i;
4550
4651 for (i = 0; i < length; i++) {
47− if ((entry[i] & used_bits[i]) != target[i])
52+ __le64 used = used_bits[i] & ~safe[i];
53+
54+ if ((entry[i] & used) != (target[i] & used))
4855 differs = true;
4956 }
5057 return differs;
@@ -56,12 +63,24 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) · 5 unmodified lines
5663 struct arm_smmu_test_writer *test_writer =
5764 container_of(writer, struct arm_smmu_test_writer, writer);
5865 __le64 *entry_used_bits;
66+ __le64 *safe_target;
67+ __le64 *safe_init;
5968
6069 entry_used_bits = kunit_kzalloc(
6170 test_writer->test, sizeof(*entry_used_bits) * NUM_ENTRY_QWORDS,
6271 GFP_KERNEL);
6372 KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits);
6473
74+ safe_target = kunit_kzalloc(test_writer->test,
75+ sizeof(*safe_target) * NUM_ENTRY_QWORDS,
76+ GFP_KERNEL);
77+ KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_target);
78+
79+ safe_init = kunit_kzalloc(test_writer->test,
80+ sizeof(*safe_init) * NUM_ENTRY_QWORDS,
81+ GFP_KERNEL);
82+ KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_init);
83+
6584 pr_debug("STE value is now set to: ");
6685 print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8,
6786 test_writer->entry,
@@ -79,14 +98,23 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) · 11 unmodified lines
7998 * configuration.
8099 */
81100 writer->ops->get_used(test_writer->entry, entry_used_bits);
101+ if (writer->ops->get_update_safe)
102+ writer->ops->get_update_safe(test_writer->entry,
103+ test_writer->init_entry,
104+ safe_init);
105+ if (writer->ops->get_update_safe)
106+ writer->ops->get_update_safe(test_writer->entry,
107+ test_writer->target_entry,
108+ safe_target);
82109 KUNIT_EXPECT_FALSE(
83110 test_writer->test,
84111 arm_smmu_entry_differs_in_used_bits(
85112 test_writer->entry, entry_used_bits,
86− test_writer->init_entry, NUM_ENTRY_QWORDS) &&
113+ test_writer->init_entry, safe_init,
114+ NUM_ENTRY_QWORDS) &&
87115 arm_smmu_entry_differs_in_used_bits(
88116 test_writer->entry, entry_used_bits,
89− test_writer->target_entry,
117+ test_writer->target_entry, safe_target,
90118 NUM_ENTRY_QWORDS));
91119 }
92120 }
@@ -106,6 +134,7 @@ arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, · 13 unmodified lines
106134 static const struct arm_smmu_entry_writer_ops test_ste_ops = {
107135 .sync = arm_smmu_test_writer_record_syncs,
108136 .get_used = arm_smmu_get_ste_used,
137+ .get_update_safe = arm_smmu_get_ste_update_safe,
109138 };
110139
111140 static const struct arm_smmu_entry_writer_ops test_cd_ops = {
@@ -185,6 +214,18 @@ static void arm_smmu_test_make_cdtable_ste(struct arm_smmu_ste *ste, · 73 unmodified lines
185214 };
186215
187216 arm_smmu_make_cdtable_ste(ste, &master, ats_enabled, s1dss);
217+ if (feat & ARM_SMMU_MASTER_TEST_NESTED) {
218+ struct arm_smmu_ste s2ste;
219+ int i;
220+
221+ arm_smmu_test_make_s2_ste(&s2ste,
222+ feat & ~ARM_SMMU_MASTER_TEST_NESTED);
223+ ste->data[0] |= cpu_to_le64(
224+ FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_NESTED));
225+ ste->data[1] |= cpu_to_le64(STRTAB_STE_1_MEV);
226+ for (i = 2; i < NUM_ENTRY_QWORDS; i++)
227+ ste->data[i] = s2ste.data[i];
228+ }
188229 }
189230
190231 static void arm_smmu_v3_write_ste_test_bypass_to_abort(struct kunit *test)
@@ -542,6 +583,35 @@ static void arm_smmu_v3_write_ste_test_s2_to_s1_stall(struct kunit *test) · 351 unmodified lines
542583 NUM_EXPECTED_SYNCS(3));
543584 }
544585
586+static void
587+arm_smmu_v3_write_ste_test_nested_s1dssbypass_to_s1bypass(struct kunit *test)
588+{
589+ struct arm_smmu_ste s1_ste;
590+ struct arm_smmu_ste s2_ste;
591+
592+ arm_smmu_test_make_cdtable_ste(
593+ &s1_ste, STRTAB_STE_1_S1DSS_BYPASS, fake_cdtab_dma_addr,
594+ ARM_SMMU_MASTER_TEST_ATS | ARM_SMMU_MASTER_TEST_NESTED);
595+ arm_smmu_test_make_s2_ste(&s2_ste, 0);
596+ /* Expect an additional sync to unset ignored bits: EATS and MEV */
597+ arm_smmu_v3_test_ste_expect_hitless_transition(test, &s1_ste, &s2_ste,
598+ NUM_EXPECTED_SYNCS(3));
599+}
600+
601+static void
602+arm_smmu_v3_write_ste_test_nested_s1bypass_to_s1dssbypass(struct kunit *test)
603+{
604+ struct arm_smmu_ste s1_ste;
605+ struct arm_smmu_ste s2_ste;
606+
607+ arm_smmu_test_make_cdtable_ste(
608+ &s1_ste, STRTAB_STE_1_S1DSS_BYPASS, fake_cdtab_dma_addr,
609+ ARM_SMMU_MASTER_TEST_ATS | ARM_SMMU_MASTER_TEST_NESTED);
610+ arm_smmu_test_make_s2_ste(&s2_ste, 0);
611+ arm_smmu_v3_test_ste_expect_hitless_transition(test, &s2_ste, &s1_ste,
612+ NUM_EXPECTED_SYNCS(2));
613+}
614+
545615 static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test)
546616 {
547617 struct arm_smmu_cd cd = {};
@@ -588,6 +658,8 @@ static struct kunit_case arm_smmu_v3_test_cases[] = { · 40 unmodified lines
588658 KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_change_asid),
589659 KUNIT_CASE(arm_smmu_v3_write_ste_test_s1_to_s2_stall),
590660 KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_s1_stall),
661+ KUNIT_CASE(arm_smmu_v3_write_ste_test_nested_s1dssbypass_to_s1bypass),
662+ KUNIT_CASE(arm_smmu_v3_write_ste_test_nested_s1bypass_to_s1dssbypass),
591663 KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear),
592664 KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release),
593665 {},
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c+120−33
@@ -487,20 +487,26 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) · 486 unmodified lines
487487 */
488488 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
489489 {
490− int val;
491−
492490 /*
493− * We can try to avoid the cmpxchg() loop by simply incrementing the
494− * lock counter. When held in exclusive state, the lock counter is set
495− * to INT_MIN so these increments won't hurt as the value will remain
496− * negative.
491+ * When held in exclusive state, the lock counter is set to INT_MIN
492+ * so these increments won't hurt as the value will remain negative.
493+ * The increment will also signal the exclusive locker that there are
494+ * shared waiters.
497495 */
498496 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
499497 return;
500498
501− do {
502− val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
503− } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
499+ /*
500+ * Someone else is holding the lock in exclusive state, so wait
501+ * for them to finish. Since we already incremented the lock counter,
502+ * no exclusive lock can be acquired until we finish. We don't need
503+ * the return value since we only care that the exclusive lock is
504+ * released (i.e. the lock counter is non-negative).
505+ * Once the exclusive locker releases the lock, the sign bit will
506+ * be cleared and our increment will make the lock counter positive,
507+ * allowing us to proceed.
508+ */
509+ atomic_cond_read_relaxed(&cmdq->lock, VAL > 0);
504510 }
505511
506512 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
@@ -527,9 +533,14 @@ static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq) · 20 unmodified lines
527533 __ret; \
528534 })
529535
536+/*
537+ * Only clear the sign bit when releasing the exclusive lock this will
538+ * allow any shared_lock() waiters to proceed without the possibility
539+ * of entering the exclusive lock in a tight loop.
540+ */
530541 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
531542 ({ \
532− atomic_set_release(&cmdq->lock, 0); \
543+ atomic_fetch_andnot_release(INT_MIN, &cmdq->lock); \
533544 local_irq_restore(flags); \
534545 })
535546
@@ -1082,6 +1093,49 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) · 546 unmodified lines
10821093 }
10831094 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used);
10841095
1096+VISIBLE_IF_KUNIT
1097+void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
1098+ __le64 *safe_bits)
1099+{
1100+ const __le64 eats_s1chk =
1101+ FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK);
1102+ const __le64 eats_trans =
1103+ FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS);
1104+
1105+ /*
1106+ * When an STE changes EATS_TRANS, the sequencing code in the attach
1107+ * logic already will have the PCI cap for ATS disabled. Thus at this
1108+ * moment we can expect that the device will not generate ATS queries
1109+ * and so we don't care about the sequencing of EATS. The purpose of
1110+ * EATS_TRANS is to protect the system from hostile untrusted devices
1111+ * that issue ATS when the PCI config space is disabled. However, if
1112+ * EATS_TRANS is being changed, then we must have already trusted the
1113+ * device as the EATS_TRANS security block is being disabled.
1114+ *
1115+ * Note: now the EATS_TRANS update is moved to the first entry_set().
1116+ * Changing S2S and EATS might transiently result in S2S=1 and EATS=1
1117+ * which is a bad STE (see "5.2 Stream Table Entry"). In such a case,
1118+ * we can't do a hitless update. Also, it should not be added to the
1119+ * safe bits with STRTAB_STE_1_EATS_S1CHK, because EATS=0b11 would be
1120+ * effectively an errant 0b00 configuration.
1121+ */
1122+ if (!((cur[1] | target[1]) & cpu_to_le64(eats_s1chk)) &&
1123+ !((cur[2] | target[2]) & cpu_to_le64(STRTAB_STE_2_S2S)))
1124+ safe_bits[1] |= cpu_to_le64(eats_trans);
1125+
1126+ /*
1127+ * MEV does not meaningfully impact the operation of the HW, it only
1128+ * changes how many fault events are generated, thus we can relax it
1129+ * when computing the ordering. The spec notes the device can act like
1130+ * MEV=1 anyhow:
1131+ *
1132+ * Note: Software must expect, and be able to deal with, coalesced
1133+ * fault records even when MEV == 0.
1134+ */
1135+ safe_bits[1] |= cpu_to_le64(STRTAB_STE_1_MEV);
1136+}
1137+EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_update_safe);
1138+
10851139 /*
10861140 * Figure out if we can do a hitless update of entry to become target. Returns a
10871141 * bit mask where 1 indicates that qword needs to be set disruptively.
@@ -1094,13 +1148,22 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, · 6 unmodified lines
10941148 {
10951149 __le64 target_used[NUM_ENTRY_QWORDS] = {};
10961150 __le64 cur_used[NUM_ENTRY_QWORDS] = {};
1151+ __le64 safe[NUM_ENTRY_QWORDS] = {};
10971152 u8 used_qword_diff = 0;
10981153 unsigned int i;
10991154
11001155 writer->ops->get_used(entry, cur_used);
11011156 writer->ops->get_used(target, target_used);
1157+ if (writer->ops->get_update_safe)
1158+ writer->ops->get_update_safe(entry, target, safe);
11021159
11031160 for (i = 0; i != NUM_ENTRY_QWORDS; i++) {
1161+ /*
1162+ * Safe is only used for bits that are used by both entries,
1163+ * otherwise it is sequenced according to the unused entry.
1164+ */
1165+ safe[i] &= target_used[i] & cur_used[i];
1166+
11041167 /*
11051168 * Check that masks are up to date, the make functions are not
11061169 * allowed to set a bit to 1 if the used function doesn't say it
@@ -1109,6 +1172,7 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, · 2 unmodified lines
11091172 WARN_ON_ONCE(target[i] & ~target_used[i]);
11101173
11111174 /* Bits can change because they are not currently being used */
1175+ cur_used[i] &= ~safe[i];
11121176 unused_update[i] = (entry[i] & cur_used[i]) |
11131177 (target[i] & ~cur_used[i]);
11141178 /*
@@ -1121,7 +1185,7 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, · 6 unmodified lines
11211185 return used_qword_diff;
11221186 }
11231187
1124−static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
1188+static void entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
11251189 const __le64 *target, unsigned int start,
11261190 unsigned int len)
11271191 {
@@ -1137,7 +1201,6 @@ static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, · 9 unmodified lines
11371201
11381202 if (changed)
11391203 writer->ops->sync(writer);
1140− return changed;
11411204 }
11421205
11431206 /*
@@ -1207,12 +1270,9 @@ void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, · 63 unmodified lines
12071270 entry_set(writer, entry, target, 0, 1);
12081271 } else {
12091272 /*
1210− * No inuse bit changed. Sanity check that all unused bits are 0
1211− * in the entry. The target was already sanity checked by
1212− * compute_qword_diff().
1273+ * No inuse bit changed, though safe bits may have changed.
12131274 */
1214− WARN_ON_ONCE(
1215− entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS));
1275+ entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
12161276 }
12171277 }
12181278 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry);
@@ -1543,6 +1603,7 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) · 324 unmodified lines
15431603 static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
15441604 .sync = arm_smmu_ste_writer_sync_entry,
15451605 .get_used = arm_smmu_get_ste_used,
1606+ .get_update_safe = arm_smmu_get_ste_update_safe,
15461607 };
15471608
15481609 static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
@@ -2551,7 +2612,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, · 1002 unmodified lines
25512612 ARM_SMMU_FEAT_VAX) ? 52 : 48;
25522613
25532614 pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS);
2554− pgtbl_cfg.oas = smmu->ias;
2615+ pgtbl_cfg.oas = smmu->oas;
25552616 if (enable_dirty)
25562617 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD;
25572618 fmt = ARM_64_LPAE_S1;
@@ -2561,7 +2622,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, · 3 unmodified lines
25612622 case ARM_SMMU_DOMAIN_S2:
25622623 if (enable_dirty)
25632624 return -EOPNOTSUPP;
2564− pgtbl_cfg.ias = smmu->ias;
2625+ pgtbl_cfg.ias = smmu->oas;
25652626 pgtbl_cfg.oas = smmu->oas;
25662627 fmt = ARM_64_LPAE_S2;
25672628 finalise_stage_fn = arm_smmu_domain_finalise_s2;
@@ -3125,7 +3186,8 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, · 557 unmodified lines
31253186 struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
31263187 struct arm_smmu_cd *cd, struct iommu_domain *old)
31273188 {
3128− struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
3189+ struct iommu_domain *sid_domain =
3190+ iommu_driver_get_domain_for_dev(master->dev);
31293191 struct arm_smmu_attach_state state = {
31303192 .master = master,
31313193 .ssid = pasid,
@@ -3191,7 +3253,7 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, · 59 unmodified lines
31913253 */
31923254 if (!arm_smmu_ssids_in_use(&master->cd_table)) {
31933255 struct iommu_domain *sid_domain =
3194− iommu_get_domain_for_dev(master->dev);
3256+ iommu_driver_get_domain_for_dev(master->dev);
31953257
31963258 if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
31973259 sid_domain->type == IOMMU_DOMAIN_BLOCKED)
@@ -4395,13 +4457,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) · 1197 unmodified lines
43954457 }
43964458
43974459 /* We only support the AArch64 table format at present */
4398− switch (FIELD_GET(IDR0_TTF, reg)) {
4399− case IDR0_TTF_AARCH32_64:
4400− smmu->ias = 40;
4401− fallthrough;
4402− case IDR0_TTF_AARCH64:
4403− break;
4404− default:
4460+ if (!(FIELD_GET(IDR0_TTF, reg) & IDR0_TTF_AARCH64)) {
44054461 dev_err(smmu->dev, "AArch64 table format not supported!\n");
44064462 return -ENXIO;
44074463 }
@@ -4514,8 +4570,6 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) · 106 unmodified lines
45144570 dev_warn(smmu->dev,
45154571 "failed to set DMA mask for table walker\n");
45164572
4517− smmu->ias = max(smmu->ias, smmu->oas);
4518−
45194573 if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
45204574 (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
45214575 smmu->features |= ARM_SMMU_FEAT_NESTING;
@@ -4525,11 +4579,40 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) · 3 unmodified lines
45254579 if (arm_smmu_sva_supported(smmu))
45264580 smmu->features |= ARM_SMMU_FEAT_SVA;
45274581
4528− dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
4529− smmu->ias, smmu->oas, smmu->features);
4582+ dev_info(smmu->dev, "oas %lu-bit (features 0x%08x)\n",
4583+ smmu->oas, smmu->features);
45304584 return 0;
45314585 }
45324586
4587+#ifdef CONFIG_TEGRA241_CMDQV
4588+static void tegra_cmdqv_dt_probe(struct device_node *smmu_node,
4589+ struct arm_smmu_device *smmu)
4590+{
4591+ struct platform_device *pdev;
4592+ struct device_node *np;
4593+
4594+ np = of_parse_phandle(smmu_node, "nvidia,cmdqv", 0);
4595+ if (!np)
4596+ return;
4597+
4598+ /* Tegra241 CMDQV driver is responsible for put_device() */
4599+ pdev = of_find_device_by_node(np);
4600+ of_node_put(np);
4601+ if (!pdev)
4602+ return;
4603+
4604+ smmu->impl_dev = &pdev->dev;
4605+ smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
4606+ dev_dbg(smmu->dev, "found companion CMDQV device: %s\n",
4607+ dev_name(smmu->impl_dev));
4608+}
4609+#else
4610+static void tegra_cmdqv_dt_probe(struct device_node *smmu_node,
4611+ struct arm_smmu_device *smmu)
4612+{
4613+}
4614+#endif
4615+
45334616 #ifdef CONFIG_ACPI
45344617 #ifdef CONFIG_TEGRA241_CMDQV
45354618 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
@@ -4542,10 +4625,11 @@ static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node, · 6 unmodified lines
45424625 adev = acpi_dev_get_first_match_dev("NVDA200C", uid, -1);
45434626 if (adev) {
45444627 /* Tegra241 CMDQV driver is responsible for put_device() */
4545− smmu->impl_dev = &adev->dev;
4628+ smmu->impl_dev = get_device(acpi_get_first_physical_node(adev));
45464629 smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
45474630 dev_info(smmu->dev, "found companion CMDQV device: %s\n",
45484631 dev_name(smmu->impl_dev));
4632+ acpi_dev_put(adev);
45494633 }
45504634 kfree(uid);
45514635 }
@@ -4634,6 +4718,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev, · 82 unmodified lines
46344718 if (of_dma_is_coherent(dev->of_node))
46354719 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
46364720
4721+ if (of_device_is_compatible(dev->of_node, "nvidia,tegra264-smmu"))
4722+ tegra_cmdqv_dt_probe(dev->of_node, smmu);
4723+
46374724 return ret;
46384725 }
46394726
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h+4−2
@@ -43,7 +43,6 @@ struct arm_vsmmu; · 42 unmodified lines
4343 #define IDR0_COHACC (1 << 4)
4444 #define IDR0_TTF GENMASK(3, 2)
4545 #define IDR0_TTF_AARCH64 2
46−#define IDR0_TTF_AARCH32_64 3
4746 #define IDR0_S1P (1 << 1)
4847 #define IDR0_S2P (1 << 0)
4948
@@ -784,7 +783,6 @@ struct arm_smmu_device { · 734 unmodified lines
784783 int gerr_irq;
785784 int combined_irq;
786785
787− unsigned long ias; /* IPA */
788786 unsigned long oas; /* PA */
789787 unsigned long pgsize_bitmap;
790788
@@ -900,6 +898,8 @@ struct arm_smmu_entry_writer { · 109 unmodified lines
900898
901899 struct arm_smmu_entry_writer_ops {
902900 void (*get_used)(const __le64 *entry, __le64 *used);
901+ void (*get_update_safe)(const __le64 *cur, const __le64 *target,
902+ __le64 *safe_bits);
903903 void (*sync)(struct arm_smmu_entry_writer *writer);
904904 };
905905
@@ -911,6 +911,8 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, · 5 unmodified lines
911911
912912 #if IS_ENABLED(CONFIG_KUNIT)
913913 void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits);
914+void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
915+ __le64 *safe_bits);
914916 void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur,
915917 const __le64 *target);
916918 void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits);
drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c+12−72
@@ -3,17 +3,15 @@ · 2 unmodified lines
33
44 #define dev_fmt(fmt) "tegra241_cmdqv: " fmt
55
6−#include <linux/acpi.h>
76 #include <linux/debugfs.h>
87 #include <linux/dma-mapping.h>
98 #include <linux/interrupt.h>
109 #include <linux/iommu.h>
1110 #include <linux/iommufd.h>
1211 #include <linux/iopoll.h>
12+#include <linux/platform_device.h>
1313 #include <uapi/linux/iommufd.h>
1414
15−#include <acpi/acpixf.h>
16−
1715 #include "arm-smmu-v3.h"
1816
1917 /* CMDQV register page base and size defines */
@@ -854,69 +852,6 @@ static struct arm_smmu_impl_ops tegra241_cmdqv_impl_ops = { · 834 unmodified lines
854852
855853 /* Probe Functions */
856854
857−static int tegra241_cmdqv_acpi_is_memory(struct acpi_resource *res, void *data)
858−{
859− struct resource_win win;
860−
861− return !acpi_dev_resource_address_space(res, &win);
862−}
863−
864−static int tegra241_cmdqv_acpi_get_irqs(struct acpi_resource *ares, void *data)
865−{
866− struct resource r;
867− int *irq = data;
868−
869− if (*irq <= 0 && acpi_dev_resource_interrupt(ares, 0, &r))
870− *irq = r.start;
871− return 1; /* No need to add resource to the list */
872−}
873−
874−static struct resource *
875−tegra241_cmdqv_find_acpi_resource(struct device *dev, int *irq)
876−{
877− struct acpi_device *adev = to_acpi_device(dev);
878− struct list_head resource_list;
879− struct resource_entry *rentry;
880− struct resource *res = NULL;
881− int ret;
882−
883− INIT_LIST_HEAD(&resource_list);
884− ret = acpi_dev_get_resources(adev, &resource_list,
885− tegra241_cmdqv_acpi_is_memory, NULL);
886− if (ret < 0) {
887− dev_err(dev, "failed to get memory resource: %d\n", ret);
888− return NULL;
889− }
890−
891− rentry = list_first_entry_or_null(&resource_list,
892− struct resource_entry, node);
893− if (!rentry) {
894− dev_err(dev, "failed to get memory resource entry\n");
895− goto free_list;
896− }
897−
898− /* Caller must free the res */
899− res = kzalloc(sizeof(*res), GFP_KERNEL);
900− if (!res)
901− goto free_list;
902−
903− *res = *rentry->res;
904−
905− acpi_dev_free_resource_list(&resource_list);
906−
907− INIT_LIST_HEAD(&resource_list);
908−
909− if (irq)
910− ret = acpi_dev_get_resources(adev, &resource_list,
911− tegra241_cmdqv_acpi_get_irqs, irq);
912− if (ret < 0 || !irq || *irq <= 0)
913− dev_warn(dev, "no interrupt. errors will not be reported\n");
914−
915−free_list:
916− acpi_dev_free_resource_list(&resource_list);
917− return res;
918−}
919−
920855 static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)
921856 {
922857 struct tegra241_cmdqv *cmdqv =
@@ -1042,18 +977,23 @@ iounmap: · 119 unmodified lines
1042977
1043978 struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
1044979 {
980+ struct platform_device *pdev = to_platform_device(smmu->impl_dev);
1045981 struct arm_smmu_device *new_smmu;
1046− struct resource *res = NULL;
982+ struct resource *res;
1047983 int irq;
1048984
1049− if (!smmu->dev->of_node)
1050− res = tegra241_cmdqv_find_acpi_resource(smmu->impl_dev, &irq);
1051− if (!res)
985+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
986+ if (!res) {
987+ dev_err(&pdev->dev, "no memory resource found for CMDQV\n");
1052988 goto out_fallback;
989+ }
1053990
1054− new_smmu = __tegra241_cmdqv_probe(smmu, res, irq);
1055− kfree(res);
991+ irq = platform_get_irq_optional(pdev, 0);
992+ if (irq <= 0)
993+ dev_warn(&pdev->dev,
994+ "no interrupt. errors will not be reported\n");
1056995
996+ new_smmu = __tegra241_cmdqv_probe(smmu, res, irq);
1057997 if (new_smmu)
1058998 return new_smmu;
1059999
drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c+28−0
@@ -41,12 +41,40 @@ static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { · 40 unmodified lines
4141 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) },
4242 { .compatible = "qcom,fastrpc",
4343 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) },
44+ { .compatible = "qcom,qcm2290-mdss",
45+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
46+ { .compatible = "qcom,sa8775p-mdss",
47+ .data = (const void *) (PREFETCH_DEFAULT | CMTLB) },
4448 { .compatible = "qcom,sc7280-mdss",
4549 .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
4650 { .compatible = "qcom,sc7280-venus",
4751 .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
52+ { .compatible = "qcom,sc8180x-mdss",
53+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
54+ { .compatible = "qcom,sc8280xp-mdss",
55+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
56+ { .compatible = "qcom,sm6115-mdss",
57+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
58+ { .compatible = "qcom,sm6125-mdss",
59+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
60+ { .compatible = "qcom,sm6350-mdss",
61+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
62+ { .compatible = "qcom,sm8150-mdss",
63+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
64+ { .compatible = "qcom,sm8250-mdss",
65+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
66+ { .compatible = "qcom,sm8350-mdss",
67+ .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
68+ { .compatible = "qcom,sm8450-mdss",
69+ .data = (const void *) (PREFETCH_DEFAULT | CMTLB) },
4870 { .compatible = "qcom,sm8550-mdss",
4971 .data = (const void *) (PREFETCH_DEFAULT | CMTLB) },
72+ { .compatible = "qcom,sm8650-mdss",
73+ .data = (const void *) (PREFETCH_DEFAULT | CMTLB) },
74+ { .compatible = "qcom,sm8750-mdss",
75+ .data = (const void *) (PREFETCH_DEFAULT | CMTLB) },
76+ { .compatible = "qcom,x1e80100-mdss",
77+ .data = (const void *) (PREFETCH_DEFAULT | CMTLB) },
5078 { }
5179 };
5280
drivers/iommu/arm/arm-smmu/qcom_iommu.c+2−6
@@ -761,14 +761,10 @@ static struct platform_driver qcom_iommu_ctx_driver = { · 760 unmodified lines
761761
762762 static bool qcom_iommu_has_secure_context(struct qcom_iommu_dev *qcom_iommu)
763763 {
764− struct device_node *child;
765−
766− for_each_child_of_node(qcom_iommu->dev->of_node, child) {
764+ for_each_child_of_node_scoped(qcom_iommu->dev->of_node, child) {
767765 if (of_device_is_compatible(child, "qcom,msm-iommu-v1-sec") ||
768− of_device_is_compatible(child, "qcom,msm-iommu-v2-sec")) {
769− of_node_put(child);
766+ of_device_is_compatible(child, "qcom,msm-iommu-v2-sec"))
770767 return true;
771− }
772768 }
773769
774770 return false;
drivers/iommu/dma-iommu.c+1−3
@@ -2097,10 +2097,8 @@ void dma_iova_destroy(struct device *dev, struct dma_iova_state *state, · 2096 unmodified lines
20972097 }
20982098 EXPORT_SYMBOL_GPL(dma_iova_destroy);
20992099
2100−void iommu_setup_dma_ops(struct device *dev)
2100+void iommu_setup_dma_ops(struct device *dev, struct iommu_domain *domain)
21012101 {
2102− struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
2103−
21042102 if (dev_is_pci(dev))
21052103 dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac;
21062104
drivers/iommu/dma-iommu.h+3−2
@@ -9,7 +9,7 @@ · 8 unmodified lines
99
1010 #ifdef CONFIG_IOMMU_DMA
1111
12−void iommu_setup_dma_ops(struct device *dev);
12+void iommu_setup_dma_ops(struct device *dev, struct iommu_domain *domain);
1313
1414 int iommu_get_dma_cookie(struct iommu_domain *domain);
1515 void iommu_put_dma_cookie(struct iommu_domain *domain);
@@ -26,7 +26,8 @@ extern bool iommu_dma_forcedac; · 10 unmodified lines
2626
2727 #else /* CONFIG_IOMMU_DMA */
2828
29−static inline void iommu_setup_dma_ops(struct device *dev)
29+static inline void iommu_setup_dma_ops(struct device *dev,
30+ struct iommu_domain *domain)
3031 {
3132 }
3233
drivers/iommu/generic_pt/fmt/amdv1.h+2−1
@@ -354,7 +354,8 @@ static inline int amdv1pt_iommu_set_prot(struct pt_common *common, · 353 unmodified lines
354354 * Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to
355355 * control this. For now if the tables use sme_set then so do the ptes.
356356 */
357− if (pt_feature(common, PT_FEAT_AMDV1_ENCRYPT_TABLES))
357+ if (pt_feature(common, PT_FEAT_AMDV1_ENCRYPT_TABLES) &&
358+ !(iommu_prot & IOMMU_MMIO))
358359 pte = __sme_set(pte);
359360
360361 attrs->descriptor_bits = pte;
drivers/iommu/generic_pt/fmt/x86_64.h+2−1
@@ -227,7 +227,8 @@ static inline int x86_64_pt_iommu_set_prot(struct pt_common *common, · 226 unmodified lines
227227 * Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to
228228 * control this. For now if the tables use sme_set then so do the ptes.
229229 */
230− if (pt_feature(common, PT_FEAT_X86_64_AMD_ENCRYPT_TABLES))
230+ if (pt_feature(common, PT_FEAT_X86_64_AMD_ENCRYPT_TABLES) &&
231+ !(iommu_prot & IOMMU_MMIO))
231232 pte = __sme_set(pte);
232233
233234 attrs->descriptor_bits = pte;
drivers/iommu/generic_pt/iommu_pt.h+1−2
@@ -58,10 +58,9 @@ static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather, · 57 unmodified lines
5858 * Note that the sync frees the gather's free list, so we must
5959 * not have any pages on that list that are covered by iova/len
6060 */
61− } else if (pt_feature(common, PT_FEAT_FLUSH_RANGE)) {
62− iommu_iotlb_gather_add_range(iotlb_gather, iova, len);
6361 }
6462
63+ iommu_iotlb_gather_add_range(iotlb_gather, iova, len);
6564 iommu_pages_list_splice(free_list, &iotlb_gather->freelist);
6665 }
6766
drivers/iommu/intel/cache.c+8−1
@@ -363,6 +363,13 @@ static void qi_batch_add_pasid_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 · 362 unmodified lines
363363 qi_batch_increment_index(iommu, batch);
364364 }
365365
366+static bool intel_domain_use_piotlb(struct dmar_domain *domain)
367+{
368+ return domain->domain.type == IOMMU_DOMAIN_SVA ||
369+ domain->domain.type == IOMMU_DOMAIN_NESTED ||
370+ intel_domain_is_fs_paging(domain);
371+}
372+
366373 static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *tag,
367374 unsigned long addr, unsigned long pages,
368375 unsigned long mask, int ih)
@@ -370,7 +377,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag * · 1 unmodified line
370377 struct intel_iommu *iommu = tag->iommu;
371378 u64 type = DMA_TLB_PSI_FLUSH;
372379
373− if (intel_domain_is_fs_paging(domain)) {
380+ if (intel_domain_use_piotlb(domain)) {
374381 qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr,
375382 pages, ih, domain->qi_batch);
376383 return;
drivers/iommu/intel/iommu.c+15−18
@@ -1240,22 +1240,22 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 · 1239 unmodified lines
12401240 }
12411241
12421242 did = context_domain_id(context);
1243− context_clear_entry(context);
1243+ context_clear_present(context);
12441244 __iommu_flush_cache(iommu, context, sizeof(*context));
12451245 spin_unlock(&iommu->lock);
12461246 intel_context_flush_no_pasid(info, context, did);
1247+ context_clear_entry(context);
1248+ __iommu_flush_cache(iommu, context, sizeof(*context));
12471249 }
12481250
12491251 int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev,
12501252 ioasid_t pasid, u16 did, phys_addr_t fsptptr,
12511253 int flags, struct iommu_domain *old)
12521254 {
1253− if (!old)
1254− return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid,
1255− did, flags);
1256− return intel_pasid_replace_first_level(iommu, dev, fsptptr, pasid, did,
1257− iommu_domain_did(old, iommu),
1258− flags);
1255+ if (old)
1256+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
1257+
1258+ return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, did, flags);
12591259 }
12601260
12611261 static int domain_setup_second_level(struct intel_iommu *iommu,
@@ -1263,23 +1263,20 @@ static int domain_setup_second_level(struct intel_iommu *iommu, · 1 unmodified line
12631263 struct device *dev, ioasid_t pasid,
12641264 struct iommu_domain *old)
12651265 {
1266− if (!old)
1267− return intel_pasid_setup_second_level(iommu, domain,
1268− dev, pasid);
1269− return intel_pasid_replace_second_level(iommu, domain, dev,
1270− iommu_domain_did(old, iommu),
1271− pasid);
1266+ if (old)
1267+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
1268+
1269+ return intel_pasid_setup_second_level(iommu, domain, dev, pasid);
12721270 }
12731271
12741272 static int domain_setup_passthrough(struct intel_iommu *iommu,
12751273 struct device *dev, ioasid_t pasid,
12761274 struct iommu_domain *old)
12771275 {
1278− if (!old)
1279− return intel_pasid_setup_pass_through(iommu, dev, pasid);
1280− return intel_pasid_replace_pass_through(iommu, dev,
1281− iommu_domain_did(old, iommu),
1282− pasid);
1276+ if (old)
1277+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
1278+
1279+ return intel_pasid_setup_pass_through(iommu, dev, pasid);
12831280 }
12841281
12851282 static int domain_setup_first_level(struct intel_iommu *iommu,
drivers/iommu/intel/iommu.h+20−1
@@ -900,7 +900,26 @@ static inline int pfn_level_offset(u64 pfn, int level) · 899 unmodified lines
900900
901901 static inline void context_set_present(struct context_entry *context)
902902 {
903− context->lo |= 1;
903+ u64 val;
904+
905+ dma_wmb();
906+ val = READ_ONCE(context->lo) | 1;
907+ WRITE_ONCE(context->lo, val);
908+}
909+
910+/*
911+ * Clear the Present (P) bit (bit 0) of a context table entry. This initiates
912+ * the transition of the entry's ownership from hardware to software. The
913+ * caller is responsible for fulfilling the invalidation handshake recommended
914+ * by the VT-d spec, Section 6.5.3.3 (Guidance to Software for Invalidations).
915+ */
916+static inline void context_clear_present(struct context_entry *context)
917+{
918+ u64 val;
919+
920+ val = READ_ONCE(context->lo) & GENMASK_ULL(63, 1);
921+ WRITE_ONCE(context->lo, val);
922+ dma_wmb();
904923 }
905924
906925 static inline void context_set_fault_enable(struct context_entry *context)
drivers/iommu/intel/nested.c+4−5
@@ -136,11 +136,10 @@ static int domain_setup_nested(struct intel_iommu *iommu, · 135 unmodified lines
136136 struct device *dev, ioasid_t pasid,
137137 struct iommu_domain *old)
138138 {
139− if (!old)
140− return intel_pasid_setup_nested(iommu, dev, pasid, domain);
141− return intel_pasid_replace_nested(iommu, dev, pasid,
142− iommu_domain_did(old, iommu),
143− domain);
139+ if (old)
140+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
141+
142+ return intel_pasid_setup_nested(iommu, dev, pasid, domain);
144143 }
145144
146145 static int intel_nested_set_dev_pasid(struct iommu_domain *domain,
drivers/iommu/intel/pasid.c+22−190
@@ -153,6 +153,9 @@ retry: · 152 unmodified lines
153153 if (!entries)
154154 return NULL;
155155
156+ if (!ecap_coherent(info->iommu->ecap))
157+ clflush_cache_range(entries, VTD_PAGE_SIZE);
158+
156159 /*
157160 * The pasid directory table entry won't be freed after
158161 * allocation. No worry about the race with free and
@@ -165,10 +168,8 @@ retry: · 6 unmodified lines
165168 iommu_free_pages(entries);
166169 goto retry;
167170 }
168− if (!ecap_coherent(info->iommu->ecap)) {
169− clflush_cache_range(entries, VTD_PAGE_SIZE);
171+ if (!ecap_coherent(info->iommu->ecap))
170172 clflush_cache_range(&dir[dir_index].val, sizeof(*dir));
171− }
172173 }
173174
174175 return &entries[index];
@@ -218,7 +219,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, · 43 unmodified lines
218219 if (!info || !info->ats_enabled)
219220 return;
220221
221− if (pci_dev_is_disconnected(to_pci_dev(dev)))
222+ if (!pci_device_is_present(to_pci_dev(dev)))
222223 return;
223224
224225 sid = PCI_DEVID(info->bus, info->devfn);
@@ -272,7 +273,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, · 47 unmodified lines
272273
273274 did = pasid_get_domain_id(pte);
274275 pgtt = pasid_pte_get_pgtt(pte);
275− intel_pasid_clear_entry(dev, pasid, fault_ignore);
276+ pasid_clear_present(pte);
276277 spin_unlock(&iommu->lock);
277278
278279 if (!ecap_coherent(iommu->ecap))
@@ -286,6 +287,10 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, · 7 unmodified lines
286287 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
287288
288289 devtlb_invalidation_with_pasid(iommu, dev, pasid);
290+ intel_pasid_clear_entry(dev, pasid, fault_ignore);
291+ if (!ecap_coherent(iommu->ecap))
292+ clflush_cache_range(pte, sizeof(*pte));
293+
289294 if (!fault_ignore)
290295 intel_iommu_drain_pasid_prq(dev, pasid);
291296 }
@@ -412,50 +417,6 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev, · 120 unmodified lines
412417 return 0;
413418 }
414419
415−int intel_pasid_replace_first_level(struct intel_iommu *iommu,
416− struct device *dev, phys_addr_t fsptptr,
417− u32 pasid, u16 did, u16 old_did,
418− int flags)
419−{
420− struct pasid_entry *pte, new_pte;
421−
422− if (!ecap_flts(iommu->ecap)) {
423− pr_err("No first level translation support on %s\n",
424− iommu->name);
425− return -EINVAL;
426− }
427−
428− if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) {
429− pr_err("No 5-level paging support for first-level on %s\n",
430− iommu->name);
431− return -EINVAL;
432− }
433−
434− pasid_pte_config_first_level(iommu, &new_pte, fsptptr, did, flags);
435−
436− spin_lock(&iommu->lock);
437− pte = intel_pasid_get_entry(dev, pasid);
438− if (!pte) {
439− spin_unlock(&iommu->lock);
440− return -ENODEV;
441− }
442−
443− if (!pasid_pte_is_present(pte)) {
444− spin_unlock(&iommu->lock);
445− return -EINVAL;
446− }
447−
448− WARN_ON(old_did != pasid_get_domain_id(pte));
449−
450− *pte = new_pte;
451− spin_unlock(&iommu->lock);
452−
453− intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
454− intel_iommu_drain_pasid_prq(dev, pasid);
455−
456− return 0;
457−}
458−
459420 /*
460421 * Set up the scalable mode pasid entry for second only translation type.
461422 */
@@ -522,51 +483,6 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, · 60 unmodified lines
522483 return 0;
523484 }
524485
525−int intel_pasid_replace_second_level(struct intel_iommu *iommu,
526− struct dmar_domain *domain,
527− struct device *dev, u16 old_did,
528− u32 pasid)
529−{
530− struct pasid_entry *pte, new_pte;
531− u16 did;
532−
533− /*
534− * If hardware advertises no support for second level
535− * translation, return directly.
536− */
537− if (!ecap_slts(iommu->ecap)) {
538− pr_err("No second level translation support on %s\n",
539− iommu->name);
540− return -EINVAL;
541− }
542−
543− did = domain_id_iommu(domain, iommu);
544−
545− pasid_pte_config_second_level(iommu, &new_pte, domain, did);
546−
547− spin_lock(&iommu->lock);
548− pte = intel_pasid_get_entry(dev, pasid);
549− if (!pte) {
550− spin_unlock(&iommu->lock);
551− return -ENODEV;
552− }
553−
554− if (!pasid_pte_is_present(pte)) {
555− spin_unlock(&iommu->lock);
556− return -EINVAL;
557− }
558−
559− WARN_ON(old_did != pasid_get_domain_id(pte));
560−
561− *pte = new_pte;
562− spin_unlock(&iommu->lock);
563−
564− intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
565− intel_iommu_drain_pasid_prq(dev, pasid);
566−
567− return 0;
568−}
569−
570486 /*
571487 * Set up dirty tracking on a second only or nested translation type.
572488 */
@@ -679,38 +595,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, · 106 unmodified lines
679595 return 0;
680596 }
681597
682−int intel_pasid_replace_pass_through(struct intel_iommu *iommu,
683− struct device *dev, u16 old_did,
684− u32 pasid)
685−{
686− struct pasid_entry *pte, new_pte;
687− u16 did = FLPT_DEFAULT_DID;
688−
689− pasid_pte_config_pass_through(iommu, &new_pte, did);
690−
691− spin_lock(&iommu->lock);
692− pte = intel_pasid_get_entry(dev, pasid);
693− if (!pte) {
694− spin_unlock(&iommu->lock);
695− return -ENODEV;
696− }
697−
698− if (!pasid_pte_is_present(pte)) {
699− spin_unlock(&iommu->lock);
700− return -EINVAL;
701− }
702−
703− WARN_ON(old_did != pasid_get_domain_id(pte));
704−
705− *pte = new_pte;
706− spin_unlock(&iommu->lock);
707−
708− intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
709− intel_iommu_drain_pasid_prq(dev, pasid);
710−
711− return 0;
712−}
713−
714598 /*
715599 * Set the page snoop control for a pasid entry which has been set up.
716600 */
@@ -844,69 +728,6 @@ int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, · 127 unmodified lines
844728 return 0;
845729 }
846730
847−int intel_pasid_replace_nested(struct intel_iommu *iommu,
848− struct device *dev, u32 pasid,
849− u16 old_did, struct dmar_domain *domain)
850−{
851− struct iommu_hwpt_vtd_s1 *s1_cfg = &domain->s1_cfg;
852− struct dmar_domain *s2_domain = domain->s2_domain;
853− u16 did = domain_id_iommu(domain, iommu);
854− struct pasid_entry *pte, new_pte;
855−
856− /* Address width should match the address width supported by hardware */
857− switch (s1_cfg->addr_width) {
858− case ADDR_WIDTH_4LEVEL:
859− break;
860− case ADDR_WIDTH_5LEVEL:
861− if (!cap_fl5lp_support(iommu->cap)) {
862− dev_err_ratelimited(dev,
863− "5-level paging not supported\n");
864− return -EINVAL;
865− }
866− break;
867− default:
868− dev_err_ratelimited(dev, "Invalid stage-1 address width %d\n",
869− s1_cfg->addr_width);
870− return -EINVAL;
871− }
872−
873− if ((s1_cfg->flags & IOMMU_VTD_S1_SRE) && !ecap_srs(iommu->ecap)) {
874− pr_err_ratelimited("No supervisor request support on %s\n",
875− iommu->name);
876− return -EINVAL;
877− }
878−
879− if ((s1_cfg->flags & IOMMU_VTD_S1_EAFE) && !ecap_eafs(iommu->ecap)) {
880− pr_err_ratelimited("No extended access flag support on %s\n",
881− iommu->name);
882− return -EINVAL;
883− }
884−
885− pasid_pte_config_nestd(iommu, &new_pte, s1_cfg, s2_domain, did);
886−
887− spin_lock(&iommu->lock);
888− pte = intel_pasid_get_entry(dev, pasid);
889− if (!pte) {
890− spin_unlock(&iommu->lock);
891− return -ENODEV;
892− }
893−
894− if (!pasid_pte_is_present(pte)) {
895− spin_unlock(&iommu->lock);
896− return -EINVAL;
897− }
898−
899− WARN_ON(old_did != pasid_get_domain_id(pte));
900−
901− *pte = new_pte;
902− spin_unlock(&iommu->lock);
903−
904− intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
905− intel_iommu_drain_pasid_prq(dev, pasid);
906−
907− return 0;
908−}
909−
910731 /*
911732 * Interfaces to setup or teardown a pasid table to the scalable-mode
912733 * context table entry:
@@ -1019,7 +840,7 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) · 106 unmodified lines
1019840 }
1020841
1021842 if (context_copied(iommu, bus, devfn)) {
1022− context_clear_entry(context);
843+ context_clear_present(context);
1023844 __iommu_flush_cache(iommu, context, sizeof(*context));
1024845
1025846 /*
@@ -1039,6 +860,9 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) · 13 unmodified lines
1039860 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1040861 devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID);
1041862
863+ context_clear_entry(context);
864+ __iommu_flush_cache(iommu, context, sizeof(*context));
865+
1042866 /*
1043867 * At this point, the device is supposed to finish reset at
1044868 * its driver probe stage, so no in-flight DMA will exist,
@@ -1102,6 +926,14 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info) · 57 unmodified lines
1102926 if (!info->ats_enabled)
1103927 return;
1104928
929+ /*
930+ * Skip dev-IOTLB flush for inaccessible PCIe devices to prevent the
931+ * Intel IOMMU from waiting indefinitely for an ATS invalidation that
932+ * cannot complete.
933+ */
934+ if (!pci_device_is_present(to_pci_dev(info->dev)))
935+ return;
936+
1105937 qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn),
1106938 info->pfsid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH);
1107939
drivers/iommu/intel/pasid.h+14−14
@@ -234,9 +234,23 @@ static inline void pasid_set_wpe(struct pasid_entry *pe) · 233 unmodified lines
234234 */
235235 static inline void pasid_set_present(struct pasid_entry *pe)
236236 {
237+ dma_wmb();
237238 pasid_set_bits(&pe->val[0], 1 << 0, 1);
238239 }
239240
241+/*
242+ * Clear the Present (P) bit (bit 0) of a scalable-mode PASID table entry.
243+ * This initiates the transition of the entry's ownership from hardware
244+ * to software. The caller is responsible for fulfilling the invalidation
245+ * handshake recommended by the VT-d spec, Section 6.5.3.3 (Guidance to
246+ * Software for Invalidations).
247+ */
248+static inline void pasid_clear_present(struct pasid_entry *pe)
249+{
250+ pasid_set_bits(&pe->val[0], 1 << 0, 0);
251+ dma_wmb();
252+}
253+
240254 /*
241255 * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID
242256 * entry.
@@ -302,20 +316,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, · 59 unmodified lines
302316 struct device *dev, u32 pasid);
303317 int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
304318 u32 pasid, struct dmar_domain *domain);
305−int intel_pasid_replace_first_level(struct intel_iommu *iommu,
306− struct device *dev, phys_addr_t fsptptr,
307− u32 pasid, u16 did, u16 old_did, int flags);
308−int intel_pasid_replace_second_level(struct intel_iommu *iommu,
309− struct dmar_domain *domain,
310− struct device *dev, u16 old_did,
311− u32 pasid);
312−int intel_pasid_replace_pass_through(struct intel_iommu *iommu,
313− struct device *dev, u16 old_did,
314− u32 pasid);
315−int intel_pasid_replace_nested(struct intel_iommu *iommu,
316− struct device *dev, u32 pasid,
317− u16 old_did, struct dmar_domain *domain);
318−
319319 void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
320320 struct device *dev, u32 pasid,
321321 bool fault_ignore);
drivers/iommu/iommu-debug-pagealloc.c+164−0
@@ -0,0 +1,164 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (C) 2025 - Google Inc
4+ * Author: Mostafa Saleh <smostafa@google.com>
5+ * IOMMU API debug page alloc sanitizer
6+ */
7+#include <linux/atomic.h>
8+#include <linux/iommu.h>
9+#include <linux/iommu-debug-pagealloc.h>
10+#include <linux/kernel.h>
11+#include <linux/page_ext.h>
12+#include <linux/page_owner.h>
13+
14+#include "iommu-priv.h"
15+
16+static bool needed;
17+DEFINE_STATIC_KEY_FALSE(iommu_debug_initialized);
18+
19+struct iommu_debug_metadata {
20+ atomic_t ref;
21+};
22+
23+static __init bool need_iommu_debug(void)
24+{
25+ return needed;
26+}
27+
28+struct page_ext_operations page_iommu_debug_ops = {
29+ .size = sizeof(struct iommu_debug_metadata),
30+ .need = need_iommu_debug,
31+};
32+
33+static struct iommu_debug_metadata *get_iommu_data(struct page_ext *page_ext)
34+{
35+ return page_ext_data(page_ext, &page_iommu_debug_ops);
36+}
37+
38+static void iommu_debug_inc_page(phys_addr_t phys)
39+{
40+ struct page_ext *page_ext = page_ext_from_phys(phys);
41+ struct iommu_debug_metadata *d;
42+
43+ if (!page_ext)
44+ return;
45+
46+ d = get_iommu_data(page_ext);
47+ WARN_ON(atomic_inc_return_relaxed(&d->ref) <= 0);
48+ page_ext_put(page_ext);
49+}
50+
51+static void iommu_debug_dec_page(phys_addr_t phys)
52+{
53+ struct page_ext *page_ext = page_ext_from_phys(phys);
54+ struct iommu_debug_metadata *d;
55+
56+ if (!page_ext)
57+ return;
58+
59+ d = get_iommu_data(page_ext);
60+ WARN_ON(atomic_dec_return_relaxed(&d->ref) < 0);
61+ page_ext_put(page_ext);
62+}
63+
64+/*
65+ * IOMMU page size doesn't have to match the CPU page size. So, we use
66+ * the smallest IOMMU page size to refcount the pages in the vmemmap.
67+ * That is important as both map and unmap has to use the same page size
68+ * to update the refcount to avoid double counting the same page.
69+ * And as we can't know from iommu_unmap() what was the original page size
70+ * used for map, we just use the minimum supported one for both.
71+ */
72+static size_t iommu_debug_page_size(struct iommu_domain *domain)
73+{
74+ return 1UL << __ffs(domain->pgsize_bitmap);
75+}
76+
77+static bool iommu_debug_page_count(const struct page *page)
78+{
79+ unsigned int ref;
80+ struct page_ext *page_ext = page_ext_get(page);
81+ struct iommu_debug_metadata *d = get_iommu_data(page_ext);
82+
83+ ref = atomic_read(&d->ref);
84+ page_ext_put(page_ext);
85+ return ref != 0;
86+}
87+
88+void __iommu_debug_check_unmapped(const struct page *page, int numpages)
89+{
90+ while (numpages--) {
91+ if (WARN_ON(iommu_debug_page_count(page))) {
92+ pr_warn("iommu: Detected page leak!\n");
93+ dump_page_owner(page);
94+ }
95+ page++;
96+ }
97+}
98+
99+void __iommu_debug_map(struct iommu_domain *domain, phys_addr_t phys, size_t size)
100+{
101+ size_t off, end;
102+ size_t page_size = iommu_debug_page_size(domain);
103+
104+ if (WARN_ON(!phys || check_add_overflow(phys, size, &end)))
105+ return;
106+
107+ for (off = 0 ; off < size ; off += page_size)
108+ iommu_debug_inc_page(phys + off);
109+}
110+
111+static void __iommu_debug_update_iova(struct iommu_domain *domain,
112+ unsigned long iova, size_t size, bool inc)
113+{
114+ size_t off, end;
115+ size_t page_size = iommu_debug_page_size(domain);
116+
117+ if (WARN_ON(check_add_overflow(iova, size, &end)))
118+ return;
119+
120+ for (off = 0 ; off < size ; off += page_size) {
121+ phys_addr_t phys = iommu_iova_to_phys(domain, iova + off);
122+
123+ if (!phys)
124+ continue;
125+
126+ if (inc)
127+ iommu_debug_inc_page(phys);
128+ else
129+ iommu_debug_dec_page(phys);
130+ }
131+}
132+
133+void __iommu_debug_unmap_begin(struct iommu_domain *domain,
134+ unsigned long iova, size_t size)
135+{
136+ __iommu_debug_update_iova(domain, iova, size, false);
137+}
138+
139+void __iommu_debug_unmap_end(struct iommu_domain *domain,
140+ unsigned long iova, size_t size,
141+ size_t unmapped)
142+{
143+ if ((unmapped == size) || WARN_ON_ONCE(unmapped > size))
144+ return;
145+
146+ /* If unmap failed, re-increment the refcount. */
147+ __iommu_debug_update_iova(domain, iova + unmapped,
148+ size - unmapped, true);
149+}
150+
151+void iommu_debug_init(void)
152+{
153+ if (!needed)
154+ return;
155+
156+ pr_info("iommu: Debugging page allocations, expect overhead or disable iommu.debug_pagealloc");
157+ static_branch_enable(&iommu_debug_initialized);
158+}
159+
160+static int __init iommu_debug_pagealloc(char *str)
161+{
162+ return kstrtobool(str, &needed);
163+}
164+early_param("iommu.debug_pagealloc", iommu_debug_pagealloc);
drivers/iommu/iommu-priv.h+58−0
@@ -5,6 +5,7 @@ · 4 unmodified lines
55 #define __LINUX_IOMMU_PRIV_H
66
77 #include <linux/iommu.h>
8+#include <linux/iommu-debug-pagealloc.h>
89 #include <linux/msi.h>
910
1011 static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
@@ -65,4 +66,61 @@ static inline int iommufd_sw_msi(struct iommu_domain *domain, · 54 unmodified lines
6566 int iommu_replace_device_pasid(struct iommu_domain *domain,
6667 struct device *dev, ioasid_t pasid,
6768 struct iommu_attach_handle *handle);
69+
70+#ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC
71+
72+void __iommu_debug_map(struct iommu_domain *domain, phys_addr_t phys,
73+ size_t size);
74+void __iommu_debug_unmap_begin(struct iommu_domain *domain,
75+ unsigned long iova, size_t size);
76+void __iommu_debug_unmap_end(struct iommu_domain *domain,
77+ unsigned long iova, size_t size, size_t unmapped);
78+
79+static inline void iommu_debug_map(struct iommu_domain *domain,
80+ phys_addr_t phys, size_t size)
81+{
82+ if (static_branch_unlikely(&iommu_debug_initialized))
83+ __iommu_debug_map(domain, phys, size);
84+}
85+
86+static inline void iommu_debug_unmap_begin(struct iommu_domain *domain,
87+ unsigned long iova, size_t size)
88+{
89+ if (static_branch_unlikely(&iommu_debug_initialized))
90+ __iommu_debug_unmap_begin(domain, iova, size);
91+}
92+
93+static inline void iommu_debug_unmap_end(struct iommu_domain *domain,
94+ unsigned long iova, size_t size,
95+ size_t unmapped)
96+{
97+ if (static_branch_unlikely(&iommu_debug_initialized))
98+ __iommu_debug_unmap_end(domain, iova, size, unmapped);
99+}
100+
101+void iommu_debug_init(void);
102+
103+#else
104+static inline void iommu_debug_map(struct iommu_domain *domain,
105+ phys_addr_t phys, size_t size)
106+{
107+}
108+
109+static inline void iommu_debug_unmap_begin(struct iommu_domain *domain,
110+ unsigned long iova, size_t size)
111+{
112+}
113+
114+static inline void iommu_debug_unmap_end(struct iommu_domain *domain,
115+ unsigned long iova, size_t size,
116+ size_t unmapped)
117+{
118+}
119+
120+static inline void iommu_debug_init(void)
121+{
122+}
123+
124+#endif /* CONFIG_IOMMU_DEBUG_PAGEALLOC */
125+
68126 #endif /* __LINUX_IOMMU_PRIV_H */
drivers/iommu/iommu.c+224−10
@@ -61,6 +61,11 @@ struct iommu_group { · 60 unmodified lines
6161 int id;
6262 struct iommu_domain *default_domain;
6363 struct iommu_domain *blocking_domain;
64+ /*
65+ * During a group device reset, @resetting_domain points to the physical
66+ * domain, while @domain points to the attached domain before the reset.
67+ */
68+ struct iommu_domain *resetting_domain;
6469 struct iommu_domain *domain;
6570 struct list_head entry;
6671 unsigned int owner_cnt;
@@ -232,6 +237,8 @@ static int __init iommu_subsys_init(void) · 165 unmodified lines
232237 if (!nb)
233238 return -ENOMEM;
234239
240+ iommu_debug_init();
241+
235242 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
236243 nb[i].notifier_call = iommu_bus_notifier;
237244 bus_register_notifier(iommu_buses[i], &nb[i]);
@@ -661,7 +668,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list · 423 unmodified lines
661668 }
662669
663670 if (group->default_domain)
664− iommu_setup_dma_ops(dev);
671+ iommu_setup_dma_ops(dev, group->default_domain);
665672
666673 mutex_unlock(&group->mutex);
667674
@@ -1173,12 +1180,11 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, · 505 unmodified lines
11731180 struct device *dev)
11741181 {
11751182 struct iommu_resv_region *entry;
1176− struct list_head mappings;
1183+ LIST_HEAD(mappings);
11771184 unsigned long pg_size;
11781185 int ret = 0;
11791186
11801187 pg_size = domain->pgsize_bitmap ? 1UL << __ffs(domain->pgsize_bitmap) : 0;
1181− INIT_LIST_HEAD(&mappings);
11821188
11831189 if (WARN_ON_ONCE(iommu_is_dma_domain(domain) && !pg_size))
11841190 return -EINVAL;
@@ -1949,7 +1955,7 @@ static int bus_iommu_probe(const struct bus_type *bus) · 764 unmodified lines
19491955 return ret;
19501956 }
19511957 for_each_group_device(group, gdev)
1952− iommu_setup_dma_ops(gdev->dev);
1958+ iommu_setup_dma_ops(gdev->dev, group->default_domain);
19531959 mutex_unlock(&group->mutex);
19541960
19551961 /*
@@ -2185,10 +2191,26 @@ EXPORT_SYMBOL_GPL(iommu_attach_device); · 229 unmodified lines
21852191
21862192 int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
21872193 {
2188− if (dev->iommu && dev->iommu->attach_deferred)
2189− return __iommu_attach_device(domain, dev, NULL);
2194+ /*
2195+ * This is called on the dma mapping fast path so avoid locking. This is
2196+ * racy, but we have an expectation that the driver will setup its DMAs
2197+ * inside probe while being single threaded to avoid racing.
2198+ */
2199+ if (!dev->iommu || !dev->iommu->attach_deferred)
2200+ return 0;
21902201
2191− return 0;
2202+ guard(mutex)(&dev->iommu_group->mutex);
2203+
2204+ /*
2205+ * This is a concurrent attach during a device reset. Reject it until
2206+ * pci_dev_reset_iommu_done() attaches the device to group->domain.
2207+ *
2208+ * Note that this might fail the iommu_dma_map(). But there's nothing
2209+ * more we can do here.
2210+ */
2211+ if (dev->iommu_group->resetting_domain)
2212+ return -EBUSY;
2213+ return __iommu_attach_device(domain, dev, NULL);
21922214 }
21932215
21942216 void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
@@ -2210,6 +2232,15 @@ out_unlock: · 15 unmodified lines
22102232 }
22112233 EXPORT_SYMBOL_GPL(iommu_detach_device);
22122234
2235+/**
2236+ * iommu_get_domain_for_dev() - Return the DMA API domain pointer
2237+ * @dev: Device to query
2238+ *
2239+ * This function can be called within a driver bound to dev. The returned
2240+ * pointer is valid for the lifetime of the bound driver.
2241+ *
2242+ * It should not be called by drivers with driver_managed_dma = true.
2243+ */
22132244 struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
22142245 {
22152246 /* Caller must be a probed driver on dev */
@@ -2218,10 +2249,40 @@ struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) · 2 unmodified lines
22182249 if (!group)
22192250 return NULL;
22202251
2252+ lockdep_assert_not_held(&group->mutex);
2253+
22212254 return group->domain;
22222255 }
22232256 EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
22242257
2258+/**
2259+ * iommu_driver_get_domain_for_dev() - Return the driver-level domain pointer
2260+ * @dev: Device to query
2261+ *
2262+ * This function can be called by an iommu driver that wants to get the physical
2263+ * domain within an iommu callback function where group->mutex is held.
2264+ */
2265+struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev)
2266+{
2267+ struct iommu_group *group = dev->iommu_group;
2268+
2269+ lockdep_assert_held(&group->mutex);
2270+
2271+ /*
2272+ * Driver handles the low-level __iommu_attach_device(), including the
2273+ * one invoked by pci_dev_reset_iommu_done() re-attaching the device to
2274+ * the cached group->domain. In this case, the driver must get the old
2275+ * domain from group->resetting_domain rather than group->domain. This
2276+ * prevents it from re-attaching the device from group->domain (old) to
2277+ * group->domain (new).
2278+ */
2279+ if (group->resetting_domain)
2280+ return group->resetting_domain;
2281+
2282+ return group->domain;
2283+}
2284+EXPORT_SYMBOL_GPL(iommu_driver_get_domain_for_dev);
2285+
22252286 /*
22262287 * For IOMMU_DOMAIN_DMA implementations which already provide their own
22272288 * guarantees that the group and its default domain are valid and correct.
@@ -2374,6 +2435,13 @@ static int __iommu_group_set_domain_internal(struct iommu_group *group, · 146 unmodified lines
23742435 if (WARN_ON(!new_domain))
23752436 return -EINVAL;
23762437
2438+ /*
2439+ * This is a concurrent attach during a device reset. Reject it until
2440+ * pci_dev_reset_iommu_done() attaches the device to group->domain.
2441+ */
2442+ if (group->resetting_domain)
2443+ return -EBUSY;
2444+
23772445 /*
23782446 * Changing the domain is done by calling attach_dev() on the new
23792447 * domain. This switch does not have to be atomic and DMA can be
@@ -2562,10 +2630,12 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, · 182 unmodified lines
25622630 }
25632631
25642632 /* unroll mapping in case something went wrong */
2565− if (ret)
2633+ if (ret) {
25662634 iommu_unmap(domain, orig_iova, orig_size - size);
2567− else
2635+ } else {
25682636 trace_map(orig_iova, orig_paddr, orig_size);
2637+ iommu_debug_map(domain, orig_paddr, orig_size);
2638+ }
25692639
25702640 return ret;
25712641 }
@@ -2627,6 +2697,8 @@ static size_t __iommu_unmap(struct iommu_domain *domain, · 55 unmodified lines
26272697
26282698 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
26292699
2700+ iommu_debug_unmap_begin(domain, iova, size);
2701+
26302702 /*
26312703 * Keep iterating until we either unmap 'size' bytes (or more)
26322704 * or we hit an area that isn't mapped.
@@ -2647,6 +2719,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain, · 14 unmodified lines
26472719 }
26482720
26492721 trace_unmap(orig_iova, size, unmapped);
2722+ iommu_debug_unmap_end(domain, orig_iova, size, unmapped);
26502723 return unmapped;
26512724 }
26522725
@@ -3148,7 +3221,7 @@ static ssize_t iommu_group_store_type(struct iommu_group *group, · 495 unmodified lines
31483221
31493222 /* Make sure dma_ops is appropriatley set */
31503223 for_each_group_device(group, gdev)
3151− iommu_setup_dma_ops(gdev->dev);
3224+ iommu_setup_dma_ops(gdev->dev, group->default_domain);
31523225
31533226 out_unlock:
31543227 mutex_unlock(&group->mutex);
@@ -3492,6 +3565,16 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, · 337 unmodified lines
34923565 return -EINVAL;
34933566
34943567 mutex_lock(&group->mutex);
3568+
3569+ /*
3570+ * This is a concurrent attach during a device reset. Reject it until
3571+ * pci_dev_reset_iommu_done() attaches the device to group->domain.
3572+ */
3573+ if (group->resetting_domain) {
3574+ ret = -EBUSY;
3575+ goto out_unlock;
3576+ }
3577+
34953578 for_each_group_device(group, device) {
34963579 /*
34973580 * Skip PASID validation for devices without PASID support
@@ -3575,6 +3658,16 @@ int iommu_replace_device_pasid(struct iommu_domain *domain, · 77 unmodified lines
35753658 return -EINVAL;
35763659
35773660 mutex_lock(&group->mutex);
3661+
3662+ /*
3663+ * This is a concurrent attach during a device reset. Reject it until
3664+ * pci_dev_reset_iommu_done() attaches the device to group->domain.
3665+ */
3666+ if (group->resetting_domain) {
3667+ ret = -EBUSY;
3668+ goto out_unlock;
3669+ }
3670+
35783671 entry = iommu_make_pasid_array_entry(domain, handle);
35793672 curr = xa_cmpxchg(&group->pasid_array, pasid, NULL,
35803673 XA_ZERO_ENTRY, GFP_KERNEL);
@@ -3832,6 +3925,127 @@ err_unlock: · 251 unmodified lines
38323925 }
38333926 EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");
38343927
3928+/**
3929+ * pci_dev_reset_iommu_prepare() - Block IOMMU to prepare for a PCI device reset
3930+ * @pdev: PCI device that is going to enter a reset routine
3931+ *
3932+ * The PCIe r6.0, sec 10.3.1 IMPLEMENTATION NOTE recommends to disable and block
3933+ * ATS before initiating a reset. This means that a PCIe device during the reset
3934+ * routine wants to block any IOMMU activity: translation and ATS invalidation.
3935+ *
3936+ * This function attaches the device's RID/PASID(s) the group->blocking_domain,
3937+ * setting the group->resetting_domain. This allows the IOMMU driver pausing any
3938+ * IOMMU activity while leaving the group->domain pointer intact. Later when the
3939+ * reset is finished, pci_dev_reset_iommu_done() can restore everything.
3940+ *
3941+ * Caller must use pci_dev_reset_iommu_prepare() with pci_dev_reset_iommu_done()
3942+ * before/after the core-level reset routine, to unset the resetting_domain.
3943+ *
3944+ * Return: 0 on success or negative error code if the preparation failed.
3945+ *
3946+ * These two functions are designed to be used by PCI reset functions that would
3947+ * not invoke any racy iommu_release_device(), since PCI sysfs node gets removed
3948+ * before it notifies with a BUS_NOTIFY_REMOVED_DEVICE. When using them in other
3949+ * case, callers must ensure there will be no racy iommu_release_device() call,
3950+ * which otherwise would UAF the dev->iommu_group pointer.
3951+ */
3952+int pci_dev_reset_iommu_prepare(struct pci_dev *pdev)
3953+{
3954+ struct iommu_group *group = pdev->dev.iommu_group;
3955+ unsigned long pasid;
3956+ void *entry;
3957+ int ret;
3958+
3959+ if (!pci_ats_supported(pdev) || !dev_has_iommu(&pdev->dev))
3960+ return 0;
3961+
3962+ guard(mutex)(&group->mutex);
3963+
3964+ /* Re-entry is not allowed */
3965+ if (WARN_ON(group->resetting_domain))
3966+ return -EBUSY;
3967+
3968+ ret = __iommu_group_alloc_blocking_domain(group);
3969+ if (ret)
3970+ return ret;
3971+
3972+ /* Stage RID domain at blocking_domain while retaining group->domain */
3973+ if (group->domain != group->blocking_domain) {
3974+ ret = __iommu_attach_device(group->blocking_domain, &pdev->dev,
3975+ group->domain);
3976+ if (ret)
3977+ return ret;
3978+ }
3979+
3980+ /*
3981+ * Stage PASID domains at blocking_domain while retaining pasid_array.
3982+ *
3983+ * The pasid_array is mostly fenced by group->mutex, except one reader
3984+ * in iommu_attach_handle_get(), so it's safe to read without xa_lock.
3985+ */
3986+ xa_for_each_start(&group->pasid_array, pasid, entry, 1)
3987+ iommu_remove_dev_pasid(&pdev->dev, pasid,
3988+ pasid_array_entry_to_domain(entry));
3989+
3990+ group->resetting_domain = group->blocking_domain;
3991+ return ret;
3992+}
3993+EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_prepare);
3994+
3995+/**
3996+ * pci_dev_reset_iommu_done() - Restore IOMMU after a PCI device reset is done
3997+ * @pdev: PCI device that has finished a reset routine
3998+ *
3999+ * After a PCIe device finishes a reset routine, it wants to restore its IOMMU
4000+ * IOMMU activity, including new translation as well as cache invalidation, by
4001+ * re-attaching all RID/PASID of the device's back to the domains retained in
4002+ * the core-level structure.
4003+ *
4004+ * Caller must pair it with a successful pci_dev_reset_iommu_prepare().
4005+ *
4006+ * Note that, although unlikely, there is a risk that re-attaching domains might
4007+ * fail due to some unexpected happening like OOM.
4008+ */
4009+void pci_dev_reset_iommu_done(struct pci_dev *pdev)
4010+{
4011+ struct iommu_group *group = pdev->dev.iommu_group;
4012+ unsigned long pasid;
4013+ void *entry;
4014+
4015+ if (!pci_ats_supported(pdev) || !dev_has_iommu(&pdev->dev))
4016+ return;
4017+
4018+ guard(mutex)(&group->mutex);
4019+
4020+ /* pci_dev_reset_iommu_prepare() was bypassed for the device */
4021+ if (!group->resetting_domain)
4022+ return;
4023+
4024+ /* pci_dev_reset_iommu_prepare() was not successfully called */
4025+ if (WARN_ON(!group->blocking_domain))
4026+ return;
4027+
4028+ /* Re-attach RID domain back to group->domain */
4029+ if (group->domain != group->blocking_domain) {
4030+ WARN_ON(__iommu_attach_device(group->domain, &pdev->dev,
4031+ group->blocking_domain));
4032+ }
4033+
4034+ /*
4035+ * Re-attach PASID domains back to the domains retained in pasid_array.
4036+ *
4037+ * The pasid_array is mostly fenced by group->mutex, except one reader
4038+ * in iommu_attach_handle_get(), so it's safe to read without xa_lock.
4039+ */
4040+ xa_for_each_start(&group->pasid_array, pasid, entry, 1)
4041+ WARN_ON(__iommu_set_group_pasid(
4042+ pasid_array_entry_to_domain(entry), group, pasid,
4043+ group->blocking_domain));
4044+
4045+ group->resetting_domain = NULL;
4046+}
4047+EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_done);
4048+
38354049 #if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
38364050 /**
38374051 * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
drivers/pci/pci-acpi.c+11−2
@@ -9,6 +9,7 @@ · 8 unmodified lines
99
1010 #include <linux/delay.h>
1111 #include <linux/init.h>
12+#include <linux/iommu.h>
1213 #include <linux/irqdomain.h>
1314 #include <linux/pci.h>
1415 #include <linux/msi.h>
@@ -971,6 +972,7 @@ void pci_set_acpi_fwnode(struct pci_dev *dev) · 956 unmodified lines
971972 int pci_dev_acpi_reset(struct pci_dev *dev, bool probe)
972973 {
973974 acpi_handle handle = ACPI_HANDLE(&dev->dev);
975+ int ret;
974976
975977 if (!handle || !acpi_has_method(handle, "_RST"))
976978 return -ENOTTY;
@@ -978,12 +980,19 @@ int pci_dev_acpi_reset(struct pci_dev *dev, bool probe) · 1 unmodified line
978980 if (probe)
979981 return 0;
980982
983+ ret = pci_dev_reset_iommu_prepare(dev);
984+ if (ret) {
985+ pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
986+ return ret;
987+ }
988+
981989 if (ACPI_FAILURE(acpi_evaluate_object(handle, "_RST", NULL, NULL))) {
982990 pci_warn(dev, "ACPI _RST failed\n");
983− return -ENOTTY;
991+ ret = -ENOTTY;
984992 }
985993
986− return 0;
994+ pci_dev_reset_iommu_done(dev);
995+ return ret;
987996 }
988997
989998 bool acpi_pci_power_manageable(struct pci_dev *dev)
drivers/pci/pci.c+58−7
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 #include <linux/delay.h>
1414 #include <linux/dmi.h>
1515 #include <linux/init.h>
16+#include <linux/iommu.h>
1617 #include <linux/msi.h>
1718 #include <linux/of.h>
1819 #include <linux/pci.h>
@@ -25,6 +26,7 @@ · 6 unmodified lines
2526 #include <linux/logic_pio.h>
2627 #include <linux/device.h>
2728 #include <linux/pm_runtime.h>
29+#include <linux/pci-ats.h>
2830 #include <linux/pci_hotplug.h>
2931 #include <linux/vmalloc.h>
3032 #include <asm/dma.h>
@@ -4330,13 +4332,22 @@ EXPORT_SYMBOL(pci_wait_for_pending_transaction); · 4299 unmodified lines
43304332 */
43314333 int pcie_flr(struct pci_dev *dev)
43324334 {
4335+ int ret;
4336+
43334337 if (!pci_wait_for_pending_transaction(dev))
43344338 pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
43354339
4340+ /* Have to call it after waiting for pending DMA transaction */
4341+ ret = pci_dev_reset_iommu_prepare(dev);
4342+ if (ret) {
4343+ pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
4344+ return ret;
4345+ }
4346+
43364347 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
43374348
43384349 if (dev->imm_ready)
4339− return 0;
4350+ goto done;
43404351
43414352 /*
43424353 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
@@ -4345,7 +4356,10 @@ int pcie_flr(struct pci_dev *dev) · 2 unmodified lines
43454356 */
43464357 msleep(100);
43474358
4348− return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);
4359+ ret = pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);
4360+done:
4361+ pci_dev_reset_iommu_done(dev);
4362+ return ret;
43494363 }
43504364 EXPORT_SYMBOL_GPL(pcie_flr);
43514365
@@ -4373,6 +4387,7 @@ EXPORT_SYMBOL_GPL(pcie_reset_flr); · 21 unmodified lines
43734387
43744388 static int pci_af_flr(struct pci_dev *dev, bool probe)
43754389 {
4390+ int ret;
43764391 int pos;
43774392 u8 cap;
43784393
@@ -4399,10 +4414,17 @@ static int pci_af_flr(struct pci_dev *dev, bool probe) · 20 unmodified lines
43994414 PCI_AF_STATUS_TP << 8))
44004415 pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
44014416
4417+ /* Have to call it after waiting for pending DMA transaction */
4418+ ret = pci_dev_reset_iommu_prepare(dev);
4419+ if (ret) {
4420+ pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
4421+ return ret;
4422+ }
4423+
44024424 pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
44034425
44044426 if (dev->imm_ready)
4405− return 0;
4427+ goto done;
44064428
44074429 /*
44084430 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
@@ -4412,7 +4434,10 @@ static int pci_af_flr(struct pci_dev *dev, bool probe) · 3 unmodified lines
44124434 */
44134435 msleep(100);
44144436
4415− return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);
4437+ ret = pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);
4438+done:
4439+ pci_dev_reset_iommu_done(dev);
4440+ return ret;
44164441 }
44174442
44184443 /**
@@ -4433,6 +4458,7 @@ static int pci_af_flr(struct pci_dev *dev, bool probe) · 14 unmodified lines
44334458 static int pci_pm_reset(struct pci_dev *dev, bool probe)
44344459 {
44354460 u16 csr;
4461+ int ret;
44364462
44374463 if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
44384464 return -ENOTTY;
@@ -4447,6 +4473,12 @@ static int pci_pm_reset(struct pci_dev *dev, bool probe) · 8 unmodified lines
44474473 if (dev->current_state != PCI_D0)
44484474 return -EINVAL;
44494475
4476+ ret = pci_dev_reset_iommu_prepare(dev);
4477+ if (ret) {
4478+ pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
4479+ return ret;
4480+ }
4481+
44504482 csr &= ~PCI_PM_CTRL_STATE_MASK;
44514483 csr |= PCI_D3hot;
44524484 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
@@ -4457,7 +4489,9 @@ static int pci_pm_reset(struct pci_dev *dev, bool probe) · 4 unmodified lines
44574489 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
44584490 pci_dev_d3_sleep(dev);
44594491
4460− return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
4492+ ret = pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
4493+ pci_dev_reset_iommu_done(dev);
4494+ return ret;
44614495 }
44624496
44634497 /**
@@ -4885,10 +4919,20 @@ static int pci_reset_bus_function(struct pci_dev *dev, bool probe) · 421 unmodified lines
48854919 return -ENOTTY;
48864920 }
48874921
4922+ rc = pci_dev_reset_iommu_prepare(dev);
4923+ if (rc) {
4924+ pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", rc);
4925+ return rc;
4926+ }
4927+
48884928 rc = pci_dev_reset_slot_function(dev, probe);
48894929 if (rc != -ENOTTY)
4890− return rc;
4891− return pci_parent_bus_reset(dev, probe);
4930+ goto done;
4931+
4932+ rc = pci_parent_bus_reset(dev, probe);
4933+done:
4934+ pci_dev_reset_iommu_done(dev);
4935+ return rc;
48924936 }
48934937
48944938 static int cxl_reset_bus_function(struct pci_dev *dev, bool probe)
@@ -4912,6 +4956,12 @@ static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) · 17 unmodified lines
49124956 if (rc)
49134957 return -ENOTTY;
49144958
4959+ rc = pci_dev_reset_iommu_prepare(dev);
4960+ if (rc) {
4961+ pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", rc);
4962+ return rc;
4963+ }
4964+
49154965 if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) {
49164966 val = reg;
49174967 } else {
@@ -4926,6 +4976,7 @@ static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) · 8 unmodified lines
49264976 pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL,
49274977 reg);
49284978
4979+ pci_dev_reset_iommu_done(dev);
49294980 return rc;
49304981 }
49314982
drivers/pci/quirks.c+18−1
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 #include <linux/pci.h>
2222 #include <linux/isa-dma.h> /* isa_dma_bridge_buggy */
2323 #include <linux/init.h>
24+#include <linux/iommu.h>
2425 #include <linux/delay.h>
2526 #include <linux/acpi.h>
2627 #include <linux/dmi.h>
@@ -4228,6 +4229,22 @@ static const struct pci_dev_reset_methods pci_dev_reset_methods[] = { · 4201 unmodified lines
42284229 { 0 }
42294230 };
42304231
4232+static int __pci_dev_specific_reset(struct pci_dev *dev, bool probe,
4233+ const struct pci_dev_reset_methods *i)
4234+{
4235+ int ret;
4236+
4237+ ret = pci_dev_reset_iommu_prepare(dev);
4238+ if (ret) {
4239+ pci_err(dev, "failed to stop IOMMU for a PCI reset: %d\n", ret);
4240+ return ret;
4241+ }
4242+
4243+ ret = i->reset(dev, probe);
4244+ pci_dev_reset_iommu_done(dev);
4245+ return ret;
4246+}
4247+
42314248 /*
42324249 * These device-specific reset methods are here rather than in a driver
42334250 * because when a host assigns a device to a guest VM, the host may need
@@ -4242,7 +4259,7 @@ int pci_dev_specific_reset(struct pci_dev *dev, bool probe) · 8 unmodified lines
42424259 i->vendor == (u16)PCI_ANY_ID) &&
42434260 (i->device == dev->device ||
42444261 i->device == (u16)PCI_ANY_ID))
4245− return i->reset(dev, probe);
4262+ return __pci_dev_specific_reset(dev, probe, i);
42464263 }
42474264
42484265 return -ENOTTY;
include/linux/iommu-debug-pagealloc.h+32−0
@@ -0,0 +1,32 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (C) 2025 - Google Inc
4+ * Author: Mostafa Saleh <smostafa@google.com>
5+ * IOMMU API debug page alloc sanitizer
6+ */
7+
8+#ifndef __LINUX_IOMMU_DEBUG_PAGEALLOC_H
9+#define __LINUX_IOMMU_DEBUG_PAGEALLOC_H
10+
11+#ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC
12+DECLARE_STATIC_KEY_FALSE(iommu_debug_initialized);
13+
14+extern struct page_ext_operations page_iommu_debug_ops;
15+
16+void __iommu_debug_check_unmapped(const struct page *page, int numpages);
17+
18+static inline void iommu_debug_check_unmapped(const struct page *page, int numpages)
19+{
20+ if (static_branch_unlikely(&iommu_debug_initialized))
21+ __iommu_debug_check_unmapped(page, numpages);
22+}
23+
24+#else
25+static inline void iommu_debug_check_unmapped(const struct page *page,
26+ int numpages)
27+{
28+}
29+
30+#endif /* CONFIG_IOMMU_DEBUG_PAGEALLOC */
31+
32+#endif /* __LINUX_IOMMU_DEBUG_PAGEALLOC_H */
include/linux/iommu.h+14−0
@@ -910,6 +910,7 @@ extern int iommu_attach_device(struct iommu_domain *domain, · 909 unmodified lines
910910 extern void iommu_detach_device(struct iommu_domain *domain,
911911 struct device *dev);
912912 extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
913+struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev);
913914 extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
914915 extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
915916 phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
@@ -1187,6 +1188,10 @@ void iommu_detach_device_pasid(struct iommu_domain *domain, · 271 unmodified lines
11871188 struct device *dev, ioasid_t pasid);
11881189 ioasid_t iommu_alloc_global_pasid(struct device *dev);
11891190 void iommu_free_global_pasid(ioasid_t pasid);
1191+
1192+/* PCI device reset functions */
1193+int pci_dev_reset_iommu_prepare(struct pci_dev *pdev);
1194+void pci_dev_reset_iommu_done(struct pci_dev *pdev);
11901195 #else /* CONFIG_IOMMU_API */
11911196
11921197 struct iommu_ops {};
@@ -1510,6 +1515,15 @@ static inline ioasid_t iommu_alloc_global_pasid(struct device *dev) · 317 unmodified lines
15101515 }
15111516
15121517 static inline void iommu_free_global_pasid(ioasid_t pasid) {}
1518+
1519+static inline int pci_dev_reset_iommu_prepare(struct pci_dev *pdev)
1520+{
1521+ return 0;
1522+}
1523+
1524+static inline void pci_dev_reset_iommu_done(struct pci_dev *pdev)
1525+{
1526+}
15131527 #endif /* CONFIG_IOMMU_API */
15141528
15151529 #ifdef CONFIG_IRQ_MSI_IOMMU
include/linux/mm.h+5−0
@@ -36,6 +36,7 @@ · 35 unmodified lines
3636 #include <linux/rcuwait.h>
3737 #include <linux/bitmap.h>
3838 #include <linux/bitops.h>
39+#include <linux/iommu-debug-pagealloc.h>
3940
4041 struct mempolicy;
4142 struct anon_vma;
@@ -4137,12 +4138,16 @@ extern void __kernel_map_pages(struct page *page, int numpages, int enable); · 4095 unmodified lines
41374138 #ifdef CONFIG_DEBUG_PAGEALLOC
41384139 static inline void debug_pagealloc_map_pages(struct page *page, int numpages)
41394140 {
4141+ iommu_debug_check_unmapped(page, numpages);
4142+
41404143 if (debug_pagealloc_enabled_static())
41414144 __kernel_map_pages(page, numpages, 1);
41424145 }
41434146
41444147 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages)
41454148 {
4149+ iommu_debug_check_unmapped(page, numpages);
4150+
41464151 if (debug_pagealloc_enabled_static())
41474152 __kernel_map_pages(page, numpages, 0);
41484153 }
include/linux/page_ext.h+6−0
@@ -93,6 +93,7 @@ static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn) · 92 unmodified lines
9393 #endif
9494
9595 extern struct page_ext *page_ext_get(const struct page *page);
96+extern struct page_ext *page_ext_from_phys(phys_addr_t phys);
9697 extern void page_ext_put(struct page_ext *page_ext);
9798 extern struct page_ext *page_ext_lookup(unsigned long pfn);
9899
@@ -215,6 +216,11 @@ static inline struct page_ext *page_ext_get(const struct page *page) · 116 unmodified lines
215216 return NULL;
216217 }
217218
219+static inline struct page_ext *page_ext_from_phys(phys_addr_t phys)
220+{
221+ return NULL;
222+}
223+
218224 static inline void page_ext_put(struct page_ext *page_ext)
219225 {
220226 }
include/uapi/linux/iommufd.h+39−0
@@ -465,16 +465,27 @@ struct iommu_hwpt_arm_smmuv3 { · 464 unmodified lines
465465 __aligned_le64 ste[2];
466466 };
467467
468+/**
469+ * struct iommu_hwpt_amd_guest - AMD IOMMU guest I/O page table data
470+ * (IOMMU_HWPT_DATA_AMD_GUEST)
471+ * @dte: Guest Device Table Entry (DTE)
472+ */
473+struct iommu_hwpt_amd_guest {
474+ __aligned_u64 dte[4];
475+};
476+
468477 /**
469478 * enum iommu_hwpt_data_type - IOMMU HWPT Data Type
470479 * @IOMMU_HWPT_DATA_NONE: no data
471480 * @IOMMU_HWPT_DATA_VTD_S1: Intel VT-d stage-1 page table
472481 * @IOMMU_HWPT_DATA_ARM_SMMUV3: ARM SMMUv3 Context Descriptor Table
482+ * @IOMMU_HWPT_DATA_AMD_GUEST: AMD IOMMU guest page table
473483 */
474484 enum iommu_hwpt_data_type {
475485 IOMMU_HWPT_DATA_NONE = 0,
476486 IOMMU_HWPT_DATA_VTD_S1 = 1,
477487 IOMMU_HWPT_DATA_ARM_SMMUV3 = 2,
488+ IOMMU_HWPT_DATA_AMD_GUEST = 3,
478489 };
479490
480491 /**
@@ -623,6 +634,32 @@ struct iommu_hw_info_tegra241_cmdqv { · 142 unmodified lines
623634 __u8 __reserved;
624635 };
625636
637+/**
638+ * struct iommu_hw_info_amd - AMD IOMMU device info
639+ *
640+ * @efr : Value of AMD IOMMU Extended Feature Register (EFR)
641+ * @efr2: Value of AMD IOMMU Extended Feature 2 Register (EFR2)
642+ *
643+ * Please See description of these registers in the following sections of
644+ * the AMD I/O Virtualization Technology (IOMMU) Specification.
645+ * (https://docs.amd.com/v/u/en-US/48882_3.10_PUB)
646+ *
647+ * - MMIO Offset 0030h IOMMU Extended Feature Register
648+ * - MMIO Offset 01A0h IOMMU Extended Feature 2 Register
649+ *
650+ * Note: The EFR and EFR2 are raw values reported by hardware.
651+ * VMM is responsible to determine the appropriate flags to be exposed to
652+ * the VM since cetertain features are not currently supported by the kernel
653+ * for HW-vIOMMU.
654+ *
655+ * Current VMM-allowed list of feature flags are:
656+ * - EFR[GTSup, GASup, GioSup, PPRSup, EPHSup, GATS, GLX, PASmax]
657+ */
658+struct iommu_hw_info_amd {
659+ __aligned_u64 efr;
660+ __aligned_u64 efr2;
661+};
662+
626663 /**
627664 * enum iommu_hw_info_type - IOMMU Hardware Info Types
628665 * @IOMMU_HW_INFO_TYPE_NONE: Output by the drivers that do not report hardware
@@ -632,6 +669,7 @@ struct iommu_hw_info_tegra241_cmdqv { · 3 unmodified lines
632669 * @IOMMU_HW_INFO_TYPE_ARM_SMMUV3: ARM SMMUv3 iommu info type
633670 * @IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV: NVIDIA Tegra241 CMDQV (extension for ARM
634671 * SMMUv3) info type
672+ * @IOMMU_HW_INFO_TYPE_AMD: AMD IOMMU info type
635673 */
636674 enum iommu_hw_info_type {
637675 IOMMU_HW_INFO_TYPE_NONE = 0,
@@ -639,6 +677,7 @@ enum iommu_hw_info_type { · 1 unmodified line
639677 IOMMU_HW_INFO_TYPE_INTEL_VTD = 1,
640678 IOMMU_HW_INFO_TYPE_ARM_SMMUV3 = 2,
641679 IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV = 3,
680+ IOMMU_HW_INFO_TYPE_AMD = 4,
642681 };
643682
644683 /**
include/uapi/linux/vfio.h+4−0
@@ -964,6 +964,10 @@ struct vfio_device_bind_iommufd { · 963 unmodified lines
964964 * hwpt corresponding to the given pt_id.
965965 *
966966 * Return: 0 on success, -errno on failure.
967+ *
968+ * When a device is resetting, -EBUSY will be returned to reject any concurrent
969+ * attachment to the resetting device itself or any sibling device in the IOMMU
970+ * group having the resetting device.
967971 */
968972 struct vfio_device_attach_iommufd_pt {
969973 __u32 argsz;
mm/page_ext.c+27−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 #include <linux/page_table_check.h>
1212 #include <linux/rcupdate.h>
1313 #include <linux/pgalloc_tag.h>
14+#include <linux/iommu-debug-pagealloc.h>
1415
1516 /*
1617 * struct page extension
@@ -89,6 +90,9 @@ static struct page_ext_operations *page_ext_ops[] __initdata = { · 72 unmodified lines
8990 #ifdef CONFIG_PAGE_TABLE_CHECK
9091 &page_table_check_ops,
9192 #endif
93+#ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC
94+ &page_iommu_debug_ops,
95+#endif
9296 };
9397
9498 unsigned long page_ext_size;
@@ -534,6 +538,29 @@ struct page_ext *page_ext_get(const struct page *page) · 439 unmodified lines
534538 return page_ext;
535539 }
536540
541+/**
542+ * page_ext_from_phys() - Get the page_ext structure for a physical address.
543+ * @phys: The physical address to query.
544+ *
545+ * This function safely gets the `struct page_ext` associated with a given
546+ * physical address. It performs validation to ensure the address corresponds
547+ * to a valid, online struct page before attempting to access it.
548+ * It returns NULL for MMIO, ZONE_DEVICE, holes and offline memory.
549+ *
550+ * Return: NULL if no page_ext exists for this physical address.
551+ * Context: Any context. Caller may not sleep until they have called
552+ * page_ext_put().
553+ */
554+struct page_ext *page_ext_from_phys(phys_addr_t phys)
555+{
556+ struct page *page = pfn_to_online_page(__phys_to_pfn(phys));
557+
558+ if (!page)
559+ return NULL;
560+
561+ return page_ext_get(page);
562+}
563+
537564 /**
538565 * page_ext_put() - Working with page extended information is done.
539566 * @page_ext: Page extended information received from page_ext_get().
rust/bindings/bindings_helper.h+2−1
@@ -56,9 +56,10 @@ · 55 unmodified lines
5656 #include <linux/fdtable.h>
5757 #include <linux/file.h>
5858 #include <linux/firmware.h>
59−#include <linux/interrupt.h>
6059 #include <linux/fs.h>
6160 #include <linux/i2c.h>
61+#include <linux/interrupt.h>
62+#include <linux/io-pgtable.h>
6263 #include <linux/ioport.h>
6364 #include <linux/jiffies.h>
6465 #include <linux/jump_label.h>
rust/kernel/iommu/mod.rs+5−0
@@ -0,0 +1,5 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+//! Rust support related to IOMMU.
4+
5+pub mod pgtable;
rust/kernel/iommu/pgtable.rs+279−0
@@ -0,0 +1,279 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+//! IOMMU page table management.
4+//!
5+//! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h)
6+
7+use core::{
8+ marker::PhantomData,
9+ ptr::NonNull, //
10+};
11+
12+use crate::{
13+ alloc,
14+ bindings,
15+ device::{
16+ Bound,
17+ Device, //
18+ },
19+ devres::Devres,
20+ error::to_result,
21+ io::PhysAddr,
22+ prelude::*, //
23+};
24+
25+use bindings::io_pgtable_fmt;
26+
27+/// Protection flags used with IOMMU mappings.
28+pub mod prot {
29+ /// Read access.
30+ pub const READ: u32 = bindings::IOMMU_READ;
31+ /// Write access.
32+ pub const WRITE: u32 = bindings::IOMMU_WRITE;
33+ /// Request cache coherency.
34+ pub const CACHE: u32 = bindings::IOMMU_CACHE;
35+ /// Request no-execute permission.
36+ pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC;
37+ /// MMIO peripheral mapping.
38+ pub const MMIO: u32 = bindings::IOMMU_MMIO;
39+ /// Privileged mapping.
40+ pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV;
41+}
42+
43+/// Represents a requested `io_pgtable` configuration.
44+pub struct Config {
45+ /// Quirk bitmask (type-specific).
46+ pub quirks: usize,
47+ /// Valid page sizes, as a bitmask of powers of two.
48+ pub pgsize_bitmap: usize,
49+ /// Input address space size in bits.
50+ pub ias: u32,
51+ /// Output address space size in bits.
52+ pub oas: u32,
53+ /// IOMMU uses coherent accesses for page table walks.
54+ pub coherent_walk: bool,
55+}
56+
57+/// An io page table using a specific format.
58+///
59+/// # Invariants
60+///
61+/// The pointer references a valid io page table.
62+pub struct IoPageTable<F: IoPageTableFmt> {
63+ ptr: NonNull<bindings::io_pgtable_ops>,
64+ _marker: PhantomData<F>,
65+}
66+
67+// SAFETY: `struct io_pgtable_ops` is not restricted to a single thread.
68+unsafe impl<F: IoPageTableFmt> Send for IoPageTable<F> {}
69+// SAFETY: `struct io_pgtable_ops` may be accessed concurrently.
70+unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<F> {}
71+
72+/// The format used by this page table.
73+pub trait IoPageTableFmt: 'static {
74+ /// The value representing this format.
75+ const FORMAT: io_pgtable_fmt;
76+}
77+
78+impl<F: IoPageTableFmt> IoPageTable<F> {
79+ /// Create a new `IoPageTable` as a device resource.
80+ #[inline]
81+ pub fn new(
82+ dev: &Device<Bound>,
83+ config: Config,
84+ ) -> impl PinInit<Devres<IoPageTable<F>>, Error> + '_ {
85+ // SAFETY: Devres ensures that the value is dropped during device unbind.
86+ Devres::new(dev, unsafe { Self::new_raw(dev, config) })
87+ }
88+
89+ /// Create a new `IoPageTable`.
90+ ///
91+ /// # Safety
92+ ///
93+ /// If successful, then the returned `IoPageTable` must be dropped before the device is
94+ /// unbound.
95+ #[inline]
96+ pub unsafe fn new_raw(dev: &Device<Bound>, config: Config) -> Result<IoPageTable<F>> {
97+ let mut raw_cfg = bindings::io_pgtable_cfg {
98+ quirks: config.quirks,
99+ pgsize_bitmap: config.pgsize_bitmap,
100+ ias: config.ias,
101+ oas: config.oas,
102+ coherent_walk: config.coherent_walk,
103+ tlb: &raw const NOOP_FLUSH_OPS,
104+ iommu_dev: dev.as_raw(),
105+ // SAFETY: All zeroes is a valid value for `struct io_pgtable_cfg`.
106+ ..unsafe { core::mem::zeroed() }
107+ };
108+
109+ // SAFETY:
110+ // * The raw_cfg pointer is valid for the duration of this call.
111+ // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer
112+ // as cookie.
113+ // * The caller ensures that the io pgtable does not outlive the device.
114+ let ops = unsafe {
115+ bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut())
116+ };
117+
118+ // INVARIANT: We successfully created a valid page table.
119+ Ok(IoPageTable {
120+ ptr: NonNull::new(ops).ok_or(ENOMEM)?,
121+ _marker: PhantomData,
122+ })
123+ }
124+
125+ /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`.
126+ #[inline]
127+ pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops {
128+ self.ptr.as_ptr()
129+ }
130+
131+ /// Obtain a raw pointer to the underlying `struct io_pgtable`.
132+ #[inline]
133+ pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable {
134+ // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable.
135+ unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) }
136+ }
137+
138+ /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`.
139+ #[inline]
140+ pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg {
141+ // SAFETY: The `raw_pgtable()` method returns a valid pointer.
142+ unsafe { &raw mut (*self.raw_pgtable()).cfg }
143+ }
144+
145+ /// Map a physically contiguous range of pages of the same size.
146+ ///
147+ /// Even if successful, this operation may not map the entire range. In that case, only a
148+ /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In
149+ /// this case, the caller will usually call `map_pages` again for the remaining range.
150+ ///
151+ /// The returned [`Result`] indicates whether an error was encountered while mapping pages.
152+ /// Note that this may return a non-zero length even if an error was encountered. The caller
153+ /// will usually [unmap the relevant pages](Self::unmap_pages) on error.
154+ ///
155+ /// The caller must flush the TLB before using the pgtable to access the newly created mapping.
156+ ///
157+ /// # Safety
158+ ///
159+ /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
160+ /// this `map_pages` operation executes.
161+ /// * This page table must not contain any mapping that overlaps with the mapping created by
162+ /// this call.
163+ /// * If this page table is live, then the caller must ensure that it's okay to access the
164+ /// physical address being mapped for the duration in which it is mapped.
165+ #[inline]
166+ pub unsafe fn map_pages(
167+ &self,
168+ iova: usize,
169+ paddr: PhysAddr,
170+ pgsize: usize,
171+ pgcount: usize,
172+ prot: u32,
173+ flags: alloc::Flags,
174+ ) -> (usize, Result) {
175+ let mut mapped: usize = 0;
176+
177+ // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null.
178+ let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() };
179+
180+ // SAFETY: The safety requirements of this method are sufficient to call `map_pages`.
181+ let ret = to_result(unsafe {
182+ (map_pages)(
183+ self.raw_ops(),
184+ iova,
185+ paddr,
186+ pgsize,
187+ pgcount,
188+ prot as i32,
189+ flags.as_raw(),
190+ &mut mapped,
191+ )
192+ });
193+
194+ (mapped, ret)
195+ }
196+
197+ /// Unmap a range of virtually contiguous pages of the same size.
198+ ///
199+ /// This may not unmap the entire range, and returns the length of the unmapped prefix in
200+ /// bytes.
201+ ///
202+ /// # Safety
203+ ///
204+ /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
205+ /// this `unmap_pages` operation executes.
206+ /// * This page table must contain one or more consecutive mappings starting at `iova` whose
207+ /// total size is `pgcount * pgsize`.
208+ #[inline]
209+ #[must_use]
210+ pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize {
211+ // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null.
212+ let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() };
213+
214+ // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`.
215+ unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) }
216+ }
217+}
218+
219+// For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all
220+// required invalidations using a range. There is no need for it get ARM style invalidation
221+// instructions from the page table code.
222+//
223+// Support for flushing the TLB with ARM style invalidation instructions may be added in the
224+// future.
225+static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops {
226+ tlb_flush_all: Some(rust_tlb_flush_all_noop),
227+ tlb_flush_walk: Some(rust_tlb_flush_walk_noop),
228+ tlb_add_page: None,
229+};
230+
231+#[no_mangle]
232+extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {}
233+
234+#[no_mangle]
235+extern "C" fn rust_tlb_flush_walk_noop(
236+ _iova: usize,
237+ _size: usize,
238+ _granule: usize,
239+ _cookie: *mut core::ffi::c_void,
240+) {
241+}
242+
243+impl<F: IoPageTableFmt> Drop for IoPageTable<F> {
244+ fn drop(&mut self) {
245+ // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this
246+ // destructor runs.
247+ unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) };
248+ }
249+}
250+
251+/// The `ARM_64_LPAE_S1` page table format.
252+pub enum ARM64LPAES1 {}
253+
254+impl IoPageTableFmt for ARM64LPAES1 {
255+ const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt;
256+}
257+
258+impl IoPageTable<ARM64LPAES1> {
259+ /// Access the `ttbr` field of the configuration.
260+ ///
261+ /// This is the physical address of the page table, which may be passed to the device that
262+ /// needs to use it.
263+ ///
264+ /// # Safety
265+ ///
266+ /// The caller must ensure that the device stops using the page table before dropping it.
267+ #[inline]
268+ pub unsafe fn ttbr(&self) -> u64 {
269+ // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
270+ unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr }
271+ }
272+
273+ /// Access the `mair` field of the configuration.
274+ #[inline]
275+ pub fn mair(&self) -> u64 {
276+ // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
277+ unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair }
278+ }
279+}
rust/kernel/lib.rs+1−0
@@ -103,6 +103,7 @@ pub mod id_pool; · 102 unmodified lines
103103 pub mod init;
104104 pub mod io;
105105 pub mod ioctl;
106+pub mod iommu;
106107 pub mod iov;
107108 pub mod irq;
108109 pub mod jump_label;